Commit 7eb2357d authored by LAVENIER's avatar LAVENIER
Browse files

[enh] Extraction: Add 'NOT NULL' and 'NULL' criterion operator

[enh] Extraction RJB: use contractCode to limit trips
parent 7b283f57
......@@ -213,10 +213,10 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
SumarisTableMetadata table = databaseMetadata.getTable(tableName.toLowerCase());
Preconditions.checkNotNull(table);
String whereClauseContent = SumarisTableMetadatas.getSqlWhereClauseContent(table, filter, sheetName, table.getAlias());
String whereClauseContent = SumarisTableMetadatas.getInverseSqlWhereClauseContent(table, filter, sheetName, table.getAlias(), true);
if (StringUtils.isBlank(whereClauseContent)) return 0;
String deleteQuery = table.getDeleteQuery(String.format("NOT(%s)", whereClauseContent));
String deleteQuery = table.getDeleteQuery(whereClauseContent);
return queryUpdate(deleteQuery);
}
......
......@@ -39,7 +39,9 @@ import org.apache.commons.lang3.StringUtils;
import javax.annotation.Nullable;
import java.sql.Types;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.stream.Collectors;
/**
......@@ -68,101 +70,25 @@ public class SumarisTableMetadatas {
public static String getSqlWhereClauseContent(SumarisTableMetadata table, ExtractionFilterVO filter,
String sheetName, String tableAlias) {
return getSqlWhereClauseContent(table, filter, sheetName, tableAlias, false);
return getSqlWhereClauseContent(table, filter, sheetName, tableAlias, false, false);
}
public static String getSqlWhereClauseContent(SumarisTableMetadata table, ExtractionFilterVO filter,
String appliedSheetName, String tableAlias, boolean skipInvalidCriteria) {
if (filter == null || CollectionUtils.isEmpty(filter.getCriteria())) return "";
StringBuilder sql = new StringBuilder();
//StringBuilder criterionSql = new StringBuilder();
StringBuilder logicalOperator = new StringBuilder();
String aliasWithPoint = tableAlias != null ? (tableAlias + ".") : "";
filter.getCriteria().stream()
.filter(criterion -> appliedSheetName == null || appliedSheetName.equals(criterion.getSheetName()))
.forEach(criterion -> {
// Get the column to tripFilter
Preconditions.checkNotNull(criterion.getName());
SumarisColumnMetadata column = table != null ? table.getColumnMetadata(criterion.getName().toLowerCase()) : null;
if (column == null) {
if (appliedSheetName != null && !skipInvalidCriteria) {
throw new SumarisTechnicalException(String.format("Invalid criterion: column '%s' not found in table '%s'", criterion.getName(), table.getName()));
} else {
// Continue (=skip)
}
} else {
//criterionSql.setLength(0); // Reset
//criterionSql.append(aliasWithPoint)
// .append(column.getName());
sql.append(logicalOperator)
.append(aliasWithPoint)
.append(column.getName());
// Affect logical operator, for the next criterion
if (logicalOperator.length() == 0) {
if ("OR".equalsIgnoreCase(StringUtils.trim(filter.getOperator()))) {
logicalOperator.append(" OR ");
} else {
logicalOperator.append(" AND ");
}
}
String appliedSheetName,
String tableAlias,
boolean skipInvalidCriteria) {
return getSqlWhereClauseContent(table, filter, appliedSheetName, tableAlias, skipInvalidCriteria, false);
}
ExtractionFilterOperatorEnum operator = criterion.getOperator() == null ? ExtractionFilterOperatorEnum.EQUALS : ExtractionFilterOperatorEnum.fromSymbol(criterion.getOperator());
public static String getInverseSqlWhereClauseContent(SumarisTableMetadata table, ExtractionFilterVO filter,
String appliedSheetName,
String tableAlias,
boolean skipInvalidCriteria) {
return getSqlWhereClauseContent(table, filter, appliedSheetName, tableAlias, skipInvalidCriteria, true /*inverse*/);
}
if (criterion.getValue() == null && ArrayUtils.isEmpty(criterion.getValues())) {
switch (operator) {
case NOT_IN:
case NOT_EQUALS:
sql.append(" IS NOT NULL");
break;
default:
sql.append(" IS NULL");
}
} else {
switch (operator) {
case IN:
sql.append(String.format(" IN (%s)", getInValues(column, criterion)));
break;
case NOT_IN:
sql.append(String.format(" NOT IN (%s)", getInValues(column, criterion)));
break;
case EQUALS:
sql.append(String.format(" = %s", getSingleValue(column, criterion)));
break;
case NOT_EQUALS:
sql.append(String.format(" <> %s", getSingleValue(column, criterion)));
break;
case LESS_THAN:
sql.append(String.format(" < %s", getSingleValue(column, criterion)));
break;
case LESS_THAN_OR_EQUALS:
sql.append(String.format(" <= %s", getSingleValue(column, criterion)));
break;
case GREATER_THAN:
sql.append(String.format(" > %s", getSingleValue(column, criterion)));
break;
case GREATER_THAN_OR_EQUALS:
sql.append(String.format(" >= %s", getSingleValue(column, criterion)));
break;
case BETWEEN:
sql.append(String.format(" BETWEEN %s AND %s", getBetweenValueByIndex(column, criterion, 0), getBetweenValueByIndex(column, criterion, 1)));
break;
}
}
//sql.append(logicalOperator.toString()).append(criterionSql);
}
});
return sql.toString();
}
public static String getSingleValue(SumarisColumnMetadata column, ExtractionFilterCriterionVO criterion) {
if (isDateColumn(column)) return Daos.getSqlToDate(Dates.fromISODateTimeString(criterion.getValue()));
......@@ -188,6 +114,14 @@ public class SumarisTableMetadatas {
Preconditions.checkArgument(criterion.getValues().length == 2, "Invalid criterion: 'values' array must have 2 values, for operator 'BETWEEN'");
Preconditions.checkArgument(index == 0 || index == 1);
String value = criterion.getValues()[index];
if (isDateColumn(column)) {
// Parse date
Date date = Dates.safeParseDate(value, Dates.ISO_TIMESTAMP_SPEC, "yyyy-MM-dd");
Preconditions.checkNotNull(date, String.format("Invalid criterion value '%s'. Expected a date (ISO format, or 'YYYY-MM-DD')", value));
// Return TO_DATE(...)
return Daos.getSqlToDate(date);
}
return isNumericColumn(column) ? value : ("'" + value + "'");
}
......@@ -272,4 +206,159 @@ public class SumarisTableMetadatas {
if (StringUtils.isBlank(tableAlias)) return columnNames;
return columnNames.stream().map(c -> tableAlias + "." + c).collect(Collectors.toList());
}
/* -- protected functions -- */
protected static String getSqlWhereClauseContent(SumarisTableMetadata table, ExtractionFilterVO filter,
String appliedSheetName,
String tableAlias,
boolean skipInvalidCriteria,
boolean inverseTheLogic) {
if (filter == null || CollectionUtils.isEmpty(filter.getCriteria())) return "";
StringBuilder sql = new StringBuilder();
StringBuilder logicalOperator = new StringBuilder();
String aliasWithPoint = tableAlias != null ? (tableAlias + ".") : "";
filter.getCriteria().stream()
.filter(criterion -> appliedSheetName == null || appliedSheetName.equals(criterion.getSheetName()))
.forEach(criterion -> {
// Get the column to tripFilter
Preconditions.checkNotNull(criterion.getName());
SumarisColumnMetadata column = table != null ? table.getColumnMetadata(criterion.getName().toLowerCase()) : null;
if (column == null) {
if (appliedSheetName != null && !skipInvalidCriteria) {
throw new SumarisTechnicalException(String.format("Invalid criterion: column '%s' not found in table '%s'", criterion.getName(), table.getName()));
} else {
// Continue (=skip)
}
} else {
String columnAndAlias = aliasWithPoint + column.getName();
// Append logical operator (between criterion)
sql.append(logicalOperator);
ExtractionFilterOperatorEnum operator = criterion.getOperator() == null ? ExtractionFilterOperatorEnum.EQUALS : ExtractionFilterOperatorEnum.fromSymbol(criterion.getOperator());
// Inverse the logic, if need
if (inverseTheLogic) operator = operator.inverse();
boolean skipCriterion = false;
// If no value: use NULL or NOT_NULL
if (criterion.getValue() == null && ArrayUtils.isEmpty(criterion.getValues())) {
switch (operator) {
case NOT_IN:
case NOT_EQUALS:
case NOT_BETWEEN:
operator = ExtractionFilterOperatorEnum.NOT_NULL;
break;
default:
operator = ExtractionFilterOperatorEnum.NULL;
}
}
switch (operator) {
case IN:
sql.append(columnAndAlias)
.append(" IN (")
.append(getInValues(column, criterion))
.append(")");
break;
case NOT_IN:
sql.append("(")
.append(columnAndAlias)
.append(" NOT IN (")
.append(getInValues(column, criterion))
.append(")")
.append(column.isNullable() ? " OR " + columnAndAlias + " IS NULL" : "")
.append(")");
break;
case EQUALS:
sql.append(columnAndAlias)
.append(" = ")
.append(getSingleValue(column, criterion));
break;
case NOT_EQUALS:
sql.append("(")
.append(columnAndAlias)
.append(" <> ")
.append(getSingleValue(column, criterion))
.append(column.isNullable() ? " OR " + columnAndAlias + " IS NULL" : "")
.append(")");
break;
case LESS_THAN:
sql.append(columnAndAlias)
.append(" < ").append(getSingleValue(column, criterion));
break;
case LESS_THAN_OR_EQUALS:
sql.append(columnAndAlias)
.append(" <= ").append(getSingleValue(column, criterion));
break;
case GREATER_THAN:
sql.append(columnAndAlias)
.append(" > ").append(getSingleValue(column, criterion));
break;
case GREATER_THAN_OR_EQUALS:
sql.append(columnAndAlias)
.append(" >= ").append(getSingleValue(column, criterion));
break;
case BETWEEN:
sql.append(columnAndAlias)
.append(" BETWEEN ")
.append(getBetweenValueByIndex(column, criterion, 0))
.append(" AND ").append(getBetweenValueByIndex(column, criterion, 1));
break;
case NOT_BETWEEN:
sql
.append("(")
.append(columnAndAlias)
.append(" < ")
.append(getBetweenValueByIndex(column, criterion, 0))
.append(" OR ")
.append(columnAndAlias)
.append(" > ")
.append(getBetweenValueByIndex(column, criterion, 1))
.append(column.isNullable() ? " OR " + columnAndAlias + " IS NULL" : "")
.append(")");
break;
case NULL:
sql.append(columnAndAlias).append(" IS NULL");
break;
case NOT_NULL:
sql.append(columnAndAlias).append(" IS NOT NULL");
break;
default:
if (!skipInvalidCriteria) {
throw new SumarisTechnicalException(String.format("Invalid criterion: column '%s' has invalid operator '%s' for value '%s'",
criterion.getName(),
operator.getSymbol(),
criterion.getValue() != null ? criterion.getValue() : Arrays.toString(criterion.getValues())));
}
skipCriterion = true;
break;
}
// Init the logical operator, for next iteration
if (!skipCriterion && logicalOperator.length() == 0) {
boolean isOR = "OR".equalsIgnoreCase(StringUtils.trim(filter.getOperator()));
// Inverse the logic, if need
if (inverseTheLogic) isOR = !isOR;
if (isOR) {
logicalOperator.append(" OR ");
} else {
logicalOperator.append(" AND ");
}
}
}
});
return sql.toString();
}
}
......@@ -26,8 +26,8 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.dao.technical.DatabaseType;
import net.sumaris.core.dao.technical.SortDirection;
import net.sumaris.core.dao.technical.schema.SumarisTableMetadata;
......@@ -35,9 +35,7 @@ import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.dao.technical.Daos;
import net.sumaris.core.extraction.dao.technical.ExtractionBaseDaoImpl;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.technical.schema.SumarisTableMetadatas;
import net.sumaris.core.extraction.dao.technical.table.ExtractionTableDao;
import net.sumaris.core.extraction.dao.trip.AggregationTripDao;
import net.sumaris.core.extraction.dao.trip.ExtractionTripDao;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.AggRdbSpecification;
......@@ -61,7 +59,10 @@ import org.springframework.stereotype.Repository;
import javax.persistence.PersistenceException;
import java.io.IOException;
import java.net.URL;
import java.util.*;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
......@@ -826,20 +827,6 @@ public class AggregationRdbTripDaoImpl<
}
}
protected int cleanRow(String tableName, ExtractionFilterVO filter, String sheetName) {
Preconditions.checkNotNull(tableName);
if (filter == null) return 0;
SumarisTableMetadata table = databaseMetadata.getTable(tableName);
Preconditions.checkNotNull(table);
String whereClauseContent = SumarisTableMetadatas.getSqlWhereClauseContent(table, filter, sheetName, table.getAlias(), true);
if (StringUtils.isBlank(whereClauseContent)) return 0;
String deleteQuery = table.getDeleteQuery(String.format("NOT(%s)", whereClauseContent));
return queryUpdate(deleteQuery);
}
protected Map<String, List<String>> analyzeRow(final String tableName, XMLQuery xmlQuery,
String... includedNumericColumnNames) {
return analyzeRow(tableName, xmlQuery, includedNumericColumnNames, true);
......
......@@ -376,6 +376,11 @@ public class ExtractionRdbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
long count = countFrom(tableName);
// Clean row using generic filter
if (count > 0) {
cleanRow(tableName, context.getFilter(), context.getSpeciesListSheetName());
}
// Add result table to context
if (count > 0) {
context.addTableName(tableName,
......@@ -405,28 +410,28 @@ public class ExtractionRdbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
}
protected long createSpeciesLengthTable(C context) {
String tableName = context.getSpeciesLengthTableName();
XMLQuery xmlQuery = createSpeciesLengthQuery(context);
// aggregate insertion
execute(xmlQuery);
long count = countFrom(context.getSpeciesLengthTableName());
long count = countFrom(tableName);
// Clean row using generic tripFilter
if (count > 0) {
count -= cleanRow(context.getSpeciesLengthTableName(), context.getFilter(), context.getSpeciesLengthSheetName());
count -= cleanRow(tableName, context.getFilter(), context.getSpeciesLengthSheetName());
}
// Add result table to context
if (count > 0) {
context.addTableName(context.getSpeciesLengthTableName(),
context.addTableName(tableName,
context.getSpeciesLengthSheetName(),
xmlQuery.getHiddenColumnNames(),
xmlQuery.hasDistinctOption());
log.debug(String.format("Species length table: %s rows inserted", count));
}
else {
context.addRawTableName(context.getSpeciesLengthTableName());
context.addRawTableName(tableName);
}
return count;
}
......
......@@ -72,16 +72,13 @@ public class ExtractionRjbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionTripTable"));
/* // Bind some referential ids
// Bind some referential ids
xmlQuery.bind("contractCodePmfmIds", Daos.getSqlInNumbers(
PmfmEnum.CONTRACT_CODE.getId(),
PmfmEnum.SELF_SAMPLING_PROGRAM.getId()
));
// Bind some referential ids
xmlQuery.bind("contractCode", "RJB"); // TODO externalize*/
// TODO externalize
xmlQuery.bind("taxonGroupLabels", Daos.getSqlInEscapedStrings("RJB_1", "RJB_2"));
xmlQuery.bind("contractCodeLike", "%RJB"); // TODO externalize
return xmlQuery;
}
......
......@@ -47,7 +47,9 @@ public enum ProductFormatEnum implements IExtractionFormat {
// Aggregation product
AGG_RDB (AggRdbSpecification.FORMAT, AggRdbSpecification.SHEET_NAMES, AggRdbSpecification.VERSION_1_3),
AGG_COST (AggCostSpecification.FORMAT, AggCostSpecification.SHEET_NAMES, AggCostSpecification.VERSION_1_4),
AGG_SURVIVAL_TEST (AggSurvivalTestSpecification.FORMAT, AggSurvivalTestSpecification.SHEET_NAMES, AggSurvivalTestSpecification.VERSION_1_0)
AGG_SURVIVAL_TEST (AggSurvivalTestSpecification.FORMAT, AggSurvivalTestSpecification.SHEET_NAMES, AggSurvivalTestSpecification.VERSION_1_0),
//AGG_RJB (AggRjbSpecification.FORMAT, AggRjbSpecification.SHEET_NAMES, AggRjbSpecification.VERSION_1_0)
;
private String label;
......
......@@ -23,25 +23,32 @@ package net.sumaris.core.extraction.vo;
*/
import com.google.common.base.Preconditions;
import net.sumaris.core.exception.SumarisTechnicalException;
import java.util.Arrays;
public enum ExtractionFilterOperatorEnum {
IN("IN"),
NOT_IN("NOT IN"),
EQUALS("="),
NOT_EQUALS("!="),
GREATER_THAN(">"),
GREATER_THAN_OR_EQUALS(">="),
LESS_THAN("<"),
LESS_THAN_OR_EQUALS("<="),
BETWEEN("BETWEEN");
IN("IN", "NOT IN"),
NOT_IN("NOT IN", "IN"),
EQUALS("=", "!="),
NOT_EQUALS("!=", "="),
GREATER_THAN(">", "<="),
GREATER_THAN_OR_EQUALS(">=", "<"),
LESS_THAN("<", ">="),
LESS_THAN_OR_EQUALS("<=", ">"),
BETWEEN("BETWEEN", "NOT BETWEEN"),
NOT_BETWEEN("NOT BETWEEN", "BETWEEN"), // WARN not exists in SQL. Should be translated
NULL("NULL", "NOT NULL"),
NOT_NULL("NOT NULL", "NULL") // WARN not exists in SQL. Should be translated
;
private String symbol;
private String inverseSymbol;
ExtractionFilterOperatorEnum(String symbol) {
ExtractionFilterOperatorEnum(String symbol, String inverseSymbol) {
this.symbol = symbol;
this.inverseSymbol = inverseSymbol;
}
public String getSymbol() {
......@@ -53,8 +60,11 @@ public enum ExtractionFilterOperatorEnum {
return Arrays.stream(values())
.filter(op -> op.symbol.equalsIgnoreCase(operator))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Unknown operation symbol"));
.orElseThrow(() -> new IllegalArgumentException(String.format("Invalid operator's symbol '%s'", operator)));
}
public ExtractionFilterOperatorEnum inverse() {
return fromSymbol(inverseSymbol);
}
}
......@@ -61,8 +61,8 @@
<!-- other fields -->
<select alias="WEIGHT" type="number">SUM(SL.WEIGHT)</select>
<select alias="SUBSAMPLING_WEIGHT" type="number">SUM(SL.SUBSAMPLING_WEIGHT)</select>
<select alias="WEIGHT" group="weight" type="number">SUM(SL.WEIGHT)</select>
<select alias="SUBSAMPLING_WEIGHT" group="weight" type="number">SUM(SL.SUBSAMPLING_WEIGHT)</select>
<select alias="LENGTH_CODE" type="text">SL.LENGTH_CODE</select>
......
......@@ -21,24 +21,9 @@
-->
<query type="select">
<with alias="SELECTED_TRIPS">
<subquery>
<subselect alias="ID" type="number">ID</subselect>
<from alias="T">TRIP</from>
<where>EXISTS (
SELECT O.ID
FROM OPERATION O, BATCH B, TAXON_GROUP TG
WHERE
O.TRIP_FK = T.ID
AND O.ID = B.OPERATION_FK
AND TG.LABEL IN (&amp;taxonGroupLabels)
AND TG.ID=B.TAXON_GROUP_FK
)</where>
</subquery>
</with>
<from join="true">INNER JOIN SELECTED_TRIPS ST ON ST.ID = T.ID</from>
<from join="true">LEFT OUTER JOIN VESSEL_USE_MEASUREMENT VUM_CONTRACT ON VUM_CONTRACT.TRIP_FK = T.ID AND VUM_CONTRACT.PMFM_FK IN (&amp;contractCodePmfmIds)</from>
<from join="true">LEFT OUTER JOIN QUALITATIVE_VALUE QV_CONTRACT ON QV_CONTRACT.ID = VUM_CONTRACT.QUALITATIVE_VALUE_FK</from>
<where operator="AND" group="contractCodeFilter">QV_CONTRACT.LABEL like '&amp;contractCodeLike'</where>
</query>
......@@ -47,6 +47,7 @@ import org.springframework.context.event.EventListener;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.stereotype.Service;
......@@ -82,7 +83,9 @@ public class TrashServiceImpl implements TrashService {