Commit cb901976 authored by LAVENIER's avatar LAVENIER
Browse files

Merge branch 'release/1.6.2'

parents 0bc0d086 bbd693db
......@@ -3,7 +3,7 @@
<groupId>net.sumaris</groupId>
<artifactId>sumaris-pod</artifactId>
<version>1.6.1</version>
<version>1.6.2</version>
<packaging>pom</packaging>
<name>SUMARiS</name>
<description>SUMARiS :: Maven parent</description>
......
......@@ -5,7 +5,7 @@
<parent>
<groupId>net.sumaris</groupId>
<artifactId>sumaris-pod</artifactId>
<version>1.6.1</version>
<version>1.6.2</version>
</parent>
<artifactId>sumaris-core-extraction</artifactId>
......
......@@ -25,6 +25,7 @@ package net.sumaris.core.extraction.dao.technical.table;
import net.sumaris.core.dao.technical.SortDirection;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.ExtractionResultVO;
import net.sumaris.core.extraction.vo.MinMaxVO;
import net.sumaris.core.vo.technical.extraction.ExtractionTableColumnFetchOptions;
import net.sumaris.core.vo.technical.extraction.ExtractionTableColumnVO;
......@@ -60,12 +61,18 @@ public interface ExtractionTableDao {
Map<String, SQLAggregatedFunction> otherColumnNames,
int offset, int size, String sort, SortDirection direction);
Map<String, Object> getTechRows(String tableName,
ExtractionFilterVO filter,
String aggColumnName,
SQLAggregatedFunction aggFunction,
String techColumnName,
String sort, SortDirection direction);
Map<String, Object> getAggByTechRows(String tableName,
ExtractionFilterVO filter,
String aggColumnName,
SQLAggregatedFunction aggFunction,
String techColumnName,
String sort, SortDirection direction);
MinMaxVO getAggMinMaxByTech(String tableName, ExtractionFilterVO filter,
Set<String> groupByColumns,
String aggColumnName,
SQLAggregatedFunction aggFunction,
String techColumnName);
void dropTable(String tableName);
......
......@@ -26,8 +26,10 @@ package net.sumaris.core.extraction.dao.technical.table;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import lombok.NonNull;
import net.sumaris.core.dao.technical.SortDirection;
import net.sumaris.core.dao.technical.schema.SumarisColumnMetadata;
import net.sumaris.core.dao.technical.schema.SumarisDatabaseMetadata;
import net.sumaris.core.dao.technical.schema.SumarisTableMetadata;
import net.sumaris.core.exception.SumarisTechnicalException;
......@@ -38,6 +40,7 @@ import net.sumaris.core.extraction.dao.technical.schema.SumarisTableMetadatas;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.ExtractionResultVO;
import net.sumaris.core.extraction.util.ExtractionProducts;
import net.sumaris.core.extraction.vo.MinMaxVO;
import net.sumaris.core.vo.technical.extraction.ExtractionTableColumnFetchOptions;
import net.sumaris.core.vo.technical.extraction.ExtractionTableColumnVO;
import org.apache.commons.lang3.StringUtils;
......@@ -218,12 +221,19 @@ public class ExtractionTableDaoImpl extends ExtractionBaseDaoImpl implements Ext
SQLAggregatedFunction sqlAggregatedFunction = otherColumnNames.get(c);
if (sqlAggregatedFunction == null) {
return SumarisTableMetadatas.getAliasedColumnName(tableAlias, c);
} else {
// Avoid null value, when using aggregated function
//whereBuilder.append(String.format(" AND %s IS NOT NULL", SumarisTableMetadatas.getAliasedColumnName(tableAlias, c)));
return String.format("%s(COALESCE(%s, 0))",
}
else {
SumarisColumnMetadata column = table.getColumnMetadata(c);
// If nullable column: replace null values will zero
if (column == null || column.isNullable()) {
// TODO: make sure this replacement will NOT affect the AVG function
return String.format("%s(COALESCE(%s, 0))",
sqlAggregatedFunction.name().toLowerCase(),
SumarisTableMetadatas.getAliasedColumnName(tableAlias, c));
}
return String.format("%s(%s))",
sqlAggregatedFunction.name().toLowerCase(),
SumarisTableMetadatas.getAliasedColumnName(tableAlias, c));
}
......@@ -262,18 +272,16 @@ public class ExtractionTableDaoImpl extends ExtractionBaseDaoImpl implements Ext
return result;
}
@Override
public Map<String, Object> getTechRows(String tableName, ExtractionFilterVO filter,
String aggColumnName,
SQLAggregatedFunction aggFunction,
String techColumnName,
String sortColumn, SortDirection direction) {
public Map<String, Object> getAggByTechRows(String tableName, ExtractionFilterVO filter,
String aggColumnName,
SQLAggregatedFunction aggFunction,
String techColumnName,
String sortColumn, SortDirection direction) {
SumarisTableMetadata table = databaseMetadata.getTable(tableName);
Preconditions.checkNotNull(table, "Unknown table: " + tableName);
final String tableAlias = table != null ? table.getAlias() : null;
final String tableAlias = table.getAlias();
String tableWithAlias = SumarisTableMetadatas.getAliasedTableName(tableAlias, tableName);
String aggColumnNameWithFunction = String.format("%s(COALESCE(%s, 0)) AGG_VALUE",
aggFunction.name().toLowerCase(),
......@@ -281,13 +289,15 @@ public class ExtractionTableDaoImpl extends ExtractionBaseDaoImpl implements Ext
String whereClause = SumarisTableMetadatas.getSqlWhereClause(table, filter);
techColumnName = SumarisTableMetadatas.getAliasedColumnName(tableAlias, techColumnName);
direction = direction != null || StringUtils.isNotBlank(sortColumn) ? direction : SortDirection.DESC;
sortColumn = StringUtils.isNotBlank(sortColumn) ? SumarisTableMetadatas.getAliasedColumnName(tableAlias, sortColumn) : "AGG_VALUE";
String sql = SumarisTableMetadatas.getSelectGroupByQuery(
tableWithAlias,
// Select
ImmutableList.of(
SumarisTableMetadatas.getAliasedColumnName(tableAlias, techColumnName),
techColumnName,
aggColumnNameWithFunction),
whereClause,
// Group by
......@@ -302,6 +312,52 @@ public class ExtractionTableDaoImpl extends ExtractionBaseDaoImpl implements Ext
.collect(Collectors.toMap(row -> row[0].toString(), row -> row[1]));
}
public MinMaxVO getAggMinMaxByTech(String tableName, ExtractionFilterVO filter,
Set<String> groupByColumns,
String aggColumnName,
SQLAggregatedFunction aggFunction,
String techColumnName) {
SumarisTableMetadata table = databaseMetadata.getTable(tableName);
Preconditions.checkNotNull(table, "Unknown table: " + tableName);
final String tableAlias = table.getAlias();
String tableWithAlias = SumarisTableMetadatas.getAliasedTableName(tableAlias, tableName);
String aggValueColumnName = "AGG_VALUE";
String aggColumnNameWithFunction = String.format("%s(COALESCE(%s, 0)) %s",
aggFunction.name().toLowerCase(),
SumarisTableMetadatas.getAliasedColumnName(tableAlias, aggColumnName),
aggValueColumnName);
String whereClause = SumarisTableMetadatas.getSqlWhereClause(table, filter);
techColumnName = SumarisTableMetadatas.getAliasedColumnName(tableAlias, techColumnName);
String groupBySql = SumarisTableMetadatas.getSelectGroupByQuery(
tableWithAlias,
// Select
ImmutableSet.<String>builder()
.add(aggColumnNameWithFunction)
.addAll(SumarisTableMetadatas.getAliasedColumns(tableAlias, groupByColumns))
.add(techColumnName)
.build(),
whereClause,
// Group by (tech + times)
ImmutableSet.<String>builder()
.addAll(SumarisTableMetadatas.getAliasedColumns(tableAlias, groupByColumns))
.add(techColumnName)
.build(),
null, null);
String sql = String.format("SELECT min(%s), max(%s) FROM (%s)", aggValueColumnName, aggValueColumnName, groupBySql);
return query(sql, Object[].class)
.stream().findFirst()
.map(row -> MinMaxVO.builder()
.min(Double.valueOf(row[0].toString()))
.max(Double.valueOf(row[1].toString()))
.build())
.orElseGet(() -> new MinMaxVO(0d, 0d));
}
@Override
public long getRowCount(String tableName) {
Preconditions.checkNotNull(tableName);
......
......@@ -37,9 +37,11 @@ public interface AggregationRdbTripDao<C extends AggregationRdbTripContextVO,
<R extends C> R aggregate(ExtractionProductVO source, F filter, S strata);
AggregationResultVO read(String tableName, F filter, S strata, int offset, int size, String sortAttribute, SortDirection sortDirection);
AggregationResultVO getAggBySpace(String tableName, F filter, S strata, int offset, int size, String sortAttribute, SortDirection sortDirection);
Map<String, Object> readTech(String tableName, F filter, S strata, String sortAttribute, SortDirection direction);
AggregationTechResultVO getAggByTech(String tableName, F filter, S strata, String sortAttribute, SortDirection direction);
MinMaxVO getAggMinMaxByTech(String tableName, F filter, S strata);
<R extends C> void clean(R context);
......
......@@ -167,9 +167,9 @@ public class AggregationRdbTripDaoImpl<
}
@Override
public AggregationResultVO read(String tableName, F filter, S strata,
int offset, int size,
String sortAttribute, SortDirection direction) {
public AggregationResultVO getAggBySpace(String tableName, F filter, S strata,
int offset, int size,
String sortAttribute, SortDirection direction) {
Preconditions.checkNotNull(tableName);
Preconditions.checkNotNull(strata);
......@@ -199,11 +199,12 @@ public class AggregationRdbTripDaoImpl<
}
@Override
public Map<String, Object> readTech(@NonNull String tableName, @NonNull F filter, @NonNull S strata,
String sortAttribute, SortDirection direction) {
public AggregationTechResultVO getAggByTech(@NonNull String tableName, @NonNull F filter, @NonNull S strata,
String sortAttribute, SortDirection direction) {
Preconditions.checkNotNull(strata.getTechColumnName(), String.format("Missing 'strata.%s'", AggregationStrataVO.Fields.TECH_COLUMN_NAME));
Preconditions.checkNotNull(strata.getAggColumnName(), String.format("Missing 'strata.%s'", AggregationStrataVO.Fields.AGG_COLUMN_NAME));
AggregationTechResultVO result = new AggregationTechResultVO();
SumarisTableMetadata table = databaseMetadata.getTable(tableName);
......@@ -211,13 +212,35 @@ public class AggregationRdbTripDaoImpl<
Map.Entry<String, ExtractionTableDao.SQLAggregatedFunction> aggColumn = aggColumns.entrySet().stream().findFirst()
.orElseThrow(() -> new IllegalArgumentException(String.format("Missing 'strata.%s'", AggregationStrataVO.Fields.AGG_COLUMN_NAME)));
return extractionTableDao.getTechRows(tableName, filter,
result.setData(extractionTableDao.getAggByTechRows(tableName, filter,
aggColumn.getKey(),
aggColumn.getValue(),
strata.getTechColumnName(),
sortAttribute, direction);
sortAttribute, direction));
return result;
}
@Override
public MinMaxVO getAggMinMaxByTech(String tableName, F filter, S strata) {
Preconditions.checkNotNull(strata.getTechColumnName(), String.format("Missing 'strata.%s'", AggregationStrataVO.Fields.TECH_COLUMN_NAME));
Preconditions.checkNotNull(strata.getAggColumnName(), String.format("Missing 'strata.%s'", AggregationStrataVO.Fields.AGG_COLUMN_NAME));
AggregationTechResultVO result = new AggregationTechResultVO();
SumarisTableMetadata table = databaseMetadata.getTable(tableName);
Map<String, ExtractionTableDao.SQLAggregatedFunction> aggColumns = getAggColumnNames(table, strata);
Map.Entry<String, ExtractionTableDao.SQLAggregatedFunction> aggColumn = aggColumns.entrySet().stream().findFirst()
.orElseThrow(() -> new IllegalArgumentException(String.format("Missing 'strata.%s'", AggregationStrataVO.Fields.AGG_COLUMN_NAME)));
Set<String> timeColumnNames = getGroupByTimesColumnNames(strata.getTimeColumnName());
return extractionTableDao.getAggMinMaxByTech(tableName, filter,
timeColumnNames,
aggColumn.getKey(),
aggColumn.getValue(),
strata.getTechColumnName());
}
@Override
public <R extends C> void clean(R context) {
......@@ -362,11 +385,11 @@ public class AggregationRdbTripDaoImpl<
protected Set<String> getGroupByColumnNames(AggregationStrataVO strata) {
Set<String> groupByColumnNames = Sets.newLinkedHashSet();
Set<String> result = Sets.newLinkedHashSet();
if (strata == null) {
groupByColumnNames.addAll(SPATIAL_COLUMNS);
groupByColumnNames.addAll(TIME_COLUMNS);
result.addAll(SPATIAL_COLUMNS);
result.addAll(TIME_COLUMNS);
}
else {
......@@ -376,32 +399,46 @@ public class AggregationRdbTripDaoImpl<
switch (spaceStrata) {
case COLUMN_SQUARE:
groupByColumnNames.add(COLUMN_SQUARE);
result.add(COLUMN_SQUARE);
case COLUMN_SUB_POLYGON:
groupByColumnNames.add(COLUMN_SUB_POLYGON);
result.add(COLUMN_SUB_POLYGON);
case COLUMN_STATISTICAL_RECTANGLE:
groupByColumnNames.add(COLUMN_STATISTICAL_RECTANGLE);
result.add(COLUMN_STATISTICAL_RECTANGLE);
case COLUMN_AREA:
default:
groupByColumnNames.add(COLUMN_AREA);
result.add(COLUMN_AREA);
}
// Time strata
String timeStrata = strata.getTimeColumnName() != null ? strata.getTimeColumnName().toLowerCase() : COLUMN_YEAR;
timeStrata = COLUMN_ALIAS.getOrDefault(timeStrata, timeStrata); // Replace alias
String timeColumnName = strata.getTimeColumnName() != null ? strata.getTimeColumnName().toLowerCase() : COLUMN_YEAR;
result.addAll(getGroupByTimesColumnNames(timeColumnName));
}
return result;
}
switch (timeStrata) {
protected Set<String> getGroupByTimesColumnNames(String timeColumnName) {
Set<String> result = Sets.newLinkedHashSet();
if (timeColumnName == null) {
result.addAll(TIME_COLUMNS);
}
else {
timeColumnName = COLUMN_ALIAS.getOrDefault(timeColumnName, timeColumnName); // Replace alias
switch (timeColumnName) {
case COLUMN_MONTH:
groupByColumnNames.add(COLUMN_MONTH);
result.add(COLUMN_MONTH);
case COLUMN_QUARTER:
groupByColumnNames.add(COLUMN_QUARTER);
result.add(COLUMN_QUARTER);
case COLUMN_YEAR:
default:
groupByColumnNames.add(COLUMN_YEAR);
result.add(COLUMN_YEAR);
}
}
return groupByColumnNames;
return result;
}
protected Set<String> getExistingGroupByColumnNames(final AggregationStrataVO strata,
......
......@@ -10,12 +10,12 @@ package net.sumaris.core.extraction.service;
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
......@@ -39,6 +39,7 @@ import java.util.Map;
/**
* Create aggregation tables, from a data extraction.
*
* @author benoit.lavenier@e-is.pro
* @since 0.12.0
*/
......@@ -46,17 +47,18 @@ import java.util.Map;
public interface AggregationService {
@Transactional(readOnly = true)
AggregationTypeVO getByFormat(IExtractionFormat format);
AggregationTypeVO getTypeByFormat(IExtractionFormat format);
@Transactional(readOnly = true)
List<AggregationTypeVO> findByFilter(@Nullable AggregationTypeFilterVO filter, ExtractionProductFetchOptions fetchOptions);
List<AggregationTypeVO> findTypesByFilter(@Nullable AggregationTypeFilterVO filter, ExtractionProductFetchOptions fetchOptions);
@Transactional(readOnly = true)
AggregationTypeVO get(int id, ExtractionProductFetchOptions fetchOptions);
AggregationTypeVO getTypeById(int id, ExtractionProductFetchOptions fetchOptions);
/**
* Do an aggregate
*
* @param type
* @param filter
*/
......@@ -66,22 +68,26 @@ public interface AggregationService {
@Nullable AggregationStrataVO strata);
@Transactional(readOnly = true)
AggregationResultVO read(AggregationTypeVO type,
@Nullable ExtractionFilterVO filter,
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction);
AggregationResultVO getAggBySpace(AggregationTypeVO type,
@Nullable ExtractionFilterVO filter,
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction);
@Transactional(readOnly = true)
AggregationResultVO read(AggregationContextVO context,
@Nullable ExtractionFilterVO filter,
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction);
Map<String, Object> readTech(AggregationTypeVO format,
ExtractionFilterVO filter,
AggregationStrataVO strata,
String sort,
SortDirection direction);
AggregationResultVO getAggBySpace(AggregationContextVO context,
@Nullable ExtractionFilterVO filter,
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction);
AggregationTechResultVO getAggByTech(AggregationTypeVO format,
ExtractionFilterVO filter,
AggregationStrataVO strata,
String sort,
SortDirection direction);
MinMaxVO getAggMinMaxByTech(AggregationTypeVO format,
ExtractionFilterVO filter,
AggregationStrataVO strata);
@Transactional(rollbackFor = IOException.class)
File executeAndDump(AggregationTypeVO type,
......
......@@ -22,8 +22,11 @@ package net.sumaris.core.extraction.service;
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import lombok.NonNull;
import net.sumaris.core.dao.technical.SortDirection;
import net.sumaris.core.exception.SumarisTechnicalException;
......@@ -98,8 +101,11 @@ public class AggregationServiceImpl implements AggregationService {
@Autowired
private AggregationService self;
@Autowired
private ObjectMapper objectMapper;
@Override
public List<AggregationTypeVO> findByFilter(AggregationTypeFilterVO filter, ExtractionProductFetchOptions fetchOptions) {
public List<AggregationTypeVO> findTypesByFilter(AggregationTypeFilterVO filter, ExtractionProductFetchOptions fetchOptions) {
final AggregationTypeFilterVO notNullFilter = filter != null ? filter : new AggregationTypeFilterVO();
......@@ -111,20 +117,20 @@ public class AggregationServiceImpl implements AggregationService {
@Override
@Cacheable(cacheNames = ExtractionCacheNames.AGGREGATION_TYPE_BY_ID)
public AggregationTypeVO get(int id, ExtractionProductFetchOptions fetchOptions) {
public AggregationTypeVO getTypeById(int id, ExtractionProductFetchOptions fetchOptions) {
ExtractionProductVO source = productService.get(id, fetchOptions);
return toAggregationType(source);
}
@Override
@Cacheable(cacheNames = ExtractionCacheNames.AGGREGATION_TYPE_BY_FORMAT, condition = " #format != null", unless = "#result == null")
public AggregationTypeVO getByFormat(IExtractionFormat format) {
public AggregationTypeVO getTypeByFormat(IExtractionFormat format) {
return ExtractionFormats.findOneMatch(getAllAggregationTypes(null), format);
}
@Override
public AggregationContextVO execute(AggregationTypeVO type, ExtractionFilterVO filter, AggregationStrataVO strata) {
type = getByFormat(type);
type = getTypeByFormat(type);
ExtractionProductVO source;
switch (type.getCategory()) {
......@@ -158,7 +164,7 @@ public class AggregationServiceImpl implements AggregationService {
}
@Override
public AggregationResultVO read(AggregationTypeVO type, @Nullable ExtractionFilterVO filter, @Nullable AggregationStrataVO strata, int offset, int size, String sort, SortDirection direction) {
public AggregationResultVO getAggBySpace(AggregationTypeVO type, @Nullable ExtractionFilterVO filter, @Nullable AggregationStrataVO strata, int offset, int size, String sort, SortDirection direction) {
Preconditions.checkNotNull(type);
ExtractionProductVO product = productService.getByLabel(type.getLabel(),
......@@ -168,14 +174,14 @@ public class AggregationServiceImpl implements AggregationService {
String sheetName = strata.getSheetName() != null ? strata.getSheetName() : filter.getSheetName();
AggregationContextVO context = toContextVO(product, sheetName);
return read(context, filter, strata, offset, size, sort, direction);
return getAggBySpace(context, filter, strata, offset, size, sort, direction);
}
@Override
public AggregationResultVO read(@NonNull AggregationContextVO context,
@Nullable ExtractionFilterVO filter,
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction) {
public AggregationResultVO getAggBySpace(@NonNull AggregationContextVO context,
@Nullable ExtractionFilterVO filter,
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction) {
filter = filter != null ? filter : new ExtractionFilterVO();
strata = strata != null ? strata : (context.getStrata() != null ? context.getStrata() : new AggregationStrataVO());
String sheetName = strata.getSheetName() != null ? strata.getSheetName() : filter.getSheetName();
......@@ -195,7 +201,7 @@ public class AggregationServiceImpl implements AggregationService {
case AGG_RDB:
case AGG_COST:
case AGG_SURVIVAL_TEST:
return aggregationRdbTripDao.read(tableName, filter, strata, offset, size, sort, direction);
return aggregationRdbTripDao.getAggBySpace(tableName, filter, strata, offset, size, sort, direction);
default:
throw new SumarisTechnicalException(String.format("Unable to read data on type '%s': not implemented", context.getLabel()));
}
......@@ -203,11 +209,34 @@ public class AggregationServiceImpl implements AggregationService {
}
@Override
public Map<String, Object> readTech(AggregationTypeVO type,
ExtractionFilterVO filter,
AggregationStrataVO strata,
String sort,
SortDirection direction) {
public AggregationTechResultVO getAggByTech(AggregationTypeVO type,
ExtractionFilterVO filter,
AggregationStrataVO strata,
String sort,
SortDirection direction) {
Preconditions.checkNotNull(type);
filter = filter != null ? filter : new ExtractionFilterVO();
ExtractionProductVO product = productService.getByLabel(type.getLabel(),
ExtractionProductFetchOptions.TABLES);
// Convert to context VO (need the next read() function)
String sheetName = strata != null && strata.getSheetName() != null ? strata.getSheetName() : filter.getSheetName();
Preconditions.checkNotNull(sheetName, String.format("Missing 'filter.%s' or 'strata.%s",
ExtractionFilterVO.Fields.SHEET_NAME,
AggregationStrataVO.Fields.LABEL));
AggregationContextVO context = toContextVO(product, sheetName);
strata = strata != null ? strata : (context.getStrata() != null ? context.getStrata() : new AggregationStrataVO());
String tableName = StringUtils.isNotBlank(sheetName) ? context.getTableNameBySheetName(sheetName) : null;