Commit c09b4d40 authored by LAVENIER's avatar LAVENIER
Browse files

[fix] Aggregation: fix aggregation on product

[enh] Aggregation: add aggregation for format FREE1, RJB
parent ce620a64
# SUMARiS options
sumaris.name=SUMARiS
sumaris.version=@project.version@
sumaris.log.file=${user.home}/.config/${sumaris.name}/${sumaris.name}.log
# Spring: Common properties
# see https://docs.spring.io/spring-boot/docs/current/reference/html/common-application-properties.html
spring.main.banner-mode=off
# Disable web
spring.main.web-application-type=none
# DataSource configuration
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
spring.datasource.username=sa
spring.datasource.password=
spring.datasource.url=jdbc:hsqldb:hsql://localhost/sumaris
spring.datasource.platform=@env@
spring.datasource.hikari.connectionTestQuery=SELECT 1 FROM STATUS WHERE ID=1
spring.datasource.hikari.initializationFailTimeout=-1
# General JPA properties
spring.jpa.database-platform=net.sumaris.core.dao.technical.hibernate.spatial.HSQLSpatialDialect
spring.jpa.open-in-view=false
# Hibernate Specific properties
spring.jpa.hibernate.ddl-auto=none
spring.jpa.properties.hibernate.format_sql=false
spring.jpa.properties.hibernate.hbm2ddl.auto=none
#Naming strategy
spring.jpa.hibernate.naming.physical-strategy=net.sumaris.core.dao.technical.hibernate.HibernatePhysicalNamingStrategy
spring.jpa.hibernate.naming.implicit-strategy=net.sumaris.core.dao.technical.hibernate.HibernateImplicitNamingStrategy
# Liquibase
spring.liquibase.enabled=false
# Extraction options
# Embedded ActiveMQ Configuration
spring.activemq.pool.enabled=true
spring.activemq.broker-url=vm://embedded?broker.persistent=true
# Spring JMS Settings
spring.jms.jndi-name=
# Logging Levels
logging.level.ROOT=info
logging.level.net.sumaris=info
logging.level.net.sumaris.core.extraction=debug
logging.level.org.springframework=warn
logging.level.org.nuiton=warn
logging.level.org.nuiton.i18n=error
logging.level.org.ehcache=warn
logging.level.org.apache.commons.beanutils=warn
logging.level.org.apache.jena=warn
logging.level.org.hibernate=warn
#logging.level.org.hibernate.SQL=debug
logging.level.org.hibernate.spatial=warn
logging.level.org.hibernate.engine.jdbc.spi.SqlExceptionHelper=error
logging.level.org.hibernate.engine.internal.StatefulPersistenceContext=error
logging.level.org.hibernate.engine.StatefulPersistenceContext.ProxyWarnLog=error
logging.level.org.jboss.logging=warn
logging.level.hsqldb.db=warn
logging.level.liquibase=error
logging.level.liquibase-hibernate=warn
logging.level.com.zaxxer.hikari=warn
......@@ -28,8 +28,7 @@ import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.trip.rdb.AggregationRdbTripDaoImpl;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.AggCostSpecification;
import net.sumaris.core.extraction.specification.data.trip.AggSurvivalTestSpecification;
import net.sumaris.core.extraction.specification.data.trip.*;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.rdb.AggregationRdbTripContextVO;
import net.sumaris.core.extraction.vo.trip.survivalTest.AggregationSurvivalTestContextVO;
......@@ -51,7 +50,7 @@ public class AggregationCostDaoImpl<
F extends ExtractionFilterVO,
S extends AggregationStrataVO>
extends AggregationRdbTripDaoImpl<C, F, S>
implements AggSurvivalTestSpecification {
implements AggCostSpecification {
@Override
public ProductFormatEnum getFormat() {
......@@ -71,35 +70,15 @@ public class AggregationCostDaoImpl<
@Override
protected Class<? extends AggregationRdbTripContextVO> getContextClass() {
return AggregationSurvivalTestContextVO.class;
return AggregationRdbTripContextVO.class;
}
protected String getQueryFullName(C context, String queryName) {
Preconditions.checkNotNull(context);
Preconditions.checkNotNull(context.getVersion());
switch (queryName) {
case "injectionSpeciesLengthTable":
return getQueryFullName(AggCostSpecification.FORMAT, AggCostSpecification.VERSION_1_4, queryName);
default:
return super.getQueryFullName(context, queryName);
}
}
@Override
protected XMLQuery createSpeciesLengthQuery(ExtractionProductVO source, C context) {
XMLQuery xmlQuery = super.createSpeciesLengthQuery(source, context);
// Special case for COST format:
// - Hide sex columns, then replace by a new column
xmlQuery.setGroup("sex", false);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionSpeciesLengthTable"));
return xmlQuery;
protected void fillContextTableNames(C context) {
super.fillContextTableNames(context);
// Rename some columns
context.addColumnNameReplacement(RdbSpecification.COLUMN_INDIVIDUAL_SEX, CostSpecification.COLUMN_SEX);
}
}
package net.sumaris.core.extraction.dao.trip.free;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.trip.rdb.AggregationRdbTripDaoImpl;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.*;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.rdb.AggregationRdbTripContextVO;
import net.sumaris.core.vo.technical.extraction.AggregationStrataVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Repository;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
@Repository("aggregationFree1Dao")
@Lazy
@Slf4j
public class AggregationFree1DaoImpl<
C extends AggregationRdbTripContextVO,
F extends ExtractionFilterVO,
S extends AggregationStrataVO>
extends AggregationRdbTripDaoImpl<C, F, S>
implements AggSurvivalTestSpecification {
@Override
public ProductFormatEnum getFormat() {
return ProductFormatEnum.AGG_FREE;
}
@Override
public <R extends C> R aggregate(ExtractionProductVO source, F filter, S strata) {
R context = super.aggregate(source, filter, strata);
context.setFormat(ProductFormatEnum.AGG_FREE);
return context;
}
/* -- protected methods -- */
@Override
protected Class<? extends AggregationRdbTripContextVO> getContextClass() {
return AggregationRdbTripContextVO.class;
}
@Override
protected void fillContextTableNames(C context) {
super.fillContextTableNames(context);
// Rename some columns
context.addColumnNameReplacement(AggRdbSpecification.COLUMN_FISHING_TIME, AggFree1Specification.COLUMN_FISHING_DURATION);
context.addColumnNameReplacement(RdbSpecification.COLUMN_INDIVIDUAL_SEX, Free1Specification.COLUMN_SEX);
}
@Override
protected XMLQuery createStationQuery(ExtractionProductVO source, C context) {
XMLQuery xmlQuery = super.createStationQuery(source, context);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionStationTable"));
return xmlQuery;
}
protected String getQueryFullName(C context, String queryName) {
Preconditions.checkNotNull(context);
Preconditions.checkNotNull(context.getVersion());
switch (queryName) {
case "injectionStationTable":
return getQueryFullName(AggFree1Specification.FORMAT, AggFree1Specification.VERSION_1, queryName);
default:
return super.getQueryFullName(context, queryName);
}
}
}
package net.sumaris.core.extraction.dao.trip.pmfm;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.trip.rdb.AggregationRdbTripDaoImpl;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.AggCostSpecification;
import net.sumaris.core.extraction.specification.data.trip.AggPmfmTripSpecification;
import net.sumaris.core.extraction.specification.data.trip.AggSurvivalTestSpecification;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.rdb.AggregationRdbTripContextVO;
import net.sumaris.core.extraction.vo.trip.survivalTest.AggregationSurvivalTestContextVO;
import net.sumaris.core.vo.technical.extraction.AggregationStrataVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Repository;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
@Repository("aggregationPmfmTripDao")
@Lazy
@Slf4j
public class AggregationPmfmTripDaoImpl<
C extends AggregationRdbTripContextVO,
F extends ExtractionFilterVO,
S extends AggregationStrataVO>
extends AggregationRdbTripDaoImpl<C, F, S>
implements AggSurvivalTestSpecification {
@Override
public ProductFormatEnum getFormat() {
return ProductFormatEnum.AGG_PMFM_TRIP;
}
@Override
public <R extends C> R aggregate(ExtractionProductVO source, F filter, S strata) {
R context = super.aggregate(source, filter, strata);
context.setFormat(ProductFormatEnum.AGG_PMFM_TRIP);
return context;
}
/* -- protected methods -- */
@Override
protected Class<? extends AggregationRdbTripContextVO> getContextClass() {
return AggregationRdbTripContextVO.class;
}
@Override
protected XMLQuery createSpeciesLengthQuery(ExtractionProductVO source, C context) {
XMLQuery xmlQuery = super.createSpeciesLengthQuery(source, context);
// Special case for COST format:
// - Hide sex columns, then replace by a new column
xmlQuery.setGroup("sex", false);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionSpeciesLengthTable"));
return xmlQuery;
}
protected String getQueryFullName(C context, String queryName) {
Preconditions.checkNotNull(context);
Preconditions.checkNotNull(context.getVersion());
switch (queryName) {
case "injectionSpeciesLengthTable":
return getQueryFullName(AggPmfmTripSpecification.FORMAT, AggPmfmTripSpecification.VERSION_1_0, queryName);
default:
return super.getQueryFullName(context, queryName);
}
}
}
......@@ -55,10 +55,15 @@ public class ExtractionPmfmTripDaoImpl<C extends ExtractionRdbTripContextVO, F e
extends ExtractionRdbTripDaoImpl<C, F>
implements PmfmTripSpecification {
private static final String XML_QUERY_PMFM_PATH = "pmfm/v%s/%s";
private static final String XML_QUERY_PMFM_PATH = "pmfmTrip/v%s/%s";
private static final String XML_QUERY_PMFM_V1_0_PATH = String.format(XML_QUERY_PMFM_PATH,
VERSION_1_0.replaceAll("[.]", "_"), "%s");
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.PMFM_TRIP;
}
@Override
public <R extends C> R execute(F filter) {
R context = super.execute(filter);
......@@ -68,11 +73,6 @@ public class ExtractionPmfmTripDaoImpl<C extends ExtractionRdbTripContextVO, F e
return context;
}
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.PMFM_TRIP;
}
/* -- protected methods -- */
......@@ -126,6 +126,7 @@ public class ExtractionPmfmTripDaoImpl<C extends ExtractionRdbTripContextVO, F e
xmlQuery.setGroup("lengthClass", false);
xmlQuery.setGroup("numberAtLength", false);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionSpeciesLengthTable"));
return xmlQuery;
}
......@@ -137,6 +138,7 @@ public class ExtractionPmfmTripDaoImpl<C extends ExtractionRdbTripContextVO, F e
switch (queryName) {
case "injectionTripPmfm":
case "injectionOperationPmfm":
case "injectionSpeciesLengthTable":
return String.format(XML_QUERY_PMFM_V1_0_PATH, queryName);
default:
return super.getQueryFullName(context, queryName);
......
......@@ -50,6 +50,7 @@ import net.sumaris.core.util.StringUtils;
import net.sumaris.core.vo.technical.extraction.AggregationStrataVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.io.Resource;
......@@ -59,10 +60,7 @@ import org.springframework.stereotype.Repository;
import javax.persistence.PersistenceException;
import java.io.IOException;
import java.net.URL;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
......@@ -290,7 +288,7 @@ public class AggregationRdbTripDaoImpl<
XMLQuery xmlQuery = createStationQuery(source, context);
// aggregate insertion
execute(xmlQuery);
execute(context, xmlQuery);
long count = countFrom(tableName);
if (count == 0) {
......@@ -307,7 +305,7 @@ public class AggregationRdbTripDaoImpl<
// Analyze row
Map<String, List<String>> columnValues = null;
if (context.isEnableAnalyze()) {
columnValues = analyzeRow(tableName, xmlQuery, COLUMN_YEAR);
columnValues = analyzeRow(context, tableName, xmlQuery, COLUMN_YEAR);
}
// Add result table to context
......@@ -334,6 +332,11 @@ public class AggregationRdbTripDaoImpl<
xmlQuery.bind("rawStationTableName", rawStationTableName);
xmlQuery.bind("stationTableName", stationTableName);
// Bind column names, because som raw tables can have change this (e.g. Free1 format use 'FISHING_DURATION' instead of 'FISHING_TIME')
// TODO: find a way to use a replacement map, with regexp, to replace all columns
// E.g. a map 'columnNameMapping'
//xmlQuery.bind(AggRdbSpecification.COLUMN_FISHING_TIME.toUpperCase(), context.getFishingTimeColumnName().toUpperCase());
// Date
xmlQuery.setGroup("startDateFilter", context.getStartDate() != null);
xmlQuery.bind("startDate", Daos.getSqlToDate(context.getStartDate()));
......@@ -482,7 +485,7 @@ public class AggregationRdbTripDaoImpl<
XMLQuery xmlQuery = createSpeciesListQuery(source, context);
// aggregate insertion
execute(xmlQuery);
execute(context, xmlQuery);
long count = countFrom(tableName);
if (count == 0) {
......@@ -499,7 +502,7 @@ public class AggregationRdbTripDaoImpl<
// Analyze row
Map<String, List<String>> columnValues = null;
if (context.isEnableAnalyze()) {
columnValues = analyzeRow(tableName, xmlQuery, COLUMN_YEAR);
columnValues = analyzeRow(context, tableName, xmlQuery, COLUMN_YEAR);
}
// Add result table to context
......@@ -563,7 +566,7 @@ public class AggregationRdbTripDaoImpl<
if (xmlQuery == null) return -1; // Skip
// Create the table
execute(xmlQuery);
execute(context, xmlQuery);
// Create index on SL_ID
createIndex(tableName, tableName + "_IDX", ImmutableList.of("SL_ID"), false);
......@@ -630,7 +633,7 @@ public class AggregationRdbTripDaoImpl<
XMLQuery xmlQuery = createSpeciesLengthQuery(source, context);
// aggregate insertion
execute(xmlQuery);
execute(context, xmlQuery);
long count = countFrom(tableName);
if (count == 0) {
......@@ -647,7 +650,7 @@ public class AggregationRdbTripDaoImpl<
// Analyze row
Map<String, List<String>> columnValues = null;
if (context.isEnableAnalyze()) {
columnValues = analyzeRow(tableName, xmlQuery, COLUMN_YEAR);
columnValues = analyzeRow(context, tableName, xmlQuery, COLUMN_YEAR);
}
// Add result table to context
......@@ -715,7 +718,7 @@ public class AggregationRdbTripDaoImpl<
if (xmlQuery == null) return -1; // Skip
// aggregate insertion
execute(xmlQuery);
execute(context, xmlQuery);
long count = countFrom(tableName);
if (count == 0) {
......@@ -732,7 +735,7 @@ public class AggregationRdbTripDaoImpl<
// Analyze row
Map<String, List<String>> columnValues = null;
if (context.isEnableAnalyze()) {
columnValues = analyzeRow(tableName, xmlQuery, COLUMN_YEAR);
columnValues = analyzeRow(context, tableName, xmlQuery, COLUMN_YEAR);
}
// Add result table to context
......@@ -775,8 +778,23 @@ public class AggregationRdbTripDaoImpl<
return xmlQuery;
}
protected int execute(XMLQuery xmlQuery) {
return queryUpdate(xmlQuery.getSQLQueryAsString());
protected int execute(C context, XMLQuery xmlQuery) {
String sqlQuery = doSqlReplacement(context, xmlQuery.getSQLQueryAsString());
return queryUpdate(sqlQuery);
}
protected String doSqlReplacement(C context, String sqlQuery) {
sqlQuery = sqlQuery.toUpperCase();
// Do column nanes replacement
// E.g. Rename FISHING_TIME into FISHING_DURATION
if (context != null && MapUtils.isNotEmpty(context.getColumnNamesMapping())) {
for (Map.Entry<String, String> entry : context.getColumnNamesMapping().entrySet()) {
sqlQuery = sqlQuery.replaceAll(entry.getKey().toUpperCase(), entry.getValue().toUpperCase());
}
}
return sqlQuery;
}
protected long countFrom(String tableName) {
......@@ -827,14 +845,20 @@ public class AggregationRdbTripDaoImpl<
}
}
protected Map<String, List<String>> analyzeRow(final String tableName, XMLQuery xmlQuery,
String... includedNumericColumnNames) {
return analyzeRow(tableName, xmlQuery, includedNumericColumnNames, true);
protected Map<String, List<String>> analyzeRow(
final C context,
final String tableName,
XMLQuery xmlQuery,
String... includedNumericColumnNames) {
return analyzeRow(context, tableName, xmlQuery, includedNumericColumnNames, true);
}
protected Map<String, List<String>> analyzeRow(final String tableName, XMLQuery xmlQuery,
String[] includedNumericColumnNames,
boolean excludeHiddenColumns) {
protected Map<String, List<String>> analyzeRow(
final C context,
final String tableName, XMLQuery xmlQuery,
String[] includedNumericColumnNames,
boolean excludeHiddenColumns) {
Preconditions.checkNotNull(tableName);
Preconditions.checkNotNull(xmlQuery);
......@@ -844,11 +868,13 @@ public class AggregationRdbTripDaoImpl<
.filter(columnName -> !excludeHiddenColumns || !hiddenColumns.contains(columnName))
.collect(Collectors.toMap(
c -> c,
c -> query(String.format("SELECT DISTINCT %s FROM %s where %s IS NOT NULL", c, tableName, c), Object.class)
c -> query(
doSqlReplacement(context, String.format("SELECT DISTINCT %s FROM %s where %s IS NOT NULL", c, tableName, c)),
Object.class)
.stream()
.map(String::valueOf)
.collect(Collectors.toList())
)
)
);
}
......
......@@ -462,13 +462,18 @@ public class ExtractionRdbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
protected String getQueryFullName(C context, String queryName) {
Preconditions.checkNotNull(context);
Preconditions.checkNotNull(context.getLabel());
Preconditions.checkNotNull(context.getVersion());
return getQueryFullName(context.getLabel(), context.getVersion(), queryName);