Commit ac394bd8 authored by LAVENIER's avatar LAVENIER
Browse files

[fix] Fix extraction NullPointerException when strategy without pmfm

[fix] Avoid to load all PMFMS, on every RDB-like extractions, but only in format PMFM_TRIP format
[enh] Start to implement PMFM_TRIP extraction format
parent fb68458d
......@@ -93,17 +93,17 @@
<ojdbc8.version>12.2.0.1</ojdbc8.version>
<pgsql.version>42.2.5</pgsql.version>
<slf4j.version>1.7.27</slf4j.version>
<slf4j.version>1.7.30</slf4j.version>
<logback.version>1.2.3</logback.version>
<log4j.version>2.12.1</log4j.version>
<jansi.version>1.9</jansi.version>
<guava.version>23.0</guava.version>
<xml-apis.version>2.0.2</xml-apis.version>
<elasticsearch.version>6.2.2</elasticsearch.version>
<jna.version>4.5.2</jna.version>
<tyrus.version>1.15</tyrus.version>
<elasticsearch.version>7.10.2</elasticsearch.version>
<jna.version>5.8.0</jna.version>
<tyrus.version>1.17</tyrus.version>
<jackson.version>2.11.3</jackson.version><!-- /!\ 2.9.8 import kotlin error -->
<stringtemplate.version>4.0.2</stringtemplate.version>
<stringtemplate.version>4.1</stringtemplate.version>
<jTextUtilsVersion>0.3.3</jTextUtilsVersion>
<commonBeanutilsVersion>1.9.4</commonBeanutilsVersion>
<opencsv.version>2.4.3</opencsv.version>
......@@ -113,28 +113,28 @@
<hibernate-search.version>5.11.8.Final</hibernate-search.version>
<hibernate-spatial.version>5.4.27.Final</hibernate-spatial.version>
<hibernate-validator.version>6.2.0.Final</hibernate-validator.version>
<spring.version>5.2.12.RELEASE</spring.version>
<spring-web.version>5.2.12.RELEASE</spring-web.version>
<spring-data.version>2.3.6.RELEASE</spring-data.version>
<spring.version>5.2.13.RELEASE</spring.version>
<spring-web.version>5.2.13.RELEASE</spring-web.version>
<spring-data.version>2.3.7.RELEASE</spring-data.version>
<querydsl.version>4.2.2</querydsl.version>
<aspectj.version>1.9.6</aspectj.version>
<javassist.version>3.27.0-GA</javassist.version>
<cglib.version>3.3.0</cglib.version>
<ehcache.version>2.10.6</ehcache.version>
<ehcache-monitor.version>2.7.11</ehcache-monitor.version>
<lombok.version>1.18.16</lombok.version>
<lombok.version>1.18.20</lombok.version>
<xmlquery.version>1.2</xmlquery.version>
<jdom2.version>2.0.6</jdom2.version>
<jaxen.version>1.2.0</jaxen.version>
<spring-boot.version>2.3.8.RELEASE</spring-boot.version>
<spring-boot-plugin.version>2.3.8.RELEASE</spring-boot-plugin.version>
<spring-boot.version>2.3.9.RELEASE</spring-boot.version>
<spring-boot-plugin.version>2.3.9.RELEASE</spring-boot-plugin.version>
<ozimov-email.version>0.6.3</ozimov-email.version>
<reflections.version>0.9.12</reflections.version>
<rxjava.version>2.2.20</rxjava.version>
<rxjava2.version>2.2.21</rxjava2.version>
<activemq.version>5.16.0</activemq.version>
<jnr-ffi.version>2.2.1</jnr-ffi.version>
<kalium.version>0.5.0_blavenie</kalium.version>
<jnr-ffi.version>2.2.2</jnr-ffi.version>
<kalium.version>0.8.1-SNAPSHOT</kalium.version>
<!--<kalium.version>0.8.1-SNAPSHOT</kalium.version>-->
<scrypt.version>1.4.0</scrypt.version>
<geojson-jackson.version>1.14</geojson-jackson.version>
......@@ -158,7 +158,7 @@
<jmock.version>2.9.0</jmock.version>
<dbunit.version>2.6.0</dbunit.version>
<nuitonConfigVersion>3.4</nuitonConfigVersion>
<nuitonConfigVersion>3.5</nuitonConfigVersion>
<nuitonVersionVersion>1.0-rc-2</nuitonVersionVersion>
<nuitonUtilsVersion>3.0</nuitonUtilsVersion>
<nuitonI18nVersion>3.7</nuitonI18nVersion>
......@@ -419,7 +419,7 @@
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
<artifactId>rxjava</artifactId>
<version>${rxjava.version}</version>
<version>${rxjava2.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
......
......@@ -22,6 +22,8 @@ package net.sumaris.core.extraction.dao;
* #L%
*/
import net.sumaris.core.extraction.format.ProductFormatEnum;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
......@@ -29,4 +31,5 @@ public interface AggregationDao {
String TABLE_NAME_PREFIX = "AGG_";
ProductFormatEnum getFormat();
}
......@@ -22,10 +22,21 @@ package net.sumaris.core.extraction.dao;
* #L%
*/
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.vo.ExtractionContextVO;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>*
*/
public interface ExtractionDao {
public interface ExtractionDao<C extends ExtractionContextVO,
F extends ExtractionFilterVO> {
String TABLE_NAME_PREFIX = "EXT_";
String SEQUENCE_NAME_SUFFIX = "_SEQ";
LiveFormatEnum getFormat();
<R extends C> R execute(F filter);
void clean(C context);
}
......@@ -45,11 +45,8 @@ import java.util.stream.Collectors;
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface ExtractionStrategyDao<C extends ExtractionStrategyContextVO, F extends ExtractionFilterVO>
extends ExtractionDao {
extends ExtractionDao<C, F> {
<R extends C> R execute(F filter);
void clean(C context);
default ExtractionStrategyFilterVO toStrategyFilterVO(ExtractionFilterVO source){
ExtractionStrategyFilterVO target = new ExtractionStrategyFilterVO();
......
......@@ -72,6 +72,11 @@ public class ExtractionStrategyDaoImpl<C extends ExtractionStrategyContextVO, F
@Autowired
protected ResourceLoader resourceLoader;
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.STRAT;
}
@Override
public <R extends C> R execute(F filter) {
ExtractionStrategyFilterVO strategyFilter = toStrategyFilterVO(filter);
......@@ -125,11 +130,15 @@ public class ExtractionStrategyDaoImpl<C extends ExtractionStrategyContextVO, F
@Override
public void clean(C context) {
super.clean(context);
super.dropTables(context);
}
/* -- protected methods -- */
protected Class<? extends ExtractionStrategyContextVO> getContextClass() {
return ExtractionStrategyContextVO.class;
}
protected <R extends C> R createNewContext() {
Class<? extends ExtractionStrategyContextVO> contextClass = getContextClass();
Preconditions.checkNotNull(contextClass);
......@@ -141,10 +150,6 @@ public class ExtractionStrategyDaoImpl<C extends ExtractionStrategyContextVO, F
}
}
protected Class<? extends ExtractionStrategyContextVO> getContextClass() {
return ExtractionStrategyContextVO.class;
}
protected void fillContextTableNames(C context) {
// Set unique table names
......
......@@ -181,7 +181,7 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
return xmlQuery;
}
protected <C extends ExtractionContextVO> void clean(@NonNull C context) {
protected void dropTables(@NonNull ExtractionContextVO context) {
Preconditions.checkNotNull(context.getTableNamePrefix());
Set<String> tableNames = ImmutableSet.<String>builder()
......
......@@ -22,13 +22,17 @@ package net.sumaris.core.extraction.dao.trip;
* #L%
*/
import net.sumaris.core.dao.technical.SortDirection;
import net.sumaris.core.extraction.dao.AggregationDao;
import net.sumaris.core.extraction.dao.ExtractionDao;
import net.sumaris.core.extraction.vo.AggregationResultVO;
import net.sumaris.core.extraction.vo.AggregationTechResultVO;
import net.sumaris.core.extraction.vo.MinMaxVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface AggregationTripDao extends AggregationDao {
}
......@@ -25,6 +25,7 @@ package net.sumaris.core.extraction.dao.trip;
import com.google.common.collect.Lists;
import net.sumaris.core.extraction.dao.ExtractionDao;
import net.sumaris.core.extraction.specification.data.trip.RdbSpecification;
import net.sumaris.core.extraction.vo.ExtractionContextVO;
import net.sumaris.core.extraction.vo.ExtractionFilterCriterionVO;
import net.sumaris.core.extraction.vo.ExtractionFilterOperatorEnum;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
......@@ -40,8 +41,10 @@ import java.util.List;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface ExtractionTripDao extends ExtractionDao {
public interface ExtractionTripDao<
C extends ExtractionContextVO,
F extends ExtractionFilterVO>
extends ExtractionDao<C, F> {
default ExtractionTripFilterVO toTripFilterVO(ExtractionFilterVO source){
ExtractionTripFilterVO target = new ExtractionTripFilterVO();
......
package net.sumaris.core.extraction.dao.trip.cost;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import net.sumaris.core.extraction.dao.trip.AggregationTripDao;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.AggregationTripContextVO;
import net.sumaris.core.vo.technical.extraction.AggregationStrataVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface AggregationCostDao<C extends AggregationTripContextVO,
F extends ExtractionFilterVO,
S extends AggregationStrataVO> extends AggregationTripDao {
<R extends C> R aggregate(ExtractionProductVO source, F filter, S strata);
}
......@@ -25,6 +25,7 @@ package net.sumaris.core.extraction.dao.trip.cost;
import com.google.common.base.Preconditions;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.trip.rdb.AggregationRdbTripDaoImpl;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.AggCostSpecification;
import net.sumaris.core.extraction.specification.data.trip.AggSurvivalTestSpecification;
......@@ -45,11 +46,14 @@ import org.springframework.stereotype.Repository;
@Lazy
public class AggregationCostDaoImpl<C extends AggregationRdbTripContextVO, F extends ExtractionFilterVO, S extends AggregationStrataVO>
extends AggregationRdbTripDaoImpl<C, F, S>
implements AggregationCostDao<C, F, S>,
AggSurvivalTestSpecification {
implements AggSurvivalTestSpecification {
private static final Logger log = LoggerFactory.getLogger(AggregationCostDaoImpl.class);
@Override
public ProductFormatEnum getFormat() {
return ProductFormatEnum.AGG_COST;
}
@Override
public <R extends C> R aggregate(ExtractionProductVO source, F filter, S strata) {
......
package net.sumaris.core.extraction.dao.trip.cost;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import net.sumaris.core.extraction.dao.trip.rdb.ExtractionRdbTripDao;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.rdb.ExtractionRdbTripContextVO;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface ExtractionCostTripDao<C extends ExtractionRdbTripContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDao<C, F> {
}
......@@ -41,10 +41,15 @@ import org.springframework.stereotype.Repository;
@Slf4j
public class ExtractionCostTripDaoImpl<C extends ExtractionRdbTripContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDaoImpl<C, F>
implements ExtractionCostTripDao<C, F>, CostSpecification {
implements CostSpecification {
private static final String XML_QUERY_COST_PATH = "cost/v%s/%s";
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.COST;
}
@Override
public <R extends C> R execute(F filter) {
R context = super.execute(filter);
......
......@@ -40,10 +40,15 @@ import org.springframework.stereotype.Repository;
@Lazy
public class ExtractionFree1TripDaoImpl<C extends ExtractionRdbTripContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDaoImpl<C, F>
implements ExtractionFree1TripDao<C, F>, Free1Specification {
implements Free1Specification {
private static final String XML_QUERY_FREE_PATH = "free/v%s/%s";
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.FREE1;
}
@Override
public <R extends C> R execute(F filter) {
R context = super.execute(filter);
......
/*
* #%L
* SUMARiS
* %%
* Copyright (C) 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
package net.sumaris.core.extraction.dao.trip.free2;
import net.sumaris.core.extraction.dao.trip.rdb.ExtractionRdbTripDao;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.free2.ExtractionFree2ContextVO;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface ExtractionFree2TripDao<C extends ExtractionFree2ContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDao<C, F> {
}
\ No newline at end of file
......@@ -52,7 +52,7 @@ import org.springframework.stereotype.Repository;
@Slf4j
public class ExtractionFree2TripDaoImpl<C extends ExtractionFree2ContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDaoImpl<C, F>
implements ExtractionFree2TripDao<C, F>, Free2Specification {
implements Free2Specification {
private static final String XML_QUERY_FREE_PATH = "free2/v%s/%s";
......@@ -71,6 +71,11 @@ public class ExtractionFree2TripDaoImpl<C extends ExtractionFree2ContextVO, F ex
@Autowired
protected ExtractionTableDao extractionTableDao;
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.FREE2;
}
@Override
public <R extends C> R execute(F filter) {
String sheetName = filter != null ? filter.getSheetName() : null;
......
package net.sumaris.core.extraction.dao.trip.pmfm;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.trip.rdb.ExtractionRdbTripDaoImpl;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.PmfmTripSpecification;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.rdb.ExtractionRdbTripContextVO;
import net.sumaris.core.model.administration.programStrategy.AcquisitionLevelEnum;
import net.sumaris.core.model.referential.pmfm.PmfmEnum;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
import net.sumaris.core.service.administration.programStrategy.StrategyService;
import net.sumaris.core.util.StringUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Repository;
import java.util.Collections;
import java.util.List;
/**
* @author Ludovic Pecquot <ludovic.pecquot@e-is.pro>
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
@Repository("extractionPmfmTripDao")
@Lazy
@Slf4j
public class ExtractionPmfmTripDaoImpl<C extends ExtractionRdbTripContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDaoImpl<C, F>
implements PmfmTripSpecification {
@Autowired
protected StrategyService strategyService;
private static final String XML_QUERY_PMFM_PATH = "pmfm/v%s/%s";
private static final String XML_QUERY_PMFM_V1_0_PATH = String.format(XML_QUERY_PMFM_PATH,
VERSION_1_0.replaceAll("[.]", "_"), "%s");
@Override
public <R extends C> R execute(F filter) {
R context = super.execute(filter);
context.setFormat(LiveFormatEnum.PMFM_TRIP);
return context;
}
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.PMFM_TRIP;
}
/* -- protected methods -- */
protected XMLQuery createTripQuery(C context) {
XMLQuery xmlQuery = super.createTripQuery(context);
// Hide already pmfm columns
//xmlQuery.setGroup("xxx", false);
String programLabel = context.getTripFilter().getProgramLabel();
if (StringUtils.isNotBlank(programLabel)) {
injectPmfmColumns(context, xmlQuery,
Collections.singletonList(programLabel),
AcquisitionLevelEnum.TRIP,
PmfmEnum.NB_OPERATION);
}
return xmlQuery;
}
protected XMLQuery createStationQuery(C context) {
XMLQuery xmlQuery = super.createStationQuery(context);
// Special case for COST format:
// - Hide GearType (not in the COST format)
xmlQuery.setGroup("gearType", false);
// Inject Pmfm columns
injectPmfmColumns(context, xmlQuery,
getTripProgramLabels(context),
AcquisitionLevelEnum.OPERATION,
PmfmEnum.NB_OPERATION);
return xmlQuery;
}
@Override
protected XMLQuery createSpeciesLengthQuery(C context) {
XMLQuery xmlQuery = super.createSpeciesLengthQuery(context);
// Special case for COST format:
// - Hide sex columns, then replace by a new columns
xmlQuery.setGroup("sex", false);
xmlQuery.setGroup("lengthClass", false);
xmlQuery.setGroup("numberAtLength", false);
return xmlQuery;
}
protected String getQueryFullName(C context, String queryName) {
Preconditions.checkNotNull(context);
Preconditions.checkNotNull(context.getVersion());
switch (queryName) {
case "injectionPmfm":
return String.format(XML_QUERY_PMFM_V1_0_PATH, queryName);
default:
return super.getQueryFullName(context, queryName);
}
}