Commit 7b283f57 authored by LAVENIER's avatar LAVENIER
Browse files

[enh] Add RJB extraction

parent 23e38cf9
......@@ -131,7 +131,7 @@
<ozimov-email.version>0.6.3</ozimov-email.version>
<reflections.version>0.9.12</reflections.version>
<rxjava2.version>2.2.21</rxjava2.version>
<activemq.version>5.16.0</activemq.version>
<activemq.version>5.16.2</activemq.version>
<jnr-ffi.version>2.2.2</jnr-ffi.version>
<kalium.version>0.8.1-SNAPSHOT</kalium.version>
......@@ -405,6 +405,12 @@
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-broker</artifactId>
<version>${activemq.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-jms_1.1_spec</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.activemq</groupId>
......@@ -412,9 +418,9 @@
<version>${activemq.version}</version>
</dependency>
<dependency>
<groupId>javax.jms</groupId>
<artifactId>javax.jms-api</artifactId>
<version>2.0.1</version>
<groupId>jakarta.jms</groupId>
<artifactId>jakarta.jms-api</artifactId>
<version>2.0.3</version>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
......@@ -446,11 +452,6 @@
<artifactId>spring-boot-starter-tomcat</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jms</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
......@@ -738,6 +739,11 @@
<artifactId>spring-beans</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jms</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
......
Subproject commit 48b6475e624ebb692f9646c341bd9c4d4e9d454c
Subproject commit e7edc1c5740b0235d16be41ee00465c746ee0c67
......@@ -69,7 +69,7 @@ public class Daos extends net.sumaris.core.dao.technical.Daos {
* @param strings a {@link String[]} object.
* @return concatenated strings
*/
public static String getSqlInEscapedStrings(String[] strings) {
public static String getSqlInEscapedStrings(String... strings) {
if (strings == null) return "";
return Stream.of(strings)
.filter(Objects::nonNull)
......
......@@ -48,6 +48,7 @@ import net.sumaris.core.service.administration.programStrategy.ProgramService;
import net.sumaris.core.service.administration.programStrategy.StrategyService;
import net.sumaris.core.util.Beans;
import net.sumaris.core.util.StringUtils;
import net.sumaris.core.util.TimeUtils;
import net.sumaris.core.vo.administration.programStrategy.*;
import net.sumaris.core.vo.filter.PmfmStrategyFilterVO;
import net.sumaris.core.vo.filter.StrategyFilterVO;
......@@ -117,7 +118,10 @@ public class ExtractionRdbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
context.setFormat(LiveFormatEnum.RDB);
context.setTableNamePrefix(TABLE_NAME_PREFIX);
// Start log
Long startTime = null;
if (log.isInfoEnabled()) {
startTime = System.currentTimeMillis();
StringBuilder filterInfo = new StringBuilder();
String filterStr = filter != null ? tripFilter.toString("\n - ") : null;
if (StringUtils.isNotBlank(filterStr)) {
......@@ -126,7 +130,7 @@ public class ExtractionRdbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
else {
filterInfo.append("(without filter)");
}
log.info(String.format("Starting extraction #%s-%s (raw data / trips)... %s", context.getLabel(), context.getId(), filterInfo.toString()));
log.info("Starting extraction {{}-{}} (raw data / trips)... {}", context.getLabel(), context.getId(), filterInfo.toString());
}
// Fill context table names
......@@ -165,8 +169,16 @@ public class ExtractionRdbTripDaoImpl<C extends ExtractionRdbTripContextVO, F ex
catch (PersistenceException e) {
// If error, clean created tables first, then rethrow the exception
clean(context);
startTime = null; // Avoid log
throw e;
}
finally {
if (startTime != null) {
log.info("Extraction {{}-{}} finished in {}", context.getLabel(), context.getId(), TimeUtils.printDurationFrom(startTime));
}
}
}
@Override
......
package net.sumaris.core.extraction.dao.trip.rjb;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.extraction.dao.technical.Daos;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.trip.rdb.ExtractionRdbTripDaoImpl;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.CostSpecification;
import net.sumaris.core.extraction.specification.data.trip.RjbSpecification;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.trip.rdb.ExtractionRdbTripContextVO;
import net.sumaris.core.model.referential.pmfm.PmfmEnum;
import net.sumaris.core.util.StringUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Repository;
/**
* Extraction for RJB (Pocheteaux). We use individual count, instead of weight
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
@Repository("extractionRjbTripDao")
@Lazy
@Slf4j
public class ExtractionRjbTripDaoImpl<C extends ExtractionRdbTripContextVO, F extends ExtractionFilterVO>
extends ExtractionRdbTripDaoImpl<C, F>
implements RjbSpecification {
private static final String XML_QUERY_RJB_PATH = "rjb/v%s/%s";
@Override
public LiveFormatEnum getFormat() {
return LiveFormatEnum.RJB;
}
@Override
public <R extends C> R execute(F filter) {
R context = super.execute(filter);
context.setFormat(LiveFormatEnum.RJB);
return context;
}
/* -- protected methods -- */
protected XMLQuery createTripQuery(C context) {
XMLQuery xmlQuery = super.createTripQuery(context);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionTripTable"));
/* // Bind some referential ids
xmlQuery.bind("contractCodePmfmIds", Daos.getSqlInNumbers(
PmfmEnum.CONTRACT_CODE.getId(),
PmfmEnum.SELF_SAMPLING_PROGRAM.getId()
));
// Bind some referential ids
xmlQuery.bind("contractCode", "RJB"); // TODO externalize*/
// TODO externalize
xmlQuery.bind("taxonGroupLabels", Daos.getSqlInEscapedStrings("RJB_1", "RJB_2"));
return xmlQuery;
}
protected XMLQuery createStationQuery(C context) {
XMLQuery xmlQuery = super.createStationQuery(context);
// Special case for COST format:
// - Hide GearType (not in the COST format)
xmlQuery.setGroup("gearType", false);
return xmlQuery;
}
@Override
protected XMLQuery createRawSpeciesListQuery(C context, boolean excludeInvalidStation) {
XMLQuery xmlQuery = super.createRawSpeciesListQuery(context, excludeInvalidStation);
// Special case for RJB format:
// - Hide weight columns, then replace by a new columns
xmlQuery.setGroup("weight", false);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionRawSpeciesListTable"));
// TODO externalize
xmlQuery.bind("taxonGroupLabels", Daos.getSqlInEscapedStrings("RJB_1", "RJB_2"));
return xmlQuery;
}
@Override
protected XMLQuery createSpeciesListQuery(C context) {
XMLQuery xmlQuery = super.createSpeciesListQuery(context);
// Special case for RJB format:
// - Hide weight columns, then replace by a new columns
xmlQuery.setGroup("weight", false);
xmlQuery.setGroup("lengthCode", false);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionSpeciesListTable"), "beforeLengthCode");
return xmlQuery;
}
@Override
protected XMLQuery createSpeciesLengthQuery(C context) {
XMLQuery xmlQuery = super.createSpeciesLengthQuery(context);
// Special case for RJB format:
// - Hide sex columns, then replace by a new columns
xmlQuery.setGroup("sex", false);
xmlQuery.setGroup("lengthClass", false);
xmlQuery.setGroup("numberAtLength", false);
xmlQuery.injectQuery(getXMLQueryURL(context, "injectionSpeciesLengthTable"));
xmlQuery.bind("isDeadPmfmId", String.valueOf(PmfmEnum.IS_DEAD.getId()));
return xmlQuery;
}
protected String getQueryFullName(C context, String queryName) {
Preconditions.checkNotNull(context);
Preconditions.checkNotNull(context.getVersion());
String versionStr = VERSION_1_0.replaceAll("[.]", "_");
switch (queryName) {
case "injectionTripTable":
case "injectionRawSpeciesListTable":
case "injectionSpeciesListTable":
case "injectionSpeciesLengthTable":
return String.format(XML_QUERY_RJB_PATH, versionStr, queryName);
default:
return super.getQueryFullName(context, queryName);
}
}
}
......@@ -48,7 +48,8 @@ public enum LiveFormatEnum implements IExtractionFormat {
FREE1 (Free1Specification.FORMAT, Free1Specification.SHEET_NAMES, Free1Specification.VERSION_1),
FREE2 (Free2Specification.FORMAT, Free2Specification.SHEET_NAMES, Free2Specification.VERSION_1_9),
SURVIVAL_TEST (SurvivalTestSpecification.FORMAT, SurvivalTestSpecification.SHEET_NAMES, SurvivalTestSpecification.VERSION_1_0),
PMFM_TRIP(PmfmTripSpecification.FORMAT, PmfmTripSpecification.SHEET_NAMES, PmfmTripSpecification.VERSION_1_0)
PMFM_TRIP(PmfmTripSpecification.FORMAT, PmfmTripSpecification.SHEET_NAMES, PmfmTripSpecification.VERSION_1_0),
RJB(RjbSpecification.FORMAT, RjbSpecification.SHEET_NAMES, RjbSpecification.VERSION_1_0)
;
private String label;
......
/*
* #%L
* SUMARiS
* %%
* Copyright (C) 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
package net.sumaris.core.extraction.specification.data.trip;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
*/
public interface RjbSpecification extends RdbSpecification {
String FORMAT = "RJB";
String VERSION_1_0 = "1.0";
}
......@@ -48,12 +48,12 @@
<select alias="SEX" type="text">null</select><!-- always define in the HL table -->
<!-- other fields -->
<select alias="WEIGHT" type="number"><![CDATA[COALESCE(
<select alias="WEIGHT" group="weight" type="number"><![CDATA[COALESCE(
(SELECT QM.NUMERICAL_VALUE FROM QUANTIFICATION_MEASUREMENT_B QM WHERE QM.BATCH_FK=SORTING_B.ID and QM.IS_REFERENCE_QUANTIFICATION=1),
(CASE WHEN (SAMPLING_B.SAMPLING_RATIO IS NULL OR SAMPLING_B.SAMPLING_RATIO = 0) THEN NULL ELSE (SELECT QM.NUMERICAL_VALUE/SAMPLING_B.SAMPLING_RATIO FROM QUANTIFICATION_MEASUREMENT_B QM WHERE QM.BATCH_FK=SAMPLING_B.ID and QM.IS_REFERENCE_QUANTIFICATION=1) END CASE)
)]]>
</select>
<select alias="SUBSAMPLING_WEIGHT" type="number"><![CDATA[COALESCE(
<select alias="SUBSAMPLING_WEIGHT" group="weight" type="number"><![CDATA[COALESCE(
(SELECT QM.NUMERICAL_VALUE FROM QUANTIFICATION_MEASUREMENT_B QM WHERE QM.BATCH_FK=SAMPLING_B.ID and QM.IS_REFERENCE_QUANTIFICATION=1),
(SELECT QM.NUMERICAL_VALUE FROM QUANTIFICATION_MEASUREMENT_B QM WHERE QM.BATCH_FK=SORTING_B.ID and QM.IS_REFERENCE_QUANTIFICATION=1)
)]]>
......
......@@ -42,9 +42,10 @@
<select alias="SEX" type="text">SL.SEX</select><!-- always define in the HL table -->
<!-- other fields -->
<select alias="WEIGHT" type="number">CAST(SUM(SL.WEIGHT) * 1000 AS INTEGER)</select>
<select alias="SUBSAMPLING_WEIGHT" type="number">CAST(SUM(SL.SUBSAMPLING_WEIGHT) * 1000 AS INTEGER)</select>
<select alias="LENGTH_CODE" type="text">SL.LENGTH_CODE</select>
<select alias="WEIGHT" group="weight" type="number">CAST(SUM(SL.WEIGHT) * 1000 AS INTEGER)</select>
<select alias="SUBSAMPLING_WEIGHT" group="weight" type="number">CAST(SUM(SL.SUBSAMPLING_WEIGHT) * 1000 AS INTEGER)</select>
<select alias="LENGTH_CODE" group="lengthCode" type="text">SL.LENGTH_CODE</select>
<!-- need to link other tables -->
<select alias="SAMPLE_IDS" group="hsqldb" type="hidden">ARRAY_AGG(SL.SAMPLE_ID)</select>
......
......@@ -51,6 +51,7 @@
<select alias="SAMPLING_METHOD" type="text">'Observer'</select><!-- TODO: ADAP - find it from a project's PMFM of type "Hidden" ? -->
<from alias="T">TRIP</from>
<from join="true">INNER JOIN PROGRAM P ON P.ID = T.PROGRAM_FK</from>
<!-- trip return harbour -->
<from join="true">INNER JOIN LOCATION L ON L.ID = T.RETURN_LOCATION_FK</from>
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
#%L
Dali :: Core
%%
Copyright (C) 2017 Ifremer
%%
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
#L%
-->
<query type="select">
<select alias="INDIVIDUAL_COUNT" type="hidden">SORTING_B.INDIVIDUAL_COUNT</select>
<select alias="SAMPLING_INDIVIDUAL_COUNT" type="hidden">SAMPLING_B.INDIVIDUAL_COUNT</select>
<where operator="AND">TG.LABEL IN (&amp;taxonGroupLabels)</where>
</query>
<?xml version="1.0" encoding="UTF-8"?>
<!--
#%L
Dali :: Core
%%
Copyright (C) 2017 Ifremer
%%
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
#L%
-->
<query type="select">
<!-- rename INDIVIDUAL_SEX by SEX (see COST) -->
<select alias="SEX" type="text">B.SEX</select>
<!-- reinsert last RDB fields, but after SEX -->
<select alias="LENGTH_CLASS" type="number">B.LENGTH_CLASS</select>
<select alias="NUMBER_AT_LENGTH" type="number">SUM(B.INDIVIDUAL_COUNT)</select>
<select alias="IS_DEAD" type="text"><![CDATA[(
SELECT
DECODE(SM.NUMERICAL_VALUE, 0, 'N', 1, 'Y', null)
FROM
SORTING_MEASUREMENT_B SM
WHERE SM.BATCH_FK=B.ID and SM.PMFM_FK=&isDeadPmfmId
)]]></select>
<groupby>B.SEX, LENGTH_CLASS, IS_DEAD</groupby>
</query>
<?xml version="1.0" encoding="UTF-8"?>
<!--
#%L
Dali :: Core
%%
Copyright (C) 2017 Ifremer
%%
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
#L%
-->
<query type="select">
<!-- new fields -->
<select alias="INDIVIDUAL_COUNT" type="text">SUM(SL.INDIVIDUAL_COUNT)</select>
<select alias="SAMPLING_INDIVIDUAL_COUNT" type="text">SUM(SL.SAMPLING_INDIVIDUAL_COUNT)</select>
<select alias="LENGTH_CODE" group="lengthCode" type="text">SL.LENGTH_CODE</select>
</query>
<?xml version="1.0" encoding="UTF-8"?>
<!--
#%L
Dali :: Core
%%
Copyright (C) 2017 Ifremer
%%
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
#L%
-->
<query type="select">
<with alias="SELECTED_TRIPS">
<subquery>
<subselect alias="ID" type="number">ID</subselect>
<from alias="T">TRIP</from>
<where>EXISTS (
SELECT O.ID
FROM OPERATION O, BATCH B, TAXON_GROUP TG
WHERE
O.TRIP_FK = T.ID
AND O.ID = B.OPERATION_FK
AND TG.LABEL IN (&amp;taxonGroupLabels)
AND TG.ID=B.TAXON_GROUP_FK
)</where>
</subquery>
</with>
<from join="true">INNER JOIN SELECTED_TRIPS ST ON ST.ID = T.ID</from>
</query>
......@@ -174,12 +174,14 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jms</artifactId>
<scope>provided</scope>
<scope>compile</scope>
<!--<optional>true</optional>-->
</dependency>
<dependency>
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-broker</artifactId>
<scope>provided</scope>
<scope>compile</scope>
<!--<optional>true</optional>-->
</dependency>
<dependency>
......
......@@ -92,11 +92,6 @@
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>javax.jms</groupId>
<artifactId>javax.jms-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.querydsl</groupId>
<artifactId>querydsl-jpa</artifactId>
......@@ -113,6 +108,11 @@
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-broker</artifactId>
</dependency>
<dependency>
<groupId>jakarta.jms</groupId>
<artifactId>jakarta.jms-api</artifactId>
<scope>compile</scope>
</dependency>
<!-- Nuiton-* -->
<dependency>
......
......@@ -29,7 +29,9 @@ import net.sumaris.core.event.entity.EntityInsertEvent;
import net.sumaris.core.event.entity.EntityUpdateEvent;
import net.sumaris.core.event.entity.IEntityEvent;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.jms.annotation.EnableJms;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
......@@ -37,12 +39,16 @@ import org.springframework.transaction.event.TransactionPhase;
import org.springframework.transaction.event.TransactionalEventListener;
import javax.annotation.PostConstruct;
import javax.jms.Destination;
import javax.jms.JMSContext;
import java.util.Arrays;
import java.util.stream.Collectors;
@Component
@Slf4j
@ConditionalOnClass({JMSContext.class, JmsTemplate.class})
public class EntityJmsNotifier {
public class JmsEntityNotifier {