Commit 6f9d502a authored by PECQUOT's avatar PECQUOT
Browse files

Merge remote-tracking branch 'origin/develop' into develop

# Conflicts:
#	sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/StrategyRepositoryImpl.java
#	sumaris-core/src/main/java/net/sumaris/core/service/data/LandingServiceImpl.java
#	sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog-1.10.3.xml
#	sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog.xml
parents 847d73af 970a2e14
......@@ -27,10 +27,9 @@ variables:
CONTAINER_SNAPSHOT_IMAGE: $CI_REGISTRY_IMAGE:develop
MAVEN_LOCAL_REPO: /root/.m2/repository/
MAVEN_REPO_URL: https://gitlab.ifremer.fr/api/v4/projects/1272/packages/maven
APP_CORE_MODULE: sumaris-core
APP_DB_MODULE: sumaris-db
APP_SHARED_MODULE: sumaris-core-shared
APP_CORE_MODULE: sumaris-core
APP_DB_MODULE: sumaris-db
APP_WAR_MODULE: sumaris-server
ARTIFACT_WAR_FILE: "${APP_WAR_MODULE}/target/*.war"
ARTIFACT_CONFIG_FILES: "${APP_WAR_MODULE}/target/classes/*.properties"
......@@ -97,22 +96,23 @@ docker:ci:
when: on_failure
allow_failure: true
script:
# Create the target directory
- mkdir -p target/ci && cd target/ci
# Create the Dockerfile
- echo 'FROM jlrigau/maven-git' > Dockerfile
- echo 'WORKDIR /tmp/.build-cache' >> Dockerfile
- echo 'COPY ./ ./' >> Dockerfile
- echo "ENV MAVEN_LOCAL_REPO=${MAVEN_LOCAL_REPO}" >> Dockerfile
- echo 'RUN du -s `find ${MAVEN_LOCAL_REPO} -maxdepth 1 | egrep -v "^\.$"` > /tmp/before.txt' >> Dockerfile
- echo 'RUN mvn install -s ./ci_settings.xml -q -DskipTests' >> Dockerfile
- echo 'RUN mvn clean -s ./ci_settings.xml -q' >> Dockerfile
- echo 'RUN du -s `find ${MAVEN_LOCAL_REPO} -maxdepth 1 | egrep -v "^\.$"` > /tmp/after.txt' >> Dockerfile
- echo "RUN diff /tmp/before.txt /tmp/after.txt || true" >> Dockerfile
- echo 'WORKDIR /build' >> Dockerfile
# Build and push the CI image
- docker build --pull -t ${CI_BUILD_IMAGE} .
- docker push ${CI_BUILD_IMAGE}
# Create the target directory
- mkdir -p target/ci
# Create the Dockerfile
- echo 'FROM tirrell/maven-git' > target/ci/Dockerfile
- echo 'WORKDIR /tmp/.build-cache' >> target/ci/Dockerfile
- echo 'COPY ./ ./' >> target/ci/Dockerfile
- echo "RUN mkdir -p ${MAVEN_LOCAL_REPO}" >> target/ci/Dockerfile
- echo "ENV MAVEN_LOCAL_REPO=${MAVEN_LOCAL_REPO}" >> target/ci/Dockerfile
- echo 'RUN du -s `find ${MAVEN_LOCAL_REPO} -maxdepth 3 | egrep -v "^\.$"` > /tmp/before.txt' >> target/ci/Dockerfile
- echo 'RUN mvn install -s ./ci_settings.xml -q -DskipTests' >> target/ci/Dockerfile
- echo 'RUN mvn clean -s ./ci_settings.xml -q' >> target/ci/Dockerfile
- echo 'RUN du -s `find ${MAVEN_LOCAL_REPO} -maxdepth 3 | egrep -v "^\.$"` > /tmp/after.txt' >> target/ci/Dockerfile
- echo "RUN diff /tmp/before.txt /tmp/after.txt || true" >> target/ci/Dockerfile
- echo 'WORKDIR /build' >> target/ci/Dockerfile
# Build and push the CI image
- docker build --pull -t ${CI_BUILD_IMAGE} -f target/ci/Dockerfile .
- docker push ${CI_BUILD_IMAGE}
only:
- develop
artifacts:
......
......@@ -124,7 +124,7 @@
<ehcache.version>3.8.1</ehcache.version>
<ehcache-monitor.version>2.7.11</ehcache-monitor.version>
<lombok.version>1.18.20</lombok.version>
<xmlquery.version>1.2</xmlquery.version>
<xmlquery.version>1.4</xmlquery.version>
<jdom2.version>2.0.6</jdom2.version>
<jaxen.version>1.2.0</jaxen.version>
<spring-boot.version>2.4.5</spring-boot.version>
......@@ -836,7 +836,6 @@
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${pgsql.version}</version>
<classifier>jar</classifier>
</dependency>
<!-- LiquiBase -->
......
Subproject commit 88f496e152bbb813317a8ee7bc2220714457c200
Subproject commit fe643f48f8687957d8b20871f95a36e19fedb5b4
......@@ -30,6 +30,7 @@ import net.sumaris.core.extraction.exception.UnknownFormatException;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.service.AggregationService;
import net.sumaris.core.extraction.service.ExtractionProductService;
import net.sumaris.core.extraction.service.ExtractionServiceLocator;
import net.sumaris.core.extraction.vo.AggregationTypeVO;
import net.sumaris.core.model.referential.StatusEnum;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
......@@ -45,15 +46,15 @@ import java.util.stream.Collectors;
*
*/
@Slf4j
public class AgggregationAction {
public class AggregationAction {
/**
* <p>run.</p>
*/
public void run() {
ExtractionConfiguration config = ExtractionConfiguration.instance();
AggregationService aggregationService = ServiceLocator.instance().getService("aggregationService", AggregationService.class);
ExtractionProductService productService = ServiceLocator.instance().getService("extractionProductService", ExtractionProductService.class);
ExtractionProductService productService = ExtractionServiceLocator.extractionProductService();
AggregationService aggregationService = ExtractionServiceLocator.aggregationService();
String formatLabel = config.getExtractionCliOutputFormat();
AggregationTypeVO type = null;
......
......@@ -28,7 +28,10 @@ import lombok.extern.slf4j.Slf4j;
import net.sumaris.cli.action.ActionUtils;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.exception.UnknownFormatException;
import net.sumaris.core.extraction.service.AggregationService;
import net.sumaris.core.extraction.service.ExtractionProductService;
import net.sumaris.core.extraction.service.ExtractionService;
import net.sumaris.core.extraction.service.ExtractionServiceLocator;
import net.sumaris.core.extraction.util.ExtractionFormats;
import net.sumaris.core.extraction.vo.ExtractionTypeVO;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
......@@ -52,7 +55,7 @@ public class ExtractionAction {
*/
public void run() {
ExtractionConfiguration config = ExtractionConfiguration.instance();
ExtractionService service = ServiceLocator.instance().getService("extractionService", ExtractionService.class);
ExtractionService service = ExtractionServiceLocator.extractionService();
String formatLabel = config.getExtractionCliOutputFormat();
IExtractionFormat format = null;
......
package net.sumaris.core.extraction.action;
/*
* #%L
* SIH-Adagio :: Shared
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2012 - 2014 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.model.technical.history.ProcessingFrequencyEnum;
import net.sumaris.server.job.ExtractionJob;
/**
* <p>DatabaseChangeLogAction class.</p>
*
*/
@Slf4j
public class ExtractionProductUpdateAction {
/**
* <p>Update a product (execute extraction or aggregation).</p>
*/
public void run() throws InterruptedException {
// Create the job
ExtractionJob job = new ExtractionJob();
// Run it !
ProcessingFrequencyEnum frequency = ExtractionConfiguration.instance().getExtractionCliFrequency();
job.execute(frequency);
// Waiting 10s, to let DB drop tables (asynchronously)
Thread.sleep(10000);
}
}
......@@ -24,9 +24,9 @@ package net.sumaris.core.extraction.config;
* #L%
*/
import net.sumaris.core.extraction.action.AgggregationAction;
import net.sumaris.core.extraction.action.AggregationAction;
import net.sumaris.core.extraction.action.ExtractionAction;
import net.sumaris.core.extraction.action.ExtractionProductAction;
import net.sumaris.core.extraction.action.ExtractionProductUpdateAction;
import org.nuiton.config.ConfigActionDef;
/**
......@@ -37,8 +37,8 @@ import org.nuiton.config.ConfigActionDef;
public enum ExtractionConfigurationAction implements ConfigActionDef {
EXTRACTION(ExtractionAction.class.getName() + "#run", "Execute an extraction", "--extraction"),
AGGREGATION(AgggregationAction.class.getName() + "#run", "Execute an aggregation", "--aggregation"),
EXTRACTION_PRODUCT_UPDATE(ExtractionProductAction.class.getName() + "#update", "Update extraction products", "--extraction-product-update");
AGGREGATION(AggregationAction.class.getName() + "#run", "Execute an aggregation", "--aggregation"),
EXTRACTION_PRODUCT_UPDATE(ExtractionProductUpdateAction.class.getName() + "#run", "Update extraction products", "--extraction-product-update");
private final String action;
private final String description;
......
......@@ -28,10 +28,9 @@ import net.sumaris.core.exception.DataNotFoundException;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.dao.technical.Daos;
import net.sumaris.core.extraction.dao.technical.ExtractionBaseDaoImpl;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.technical.xml.XMLQuery;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.specification.administration.StratSpecification;
import net.sumaris.core.extraction.vo.ExtractionFilterCriterionVO;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.administration.ExtractionStrategyContextVO;
import net.sumaris.core.extraction.vo.administration.ExtractionStrategyFilterVO;
......
......@@ -34,6 +34,7 @@ import net.sumaris.core.dao.technical.schema.SumarisTableMetadata;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.dao.technical.schema.SumarisTableMetadatas;
import net.sumaris.core.extraction.dao.technical.xml.XMLQuery;
import net.sumaris.core.extraction.vo.ExtractionContextVO;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.model.referential.IItemReferentialEntity;
......@@ -59,7 +60,6 @@ import java.util.stream.Stream;
public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
protected static final String XML_QUERY_PATH = "xmlQuery";
protected static final String XSL_ORACLE_FILENAME = "xmlQuery/queryOracle.xsl";
@Autowired
protected ExtractionConfiguration configuration;
......@@ -176,14 +176,7 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
* @return
*/
protected XMLQuery createXMLQuery() {
XMLQuery xmlQuery = applicationContext.getBean("xmlQuery", XMLQuery.class);
if (this.databaseType == DatabaseType.oracle) {
xmlQuery.setXSLFileName(XSL_ORACLE_FILENAME);
}
else if (this.databaseType == DatabaseType.postgresql){
xmlQuery.setLowercase(true);
}
return xmlQuery;
return new XMLQuery(databaseType);
}
protected void dropTables(@NonNull ExtractionContextVO context) {
......@@ -257,4 +250,22 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
return query;
}
/**
* Enable/Disable group, depending on the DBMS
* @param xmlQuery
*/
protected void setDbms(XMLQuery xmlQuery) {
if (this.databaseType != null) {
switch (this.databaseType) {
case hsqldb:
case oracle:
xmlQuery.setDbms(this.databaseType.name());
break;
case postgresql:
xmlQuery.setDbms("pgsql");
break;
}
}
}
}
/*
* Created on 29 juil. 2004
*/
package net.sumaris.core.extraction.dao.technical;
/*-
* #%L
* Quadrige3 Core :: Shared
* %%
* Copyright (C) 2017 - 2018 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.base.Preconditions;
import fr.ifremer.common.xmlquery.HSQLDBSingleXMLQuery;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.dao.technical.xml.XPaths;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.jdom2.Attribute;
import org.jdom2.Element;
import org.jdom2.filter.Filters;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author ludovic.pecquot@e-is.pro
*/
@Component("xmlQuery")
@Scope("prototype")
public class XMLQuery extends HSQLDBSingleXMLQuery {
private String xslFileName;
private boolean lowercase;
public XMLQuery() {
super();
xslFileName = super.getXSLFileName();
this.lowercase = false;
}
public XMLQuery(boolean lowercase) {
super();
xslFileName = super.getXSLFileName();
this.lowercase = lowercase;
}
@Override
protected String getXSLFileName() {
return xslFileName;
}
protected void setXSLFileName(String xslFileName) {
this.xslFileName = xslFileName;
}
// let default values here for HSQLDB
public boolean isLowercase() {
return lowercase;
}
public void setLowercase(boolean lowercase) {
this.lowercase = lowercase;
}
/**
* Get column names, with type="hidden"
*
* @return
*/
public Set<String> getHiddenColumnNames() {
try {
List<Element> selectElements = XPaths.compile("//query/select[contains(@type, 'hidden')]", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(attribute -> attribute.getValue())
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
public Set<String> getVisibleColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"hidden".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getNotNumericColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"number".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getColumnNames(final Predicate<Element> filter) {
Preconditions.checkNotNull(filter);
try {
List<Element> selectElements = XPaths.compile("//query/select", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
// Apply filter
.filter(filter::evaluate)
// Get alias
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(attribute -> attribute.getValue())
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
/**
* Return if option="DISTINCT" has been set on the query
*
* @return
*/
public boolean hasDistinctOption() {
Attribute optionAtr = getFirstQueryTag().getAttribute("option");
return optionAtr != null && "distinct".equalsIgnoreCase(optionAtr.getValue());
}
public String getSQLQueryAsString(){
String query = super.getSQLQueryAsString();
if (this.lowercase){
query = query.toLowerCase();
}
return query;
}
}
/*
* #%L
* SUMARiS
* %%
* Copyright (C) 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
/*
* Created on 29 juil. 2004
*/
package net.sumaris.core.extraction.dao.technical.xml;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import fr.ifremer.common.xmlquery.*;
import lombok.NonNull;
import net.sumaris.core.dao.technical.DatabaseType;
import net.sumaris.core.exception.SumarisTechnicalException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.jdom2.Attribute;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.filter.Filters;
import org.jdom2.output.Format;
import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author ludovic.pecquot@e-is.pro
*/
public class XMLQuery {
private DatabaseType dbms;
private AsbtractSingleXMLQuery delegate;
public XMLQuery(@NonNull DatabaseType dbms) {
super();
this.dbms = dbms;
switch (dbms) {
case hsqldb:
delegate = new HSQLDBSingleXMLQuery();
delegate.bind("true", "1");
delegate.bind("false", "0");
break;
case postgresql:
delegate = new PgsqlSingleXMLQuery();
delegate.bind("true", "True");
delegate.bind("false", "False");
break;
case oracle:
delegate = new OracleSingleXMLQuery();
delegate.bind("true", "1");
delegate.bind("false", "0");
break;
default:
throw new IllegalArgumentException("Not XMLQuery instance found for database type: " + dbms.name());
}
delegate.setDbms(dbms.name());
}
/**
* Get column names, with type="hidden"
*
* @return
*/
public Set<String> getHiddenColumnNames() {
try {
List<Element> selectElements = XPaths.compile("//query/select[contains(@type, 'hidden')]", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(Attribute::getValue)
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
public Set<String> getVisibleColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"hidden".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getNotNumericColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"number".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getColumnNames(final Predicate<Element> filter) {
Preconditions.checkNotNull(filter);
try {
List<Element> selectElements = XPaths.compile("//query/select", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
// Apply filter
.filter(filter::evaluate)
// Get alias
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(Attribute::getValue)
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new Sumar