Commit a2d8786b authored by LAVENIER's avatar LAVENIER
Browse files

Merge branch 'release/1.9.0'

parents c6a59a74 cd837d60
...@@ -12,6 +12,7 @@ stages: ...@@ -12,6 +12,7 @@ stages:
- test - test
- release - release
- docker - docker
- publish
- docs - docs
# --------------------------------------------------------------- # ---------------------------------------------------------------
...@@ -28,6 +29,8 @@ variables: ...@@ -28,6 +29,8 @@ variables:
MAVEN_REPO_URL: https://gitlab.ifremer.fr/api/v4/projects/1272/packages/maven MAVEN_REPO_URL: https://gitlab.ifremer.fr/api/v4/projects/1272/packages/maven
APP_CORE_MODULE: sumaris-core APP_CORE_MODULE: sumaris-core
APP_DB_MODULE: sumaris-db APP_DB_MODULE: sumaris-db
APP_SHARED_MODULE: sumaris-core-shared
APP_CORE_MODULE: sumaris-core
APP_WAR_MODULE: sumaris-server APP_WAR_MODULE: sumaris-server
ARTIFACT_WAR_FILE: "${APP_WAR_MODULE}/target/*.war" ARTIFACT_WAR_FILE: "${APP_WAR_MODULE}/target/*.war"
ARTIFACT_CONFIG_FILES: "${APP_WAR_MODULE}/target/classes/*.properties" ARTIFACT_CONFIG_FILES: "${APP_WAR_MODULE}/target/classes/*.properties"
...@@ -71,7 +74,7 @@ variables: ...@@ -71,7 +74,7 @@ variables:
- ${ARTIFACT_CONFIG_FILES} - ${ARTIFACT_CONFIG_FILES}
reports: reports:
dotenv: ${ENV_FILE} dotenv: ${ENV_FILE}
expire_in: 60 minutes expire_in: 72 hours
build: build:
extends: .build extends: .build
...@@ -79,6 +82,7 @@ build: ...@@ -79,6 +82,7 @@ build:
only: only:
- develop - develop
- feature/imagine - feature/imagine
- tags
failsafe-build: failsafe-build:
extends: .build extends: .build
...@@ -114,12 +118,43 @@ docker:ci: ...@@ -114,12 +118,43 @@ docker:ci:
artifacts: artifacts:
paths: paths:
- target/ci/Dockerfile - target/ci/Dockerfile
expire_in: 60 minutes expire_in: 72 hours
# --------------------------------------------------------------- # ---------------------------------------------------------------
# Test jobs # Test jobs
# --------------------------------------------------------------- # ---------------------------------------------------------------
# ---------------------------------------------------------------
# Tests jobs
# ---------------------------------------------------------------
.test:
stage: test
tags: [sih_public_runner]
script:
- mvn verify surefire-report:report -s ./ci_settings.xml -q
environment:
name: test
artifacts:
paths:
- ${APP_SHARED_MODULE}/site/surefire-report.html
- ${APP_SHARED_MODULE}/target/surefire-reports/*
- ${ARTIFACT_CORE_MODULE}/site/surefire-report.html
- ${ARTIFACT_CORE_MODULE}/target/surefire-reports/*
- ${ARTIFACT_WAR_MODULE}/target/surefire-reports/*
- ${ARTIFACT_WAR_MODULE}/site/surefire-report.html
expire_in: 72 hours
test:
extends: .test
image: ${CI_BUILD_IMAGE}
rules:
- if: $CI_COMMIT_TAG
when: never
- if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_BRANCH =~ /^feature/'
when: manual
allow_failure: true
sonarqube: sonarqube:
stage: test stage: test
tags: tags:
...@@ -143,68 +178,48 @@ sonarqube: ...@@ -143,68 +178,48 @@ sonarqube:
- develop - develop
when: manual when: manual
# ---------------------------------------------------------------
# Deploy jobs
# ---------------------------------------------------------------
#.deploy-file:
# stage: deploy
# tags: [sih_public_runner]
# script:
# - APP_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version -q -DforceStdout)
# - APP_GROUP_ID=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.groupId -q -DforceStdout)
# - mvn -s ./ci_settings.xml deploy:deploy-file -DgroupId=${APP_GROUP_ID} -DartifactId=${APP_WAR_MODULE} -Dversion=${APP_VERSION} -Dfile=${APP_WAR_MODULE}/target/${APP_WAR_MODULE}-${APP_VERSION}.war -DrepositoryId=job-maven -Durl=$MAVEN_REPO_URL
# only:
# - develop
#
#fast-deploy-file:
# extends: .deploy-file
# needs: [fast-build]
#
#failsafe-deploy-file:
# extends: .deploy-file
# needs: [failsafe-build]
# --------------------------------------------------------------- # ---------------------------------------------------------------
# Release jobs # Release jobs
# --------------------------------------------------------------- # ---------------------------------------------------------------
.release: .release:
stage: release stage: release
tags: [sih_public_runner] tags: [sih_public_runner]
before_script: before_script:
- git remote set-url origin ${CI_REPOSITORY_URL} - git remote set-url origin ${CI_REPOSITORY_URL}
- git config --global user.email "${GITLAB_USER_EMAIL}" - git config --global user.email "${GITLAB_USER_EMAIL}"
- git config --global user.name "${GITLAB_USER_NAME}" - git config --global user.name "${GITLAB_USER_NAME}"
- git reset --hard develop - git reset --hard develop
- git fetch origin - git fetch origin
script: script:
- echo "Preparing release..." - cd src/scripts
- git branch -D release/$RELEASE_VERSION || true - ./release.sh pre ${RELEASE_VERSION}
- git pull origin develop && git checkout develop && git status # - echo "Preparing release..."
- mvn -s ./ci_settings.xml -B gitflow:release-start --batch-mode -DreleaseVersion="$RELEASE_VERSION" # - git branch -D release/$RELEASE_VERSION || true
- echo "Performing release..." # - git pull origin develop && git checkout develop && git status
- mvn clean install -s ./ci_settings.xml --batch-mode -DperformRelease -DskipTests # - mvn -s ./ci_settings.xml -B gitflow:release-start --batch-mode -DreleaseVersion="$RELEASE_VERSION"
- echo "Creating test DB..." # - echo "Performing release..."
- mvn -pl ${APP_CORE_MODULE} -q -Prun,hsqldb -DskipTests --batch-mode # - mvn clean install -s ./ci_settings.xml --batch-mode -DperformRelease -DskipTests
- cd ${APP_CORE_MODULE}/target && tar -cf "${APP_DB_MODULE}-${RELEASE_VERSION}.tar.gz" db && cd ../.. # - echo "Creating test DB..."
- echo "Pushing changes to upstream" # - mvn -pl ${APP_CORE_MODULE} -q -Prun,hsqldb -DskipTests --batch-mode
- git commit -a -m "Release $RELEASE_VERSION" # - cd ${APP_CORE_MODULE}/target && tar -cf "${APP_DB_MODULE}-${RELEASE_VERSION}.tar.gz" db && cd ../..
- git status # - echo "Pushing changes to upstream"
- mvn gitflow:release-finish # - git commit -a -m "Release $RELEASE_VERSION"
after_script: # - git status
- echo "APP_VERSION=${RELEASE_VERSION}" > ${ENV_FILE} # - mvn gitflow:release-finish
artifacts: after_script:
paths: - echo "APP_VERSION=${RELEASE_VERSION}" > ${ENV_FILE}
- ${ARTIFACT_WAR_FILE} artifacts:
- ${ARTIFACT_CONFIG_FILES} paths:
- ${APP_WAR_MODULE}/target/*.tar.gz - ${ARTIFACT_WAR_FILE}
- ${APP_WAR_MODULE}/target/*.zip - ${ARTIFACT_CONFIG_FILES}
reports: - ${APP_WAR_MODULE}/target/*.tar.gz
dotenv: ${ENV_FILE} - ${APP_WAR_MODULE}/target/*.zip
only: reports:
- develop dotenv: ${ENV_FILE}
when: manual only:
- develop
when: manual
release: release:
extends: .release extends: .release
...@@ -239,7 +254,10 @@ gitlab-release: ...@@ -239,7 +254,10 @@ gitlab-release:
extends: .docker extends: .docker
stage: docker stage: docker
script: script:
# Create the target directory # Build variables: set default values
- export IMAGE_USERID=${IMAGE_USERID:-20001}
- export IMAGE_GROUPID=${IMAGE_GROUPID:-20001}
# Create the target directory, to build the image
- mkdir -p target/docker/config - mkdir -p target/docker/config
- cp -rf ${APP_WAR_MODULE}/src/main/docker/* target/docker/ - cp -rf ${APP_WAR_MODULE}/src/main/docker/* target/docker/
- cp -f ${ARTIFACT_WAR_FILE} target/docker/app.war - cp -f ${ARTIFACT_WAR_FILE} target/docker/app.war
...@@ -248,25 +266,25 @@ gitlab-release: ...@@ -248,25 +266,25 @@ gitlab-release:
- cd target/docker - cd target/docker
# Build docker image # Build docker image
- docker pull ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} || true - docker pull ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} || true
- docker build --cache-from ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} --pull -t ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} . - docker build --cache-from ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} --pull -t ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} --build-arg="USERID=${IMAGE_USERID}" --build-arg="GROUPID=${IMAGE_GROUPID}" .
#- docker build --no-cache -t ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} . #- docker build --no-cache -t ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} --build-arg="USERID=${IMAGE_USERID}" --build-arg="GROUPID=${IMAGE_GROUPID}" .
- docker push ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} - docker push ${CI_REGISTRY_IMAGE}:${IMAGE_TAG}
artifacts: artifacts:
paths: paths:
- target/docker - target/docker
expire_in: 60 minutes expire_in: 72 hours
docker:develop: docker:develop:
extends: .docker-build extends: .docker-build
variables: variables:
IMAGE_TAG: develop IMAGE_TAG: 'develop'
only: only:
- develop - develop
docker:imagine: docker:imagine:
extends: .docker-build extends: .docker-build
variables: variables:
IMAGE_TAG: imagine IMAGE_TAG: 'imagine'
only: only:
- feature/imagine - feature/imagine
...@@ -277,28 +295,81 @@ docker:release: ...@@ -277,28 +295,81 @@ docker:release:
only: only:
- tags - tags
docker:production:
extends: .docker-build
variables:
IMAGE_TAG: 'production'
IMAGE_USERID: ${PRODUCTION_IMAGE_USERID}
IMAGE_GROUPID: ${PRODUCTION_IMAGE_GROUPID}
only:
- tags
- feature/imagine
when: manual
# --------------------------------------------------------------- # ---------------------------------------------------------------
# Docs jobs # Publish jobs
# --------------------------------------------------------------- # ---------------------------------------------------------------
pages: .deploy-file:
stage: docs stage: publish
tags: [sih_public_runner] tags: [sih_public_runner]
image: python:3.7-alpine
before_script:
- git submodule foreach git fetch origin
- git submodule foreach git pull
script: script:
- Echo "Generate SVG images" - APP_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version -q -DforceStdout)
- git submodule foreach ./generate.sh - APP_GROUP_ID=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.groupId -q -DforceStdout)
- mvn -s ./ci_settings.xml site -q -Prun,hsqldb -DperformRelease - mvn deploy:deploy-file -s ./ci_settings.xml -DgroupId=${APP_GROUP_ID} -DartifactId=${APP_WAR_MODULE} -Dversion=${APP_VERSION} -Dfile=${APP_WAR_MODULE}/target/${APP_WAR_MODULE}-${APP_VERSION}.war -DrepositoryId=job-maven -Durl=$MAVEN_REPO_URL
after_script:
- mv target/site public deploy-file:
artifacts: extends: .deploy-file
paths: needs: [build]
- public only:
- develop
when: manual
failsafe-deploy-file:develop:
extends: .deploy-file
needs: [failsafe-build]
only: only:
- develop - develop
when: manual when: manual
deploy-file:release:
extends: .deploy-file
image: ${CI_BUILD_IMAGE}
needs: [build]
allow_failure: true
only:
- tags
# ---------------------------------------------------------------
# Docs jobs
# ---------------------------------------------------------------
.pages:
stage: docs
image: ${CI_BUILD_IMAGE}
tags: [sih_public_runner]
before_script:
- git submodule foreach git fetch origin
- git submodule foreach git pull
script:
- echo "---- Generating SVG, from '*.plantuml' files..."
- git submodule foreach ./generate.sh || true
- echo "---- Generating Maven site..."
- mvn site -s ./ci_settings.xml -q -Phsqldb -DperformRelease
after_script:
- mv target/site public
allow_failure: true
artifacts:
paths:
- public
pages:develop:
extends: .pages
only:
- develop
when: manual
pages:release:
extends: .pages
only:
- tags
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
<groupId>net.sumaris</groupId> <groupId>net.sumaris</groupId>
<artifactId>sumaris-pod</artifactId> <artifactId>sumaris-pod</artifactId>
<version>1.8.7</version> <version>1.9.0</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>SUMARiS</name> <name>SUMARiS</name>
<description>SUMARiS :: Maven parent</description> <description>SUMARiS :: Maven parent</description>
......
Subproject commit 3b6e149ac009099f8234c3825cf8bb3bacb01f97 Subproject commit 7a81f76fac53e140fcd83394212b153bc9de24a0
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>net.sumaris</groupId> <groupId>net.sumaris</groupId>
<artifactId>sumaris-pod</artifactId> <artifactId>sumaris-pod</artifactId>
<version>1.8.7</version> <version>1.9.0</version>
</parent> </parent>
<artifactId>sumaris-core-extraction</artifactId> <artifactId>sumaris-core-extraction</artifactId>
......
...@@ -255,6 +255,7 @@ public class ExtractionStrategyDaoImpl<C extends ExtractionStrategyContextVO, F ...@@ -255,6 +255,7 @@ public class ExtractionStrategyDaoImpl<C extends ExtractionStrategyContextVO, F
// Bind some referential ids // Bind some referential ids
xmlQuery.bind("strategyLabelPmfmId", String.valueOf(PmfmEnum.STRATEGY_LABEL.getId())); xmlQuery.bind("strategyLabelPmfmId", String.valueOf(PmfmEnum.STRATEGY_LABEL.getId()));
xmlQuery.bind("tagIdPmfmId", String.valueOf(PmfmEnum.TAG_ID.getId()));
return xmlQuery; return xmlQuery;
} }
......
...@@ -125,10 +125,10 @@ public class Daos extends net.sumaris.core.dao.technical.Daos { ...@@ -125,10 +125,10 @@ public class Daos extends net.sumaris.core.dao.technical.Daos {
} }
public static void commitIfHsqldb(DataSource dataSource) { public static void commitIfHsqldbOrPgsql(DataSource dataSource) {
Connection conn = DataSourceUtils.getConnection(dataSource); Connection conn = DataSourceUtils.getConnection(dataSource);
try { try {
if (net.sumaris.core.extraction.dao.technical.Daos.isHsqlDatabase(conn) && DataSourceUtils.isConnectionTransactional(conn, dataSource)) { if ((net.sumaris.core.extraction.dao.technical.Daos.isHsqlDatabase(conn) || net.sumaris.core.extraction.dao.technical.Daos.isPostgresqlDatabase(conn)) && DataSourceUtils.isConnectionTransactional(conn, dataSource)) {
try { try {
conn.commit(); conn.commit();
} catch (SQLException e) { } catch (SQLException e) {
...@@ -146,7 +146,7 @@ public class Daos extends net.sumaris.core.dao.technical.Daos { ...@@ -146,7 +146,7 @@ public class Daos extends net.sumaris.core.dao.technical.Daos {
* @param columnNamesMapping * @param columnNamesMapping
* @return * @return
*/ */
public static String sqlReplaceColumnNames(String sqlQuery, Map<String, String> columnNamesMapping) { public static String sqlReplaceColumnNames(String sqlQuery, Map<String, String> columnNamesMapping, Boolean lowercase) {
if (MapUtils.isEmpty(columnNamesMapping)) return sqlQuery; // Skip if (MapUtils.isEmpty(columnNamesMapping)) return sqlQuery; // Skip
sqlQuery = sqlQuery.toUpperCase(); sqlQuery = sqlQuery.toUpperCase();
...@@ -161,6 +161,20 @@ public class Daos extends net.sumaris.core.dao.technical.Daos { ...@@ -161,6 +161,20 @@ public class Daos extends net.sumaris.core.dao.technical.Daos {
"$1" + targetColumnName + "$2"); "$1" + targetColumnName + "$2");
} }
// for postgresql, lowercase queries are needed.
if (lowercase){
sqlQuery = sqlQuery.toLowerCase();
}
return sqlQuery; return sqlQuery;
} }
/**
* Do column names replacement, but escape sql keyword (e.g. 'DATE' replacement will keep TO_DATE(...) unchanged)
* @param sqlQuery
* @param columnNamesMapping
* @return
*/
public static String sqlReplaceColumnNames(String sqlQuery, Map<String, String> columnNamesMapping) {
return sqlReplaceColumnNames(sqlQuery, columnNamesMapping, false);
}
} }
...@@ -41,12 +41,9 @@ import net.sumaris.core.service.referential.ReferentialService; ...@@ -41,12 +41,9 @@ import net.sumaris.core.service.referential.ReferentialService;
import net.sumaris.core.util.StringUtils; import net.sumaris.core.util.StringUtils;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.hibernate.dialect.Dialect; import org.hibernate.dialect.Dialect;
import org.hibernate.query.internal.NativeQueryImpl;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.dao.DataRetrievalFailureException; import org.springframework.dao.DataRetrievalFailureException;
import org.springframework.orm.hibernate5.SessionHolder;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.persistence.Query; import javax.persistence.Query;
...@@ -184,7 +181,7 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport { ...@@ -184,7 +181,7 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
xmlQuery.setXSLFileName(XSL_ORACLE_FILENAME); xmlQuery.setXSLFileName(XSL_ORACLE_FILENAME);
} }
else if (this.databaseType == DatabaseType.postgresql){ else if (this.databaseType == DatabaseType.postgresql){
xmlQuery.setLowercaseAliases(true); xmlQuery.setLowercase(true);
} }
return xmlQuery; return xmlQuery;
} }
......
...@@ -49,18 +49,18 @@ public class XMLQuery extends HSQLDBSingleXMLQuery { ...@@ -49,18 +49,18 @@ public class XMLQuery extends HSQLDBSingleXMLQuery {
private String xslFileName; private String xslFileName;
private boolean lowercaseAliases; private boolean lowercase;
public XMLQuery() { public XMLQuery() {
super(); super();
xslFileName = super.getXSLFileName(); xslFileName = super.getXSLFileName();
this.lowercaseAliases = false; this.lowercase = false;
} }
public XMLQuery(boolean lowercaseAliases) { public XMLQuery(boolean lowercase) {
super(); super();
xslFileName = super.getXSLFileName(); xslFileName = super.getXSLFileName();
this.lowercaseAliases = lowercaseAliases; this.lowercase = lowercase;
} }
@Override @Override
...@@ -74,12 +74,12 @@ public class XMLQuery extends HSQLDBSingleXMLQuery { ...@@ -74,12 +74,12 @@ public class XMLQuery extends HSQLDBSingleXMLQuery {
// let default values here for HSQLDB // let default values here for HSQLDB
public boolean isLowercaseAliases() { public boolean isLowercase() {
return lowercaseAliases; return lowercase;
} }
public void setLowercaseAliases(boolean lowercaseAliases) { public void setLowercase(boolean lowercase) {
this.lowercaseAliases = lowercaseAliases; this.lowercase = lowercase;
} }
/** /**
...@@ -155,7 +155,7 @@ public class XMLQuery extends HSQLDBSingleXMLQuery { ...@@ -155,7 +155,7 @@ public class XMLQuery extends HSQLDBSingleXMLQuery {
public String getSQLQueryAsString(){ public String getSQLQueryAsString(){
String query = super.getSQLQueryAsString(); String query = super.getSQLQueryAsString();
if (this.lowercaseAliases){ if (this.lowercase){
query = query.toLowerCase(); query = query.toLowerCase();
} }
return query; return query;
......
...@@ -196,7 +196,7 @@ public class SumarisTableMetadatas { ...@@ -196,7 +196,7 @@ public class SumarisTableMetadatas {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("SELECT COUNT(*) FROM (") sb.append("SELECT COUNT(*) FROM (")
.append(getSelectGroupByQuery(tableName, columnNames, whereClause, groupByColumnNames, null, null)) .append(getSelectGroupByQuery(tableName, columnNames, whereClause, groupByColumnNames, null, null))
.append(")"); .append(") \"select_group\"");
return sb.toString(); return sb.toString();
} }
......
...@@ -23,6 +23,7 @@ ...@@ -23,6 +23,7 @@
package net.sumaris.core.extraction.dao.trip.free; package net.sumaris.core.extraction.dao.trip.free;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import net.sumaris.core.dao.technical.DatabaseType;
import net.sumaris.core.extraction.dao.technical.Daos; import net.sumaris.core.extraction.dao.technical.Daos;
import net.sumaris.core.extraction.dao.technical.XMLQuery; import net.sumaris.core.extraction.dao.technical.XMLQuery;