Skip to content

GitLab

Commits (25)
Hide whitespace changes
Inline Side-by-side
Showing with 2137 additions and 1084 deletions
CHANGELOG.md
View file @ 7c593026
## Sprint 84 - v3.8.3
## Sprint 88 - v3.9.5
- Pas de mise à jour de modèle
## Sprint 87 - v3.9.4
- Pas de mise à jour de modèle
## Sprint 86 - v3.9.3
- Pas de mise à jour de modèle
## Sprint 85 - v3.8.3
- Pas de mise à jour de modèle
- Version corrective devant utiliser le server de synchronisation 3.6.3.2 uniquement
## Sprint 84 - v3.8.2
- Pas de mise à jour de modèle
- Version corrective devant utiliser le server de synchronisation 3.6.3.1 uniquement
## Sprint 83 - v3.9.2
- Pas de mise à jour de modèle
- Une nouvelle option a été ajoutée pour gérer les transcodages des référentiels suivant :
reefdb.transcribingItemType.label.pmfmExtraction
Sa valeur par défaut est `REEFDB-PMFM.PMFM_EXTRACTION`. Couplée avec l'option `reefdb.transcribingItemType.label.pmfmNm`, elle permet de transcoder les libellé des PSFMU lors des extractions avec la priorité suivante:
1- libellé transcodé pour extraction
2- libellé transcodé pour écran de saisie
3- libellé interne du psfmu (issu du transcodage pour écran de saisie)
4- libellé du paramètre
- Pour simplifier la configuration du serveur de synchronisation, l'option `quadrige3.synchro.import.referential.transcribingItemType.label.includes`
qui doit contenir tous les libellés des types de transcodage à importer, accepte désormais le caractère '%'.
quadrige3.synchro.import.referential.transcribingItemType.label.includes=REEFDB% (tous les types de transcodage commençant par REEFDB seront importés)
## Sprint 82 - v3.9.1
- Pas de mise à jour de modèle
## Sprint 81 - v3.9.0
- Pas de mise à jour de modèle
## Sprint 80 - v3.8.1
- Pas de mise à jour de modèle
......
pom.xml
View file @ 7c593026
......@@ -10,7 +10,7 @@
<groupId>fr.ifremer.reefdb</groupId>
<artifactId>reefdb</artifactId>
<version>3.8.3</version>
<version>3.9.5</version>
<packaging>pom</packaging>
<name>Reef DB</name>
......@@ -171,7 +171,7 @@
<maven.compiler.debug>true</maven.compiler.debug>
<!-- Quadrige3 Core version -->
<quadrige3-core.version>3.6.3.2</quadrige3-core.version>
<quadrige3-core.version>3.6.10</quadrige3-core.version>
<!-- Last ReefDb launcher version -->
<launcherVersion>3.0.3</launcherVersion>
......@@ -261,7 +261,7 @@
<repositories>
<repository>
<id>reefdb-public-group</id>
<url>http://server.e-is.pro/nexus/content/groups/reefdb</url>
<url>https://nexus.e-is.pro/nexus/content/groups/reefdb</url>
<snapshots>
<enabled>true</enabled>
<checksumPolicy>fail</checksumPolicy>
......@@ -276,7 +276,7 @@
<pluginRepositories>
<pluginRepository>
<id>reefdb-public-group</id>
<url>http://server.e-is.pro/nexus/content/groups/reefdb</url>
<url>https://nexus.e-is.pro/nexus/content/groups/reefdb</url>
<snapshots>
<enabled>true</enabled>
<checksumPolicy>fail</checksumPolicy>
......@@ -342,6 +342,18 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit-addons</groupId>
<artifactId>junit-addons</artifactId>
<version>1.4</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>fr.ifremer.quadrige3</groupId>
<artifactId>quadrige3-test-shared</artifactId>
......@@ -1119,10 +1131,10 @@
<distribution.site.repository>scpexe://${distribution.site.host}${distribution.site.path}</distribution.site.repository>
<distribution.repository.id>eis-nexus-deploy</distribution.repository.id>
<distribution.repository.url>http://server.e-is.pro/nexus/content/repositories/reefdb-releases</distribution.repository.url>
<distribution.repository.url>https://nexus.e-is.pro/nexus/content/repositories/reefdb-releases</distribution.repository.url>
<distribution.snapshotRepository.id>eis-nexus-deploy</distribution.snapshotRepository.id>
<distribution.snapshotRepository.url>http://server.e-is.pro/nexus/content/repositories/reefdb-snapshots</distribution.snapshotRepository.url>
<distribution.snapshotRepository.url>https://nexus.e-is.pro/nexus/content/repositories/reefdb-snapshots</distribution.snapshotRepository.url>
<distribution.internet.url>${distribution.repository.url}</distribution.internet.url>
</properties>
......
reefdb-converter/pom.xml
View file @ 7c593026
......@@ -76,7 +76,7 @@
<repositories>
<repository>
<id>reefdb-public-group</id>
<url>http://server.e-is.pro/nexus/content/groups/reefdb</url>
<url>https://nexus.e-is.pro/nexus/content/groups/reefdb</url>
<snapshots>
<enabled>true</enabled>
<checksumPolicy>fail</checksumPolicy>
......@@ -91,7 +91,7 @@
<pluginRepositories>
<pluginRepository>
<id>reefdb-public-group</id>
<url>http://server.e-is.pro/nexus/content/groups/reefdb</url>
<url>https://nexus.e-is.pro/nexus/content/groups/reefdb</url>
<snapshots>
<enabled>true</enabled>
<checksumPolicy>fail</checksumPolicy>
......@@ -242,12 +242,12 @@
<properties>
<distribution.repository.id>eis-nexus-deploy</distribution.repository.id>
<distribution.repository.url>http://server.e-is.pro/nexus/content/repositories/reefdb-releases</distribution.repository.url>
<distribution.repository.url>https://nexus.e-is.pro/nexus/content/repositories/reefdb-releases</distribution.repository.url>
<distribution.snapshotRepository.id>eis-nexus-deploy</distribution.snapshotRepository.id>
<distribution.snapshotRepository.url>http://server.e-is.pro/nexus/content/repositories/reefdb-snapshots</distribution.snapshotRepository.url>
<distribution.snapshotRepository.url>https://nexus.e-is.pro/nexus/content/repositories/reefdb-snapshots</distribution.snapshotRepository.url>
<distribution.internet.url>http://server.e-is.pro/nexus/content/groups/public</distribution.internet.url>
<distribution.internet.url>https://nexus.e-is.pro/nexus/content/groups/public</distribution.internet.url>
</properties>
</profile>
......
reefdb-core/pom.xml
View file @ 7c593026
......@@ -4,7 +4,7 @@
<parent>
<groupId>fr.ifremer.reefdb</groupId>
<artifactId>reefdb</artifactId>
<version>3.8.3</version>
<version>3.9.5</version>
</parent>
<artifactId>reefdb-core</artifactId>
......@@ -60,6 +60,16 @@
<artifactId>quadrige3-test-shared</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit-addons</groupId>
<artifactId>junit-addons</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
......
reefdb-core/src/license/THIRD-PARTY.properties
View file @ 7c593026
......@@ -35,8 +35,11 @@
# Please fill the missing licenses for dependencies :
#
#
#Fri Feb 09 17:30:26 CET 2018
#Fri Mar 06 09:36:42 CET 2020
com.oracle--ojdbc7--12.1.0.2.0=OTN license
commons-primitives--commons-primitives--1.0=The Apache Software License, Version 2.0
dom4j--dom4j--1.6.1=BSD License
junit-addons--junit-addons--1.4=
org.hibernate--hibernate-spatial--4.3-20140213=Lesser General Public License (LGPL)
xerces--xercesImpl--2.6.2=
xerces--xmlParserAPIs--2.6.2=
reefdb-core/src/main/java/fr/ifremer/reefdb/config/ReefDbConfiguration.java
View file @ 7c593026
......@@ -967,6 +967,10 @@ public final class ReefDbConfiguration extends QuadrigeCoreConfiguration {
return applicationConfig.getOption(ReefDbConfigurationOption.TRANSCRIBING_ITEM_TYPE_LB_PMFM_NM.getKey());
}
public String getTranscribingItemTypeLbForPmfmExtraction() {
return applicationConfig.getOption(ReefDbConfigurationOption.TRANSCRIBING_ITEM_TYPE_LB_PMFM_EXTRACTION.getKey());
}
/**
* <p>getAlternativeTaxonOriginTaxRef.</p>
*
......
reefdb-core/src/main/java/fr/ifremer/reefdb/config/ReefDbConfigurationOption.java
View file @ 7c593026
......@@ -24,7 +24,6 @@ package fr.ifremer.reefdb.config;
*/
import fr.ifremer.quadrige3.core.dao.referential.UnitId;
import fr.ifremer.quadrige3.core.dao.referential.transcribing.TranscribingItemTypeLb;
import fr.ifremer.quadrige3.core.dao.technical.QuadrigeEnumerationDef;
import org.nuiton.config.ConfigOptionDef;
import org.nuiton.version.Version;
......@@ -303,7 +302,13 @@ public enum ReefDbConfigurationOption implements ConfigOptionDef {
TRANSCRIBING_ITEM_TYPE_LB_PMFM_NM(
"reefdb.transcribingItemType.label.pmfmNm",
n("reefdb.config.option.transcribingItemType.label.pmfmNm.description"),
String.format("${%s}", TranscribingItemTypeLb.REEFDB_PMFM_NM.getKey()), // 1
String.format("${%s%s}", QuadrigeEnumerationDef.CONFIG_OPTION_PREFIX, "TranscribingItemTypeLb.REEFDB_PMFM_NM"),
String.class,
false),
TRANSCRIBING_ITEM_TYPE_LB_PMFM_EXTRACTION(
"reefdb.transcribingItemType.label.pmfmExtraction",
n("reefdb.config.option.transcribingItemType.label.pmfmExtraction.description"),
String.format("${%s%s}", QuadrigeEnumerationDef.CONFIG_OPTION_PREFIX, "TranscribingItemTypeLb.REEFDB_PMFM_EXTRACTION"),
String.class,
false),
/* constants used to calculate transition length */
......
reefdb-core/src/main/java/fr/ifremer/reefdb/dao/administration/strategy/ReefDbStrategyDao.java
View file @ 7c593026
......@@ -102,6 +102,16 @@ public interface ReefDbStrategyDao {
@Cacheable(value = PMFM_STRATEGIES_BY_PROG_LOC_DATE_CACHE)
Set<PmfmStrategyDTO> getPmfmStrategiesByProgramCodeAndLocation(String programCode, Integer monitoringLocationId, LocalDate date);
/**
* Retrieve all PMFM strategies that are applied on given programs and date range
*
* @param programCodes program codes
* @param startDate
* @param endDate
* @return set of pmfm strategies
*/
Set<PmfmStrategyDTO> getPmfmStrategiesByProgramCodesAndDates(Collection<String> programCodes, LocalDate startDate, LocalDate endDate);
/**
* Retrieve all strategies that are applied on the given program and monitoring location
*
......
reefdb-core/src/main/java/fr/ifremer/reefdb/dao/administration/strategy/ReefDbStrategyDaoImpl.java
View file @ 7c593026
......@@ -241,6 +241,39 @@ public class ReefDbStrategyDaoImpl extends StrategyDaoImpl implements ReefDbStra
return ImmutableSet.copyOf(pmfmStrategies);
}
@Override
public Set<PmfmStrategyDTO> getPmfmStrategiesByProgramCodesAndDates(Collection<String> programCodes, LocalDate startDate, LocalDate endDate) {
Set<PmfmStrategyDTO> pmfmStrategies = new HashSet<>();
List<ProgStratDTO> appliedStrategies = getAppliedStrategiesByProgramCodes(programCodes);
for (ProgStratDTO appliedStrategy: appliedStrategies) {
// filter on date range
if (appliedStrategy.getStartDate() != null && appliedStrategy.getEndDate() != null
&& !appliedStrategy.getStartDate().isAfter(endDate) && !appliedStrategy.getEndDate().isBefore(startDate)) {
// add all pmfm strategies in date range
pmfmStrategies.addAll(getPmfmsAppliedStrategy(appliedStrategy.getId()));
}
}
return ImmutableSet.copyOf(pmfmStrategies);
}
@SuppressWarnings("unchecked")
private List<ProgStratDTO> getAppliedStrategiesByProgramCodes(Collection<String> programCodes) {
Query query = createQuery("appliedStrategiesByProgramCodes").setParameterList("programCodes", programCodes);
Iterator<Object[]> rows = query.iterate();
List<ProgStratDTO> result = Lists.newArrayList();
TimeZone dbTimezone = config.getDbTimezone();
while (rows.hasNext()) {
result.add(toProgStratDTO(Arrays.asList(rows.next()).iterator(), dbTimezone));
}
return result;
}
/** {@inheritDoc} */
@Override
public List<ProgStratDTO> getStrategiesByProgramCodeAndMonitoringLocationId(String programCode, int monitoringLocationId) {
......@@ -915,7 +948,7 @@ public class ReefDbStrategyDaoImpl extends StrategyDaoImpl implements ReefDbStra
// code, name
program.setCode((String) source.next());
program.setName((String) source.next());
//TODO (report Q3): program.setStatus(Daos.getStatus((String) source.next()));
program.setStatus(Daos.getStatus((String) source.next()));
}
// strategy: id, name
......
reefdb-core/src/main/java/fr/ifremer/reefdb/dao/system/extraction/ReefDbExtractionResultDao.java
View file @ 7c593026
......@@ -24,6 +24,7 @@ package fr.ifremer.reefdb.dao.system.extraction;
*/
import fr.ifremer.quadrige3.core.dao.technical.csv.CSVDao;
import fr.ifremer.reefdb.dto.referential.pmfm.PmfmDTO;
import org.springframework.jdbc.core.RowMapper;
import java.util.List;
......@@ -36,6 +37,14 @@ import java.util.Map;
*/
public interface ReefDbExtractionResultDao extends CSVDao {
/**
* <p>queryCount.</p>
*
* @param query a {@link String} object.
* @return a {@link java.lang.Long} object.
*/
long queryCount(String query);
/**
* <p>queryCount.</p>
*
......@@ -67,6 +76,14 @@ public interface ReefDbExtractionResultDao extends CSVDao {
List<String> queryStringList(String query, Map<String, Object> queryBindings);
/**
* <p>queryUpdate</p>
*
* @param query a {@link java.lang.String} object.
* @return a int.
*/
int queryUpdate(String query);
/**
* <p>queryUpdate.</p>
*
......@@ -75,4 +92,10 @@ public interface ReefDbExtractionResultDao extends CSVDao {
* @return a int.
*/
int queryUpdate(String query, Map<String, Object> queryBindings);
String getPmfmNameForExtraction(PmfmDTO pmfm);
String getPmfmUnitNameForExtraction(PmfmDTO pmfm);
}
reefdb-core/src/main/java/fr/ifremer/reefdb/dao/system/extraction/ReefDbExtractionResultDaoImpl.java
View file @ 7c593026
......@@ -23,18 +23,24 @@ package fr.ifremer.reefdb.dao.system.extraction;
* #L%
*/
import fr.ifremer.quadrige3.core.dao.referential.transcribing.TranscribingItemExtendDao;
import fr.ifremer.quadrige3.core.dao.technical.Assert;
import fr.ifremer.quadrige3.core.dao.technical.csv.CSVDaoImpl;
import fr.ifremer.reefdb.config.ReefDbConfiguration;
import fr.ifremer.reefdb.dto.referential.pmfm.PmfmDTO;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.dao.DataRetrievalFailureException;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Repository;
import javax.annotation.Resource;
import javax.sql.DataSource;
import java.math.BigInteger;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
/**
......@@ -50,6 +56,12 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
protected final Properties connectionProperties;
@Resource
protected ReefDbConfiguration config;
@Resource(name = "transcribingItemDao")
private TranscribingItemExtendDao transcribingItemDao;
/**
* <p>Constructor for ReefDbExtractionResultDaoImpl.</p>
*
......@@ -78,24 +90,19 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
this.connectionProperties = connectionProperties;
}
@Override
public long queryCount(String query) {
return queryCount(query, null);
}
/**
* {@inheritDoc}
*/
@Override
public Long queryCount(String query, Map<String, Object> queryBindings) {
return query(connectionProperties, query, queryBindings, resultSet -> {
Object result = resultSet.getObject(1);
if (log.isDebugEnabled()) {
log.debug(String.format("queryCount result: %s", result));
}
if (result instanceof Integer) {
return ((Integer) result).longValue();
} else if (result instanceof BigInteger) {
return ((BigInteger) result).longValue();
}
return (Long) result;
});
Long count = queryCount(connectionProperties, query, queryBindings);
if (count == null) throw new DataRetrievalFailureException(String.format("query count result is null: %s ; bindings=%s", query, queryBindings));
return count;
}
/**
......@@ -106,6 +113,11 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
return query(connectionProperties, sql, queryBindings, rowMapper);
}
@Override
public int queryUpdate(String query) {
return queryUpdate(query, null);
}
/**
* {@inheritDoc}
*/
......@@ -114,6 +126,25 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
return queryUpdate(connectionProperties, query, paramMap);
}
@Override
public String getPmfmNameForExtraction(PmfmDTO pmfm) {
Assert.notNull(pmfm);
return Optional
.ofNullable(transcribingItemDao.getTranscribingItemById(config.getTranscribingItemTypeLbForPmfmExtraction(), pmfm.getId()))
.orElse(
Optional.ofNullable(transcribingItemDao.getTranscribingItemById(config.getTranscribingItemTypeLbForPmfmNm(), pmfm.getId()))
.orElse(
Optional.ofNullable(pmfm.getName())
.orElse(pmfm.getParameter().getName())
)
); }
@Override
public String getPmfmUnitNameForExtraction(PmfmDTO pmfm) {
Assert.notNull(pmfm);
return pmfm.getUnit() == null || config.getExtractionUnitIdsToIgnore().contains(pmfm.getUnit().getId()) ? "" : pmfm.getUnit().getName();
}
/**
* {@inheritDoc}
*/
......@@ -126,4 +157,5 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
public List<String> queryStringList(String query, Map<String, Object> queryBindings) {
return query(connectionProperties, query, queryBindings, (resultSet, i) -> resultSet.getString(1));
}
}
reefdb-core/src/main/java/fr/ifremer/reefdb/decorator/DecoratorServiceImpl.java
View file @ 7c593026
......@@ -287,6 +287,7 @@ public class DecoratorServiceImpl extends fr.ifremer.quadrige3.core.service.deco
}
}
@Deprecated
private class PMFMExtractionDecorator extends PMFMNameDecorator {
static final String SEPARATOR = "-";
......
reefdb-core/src/main/java/fr/ifremer/reefdb/dto/ReefDbBeans.java
View file @ 7c593026
......@@ -53,6 +53,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.nuiton.util.DateUtil;
import javax.annotation.Nonnull;
import java.sql.Array;
import java.sql.SQLException;
import java.time.format.DateTimeFormatter;
......@@ -119,6 +120,10 @@ public class ReefDbBeans extends QuadrigeBeans {
return addressBuilder.toString();
}
public static String toQuotedString(String string) {
return "'" + string.replaceAll("'", "''") + "'";
}
/**
* <p>toString.</p>
*
......@@ -177,8 +182,8 @@ public class ReefDbBeans extends QuadrigeBeans {
} else if (bean instanceof ExtractionPeriodDTO) {
ExtractionPeriodDTO period = (ExtractionPeriodDTO) bean;
return Dates.formatDate(period.getStartDate(), DEFAULT_DATE_FORMAT)
+ " => "
+ Dates.formatDate(period.getEndDate(), DEFAULT_DATE_FORMAT);
+ " => "
+ Dates.formatDate(period.getEndDate(), DEFAULT_DATE_FORMAT);
}
return bean.toString();
......@@ -496,16 +501,30 @@ public class ReefDbBeans extends QuadrigeBeans {
public static boolean haveSameMeasurements(Collection<MeasurementDTO> measurements, Collection<MeasurementDTO> otherMeasurements) {
List<Integer> pmfmIds = measurements.stream()
.filter(measurement -> !isMeasurementEmpty(measurement))
.map(measurement -> measurement.getPmfm().getId())
.collect(Collectors.toList());
.filter(measurement -> !isMeasurementEmpty(measurement))
.map(measurement -> measurement.getPmfm().getId())
.collect(Collectors.toList());
List<Integer> otherPmfmIds = otherMeasurements.stream()
.filter(measurement -> !isMeasurementEmpty(measurement))
.map(measurement -> measurement.getPmfm().getId())
.collect(Collectors.toList());
.filter(measurement -> !isMeasurementEmpty(measurement))
.map(measurement -> measurement.getPmfm().getId())
.collect(Collectors.toList());
return pmfmIds.containsAll(otherPmfmIds) && otherPmfmIds.containsAll(pmfmIds);
}
/**
* Compare measurement values only (not pmfm or other propoerty)
*
* @param measurement1 first measurement
* @param measurement2 second measurement
* @return true if their values are equals or null
*/
public static boolean measurementValuesEquals(@Nonnull MeasurementDTO measurement1, @Nonnull MeasurementDTO measurement2) {
return (measurement1.getNumericalValue() == null && measurement2.getNumericalValue() == null
&& measurement1.getQualitativeValue() == null && measurement2.getQualitativeValue() == null)
|| (measurement1.getNumericalValue() != null && measurement1.getNumericalValue().equals(measurement2.getNumericalValue()))
|| (measurement1.getQualitativeValue() != null && measurement1.getQualitativeValue().equals(measurement2.getQualitativeValue()));
}
/**
* Get the unified comments from all individual measurements
*
......@@ -520,6 +539,38 @@ public class ReefDbBeans extends QuadrigeBeans {
return getUnifiedString(comments, DEFAULT_STRING_SEPARATOR);
}
/**
* Method used as external function for HSQLDB
* see fr.ifremer.reefdb.service.extraction.ExtractionPerformServiceImpl#createDistinctFunction(java.lang.String, java.lang.String, java.lang.String)
*
* @param array the input array (any type)
* @param nonDistinctResult the result text if the array contains non distinct values
* @return the unified string
*/
@SuppressWarnings(value = "unused")
public static String getDistinctSQLString(Array array, String nonDistinctResult) {
try {
Set<String> strings = new HashSet<>();
for (Object object : (Object[]) array.getArray()) {
if (object != null) {
strings.add(object.toString());
}
}
return strings.isEmpty()
? ""
: strings.size() == 1
? strings.stream().findFirst().get()
: nonDistinctResult;
} catch (SQLException e) {
return "";
}
}
/**
* Method used as external function for HSQLDB
* see fr.ifremer.reefdb.service.extraction.ExtractionServiceImpl#createConcatDistinctFunction(java.lang.String, java.lang.String, java.lang.String)
......@@ -642,9 +693,9 @@ public class ReefDbBeans extends QuadrigeBeans {
return new ArrayList<>();
}
return filterCollection(bean.getErrors(), error -> error.isError()
&& (controlOnly == null || (controlOnly == error.isControl()))
&& error.containsPropertyName(propertyName)
&& (pmfmId == null || pmfmId.equals(error.getPmfmId())));
&& (controlOnly == null || (controlOnly == error.isControl()))
&& error.containsPropertyName(propertyName)
&& (pmfmId == null || pmfmId.equals(error.getPmfmId())));
}
/**
......@@ -694,14 +745,14 @@ public class ReefDbBeans extends QuadrigeBeans {
*/
public static List<ErrorDTO> getWarnings(ErrorAware bean, final String propertyName, final Integer pmfmId, final Boolean controlOnly) {
if (bean == null || CollectionUtils.isEmpty(bean.getErrors())
// if an error is found, return empty list because error is priority
|| !getErrors(bean, propertyName, pmfmId, controlOnly).isEmpty()) {
// if an error is found, return empty list because error is priority
|| !getErrors(bean, propertyName, pmfmId, controlOnly).isEmpty()) {
return new ArrayList<>();
}
return filterCollection(bean.getErrors(), error -> error.isWarning()
&& (controlOnly == null || (controlOnly == error.isControl()))
&& error.containsPropertyName(propertyName)
&& (pmfmId == null || pmfmId.equals(error.getPmfmId())));
&& (controlOnly == null || (controlOnly == error.isControl()))
&& error.containsPropertyName(propertyName)
&& (pmfmId == null || pmfmId.equals(error.getPmfmId())));
}
/**
......@@ -794,13 +845,13 @@ public class ReefDbBeans extends QuadrigeBeans {
private static Predicate<ErrorDTO> errorEqualPredicate(ErrorDTO thisError) {
return error -> error.isError() == thisError.isError()
&& error.isWarning() == thisError.isWarning()
&& error.isControl() == thisError.isControl()
&& Objects.equals(error.getControlElementCode(), thisError.getControlElementCode())
&& Objects.deepEquals(error.getPropertyName().toArray(), thisError.getPropertyName().toArray())
&& Objects.equals(error.getPmfmId(), thisError.getPmfmId())
&& Objects.equals(error.getIndividualId(), thisError.getIndividualId())
&& Objects.equals(error.getMessage(), thisError.getMessage());
&& error.isWarning() == thisError.isWarning()
&& error.isControl() == thisError.isControl()
&& Objects.equals(error.getControlElementCode(), thisError.getControlElementCode())
&& Objects.deepEquals(error.getPropertyName().toArray(), thisError.getPropertyName().toArray())
&& Objects.equals(error.getPmfmId(), thisError.getPmfmId())
&& Objects.equals(error.getIndividualId(), thisError.getIndividualId())
&& Objects.equals(error.getMessage(), thisError.getMessage());
}
/**
......@@ -811,9 +862,9 @@ public class ReefDbBeans extends QuadrigeBeans {
*/
public static boolean isPmfmMandatory(ControlRuleDTO controlRule) {
return ControlElementValues.MEASUREMENT.equals(controlRule.getControlElement()) &&
(ControlFeatureMeasurementValues.PMFM.equals(controlRule.getControlFeature())
|| ControlFeatureMeasurementValues.NUMERICAL_VALUE.equals(controlRule.getControlFeature())
|| ControlFeatureMeasurementValues.QUALITATIVE_VALUE.equals(controlRule.getControlFeature()));
(ControlFeatureMeasurementValues.PMFM.equals(controlRule.getControlFeature())
|| ControlFeatureMeasurementValues.NUMERICAL_VALUE.equals(controlRule.getControlFeature())
|| ControlFeatureMeasurementValues.QUALITATIVE_VALUE.equals(controlRule.getControlFeature()));
}
public static boolean isPreconditionRule(ControlRuleDTO controlRule) {
......@@ -826,14 +877,14 @@ public class ReefDbBeans extends QuadrigeBeans {
public static boolean isQualitativeControlRule(ControlRuleDTO controlRule) {
return ControlFunctionValues.IS_AMONG.equals(controlRule.getFunction())
&& ControlElementValues.MEASUREMENT.equals(controlRule.getControlElement())
&& ControlFeatureMeasurementValues.QUALITATIVE_VALUE.equals(controlRule.getControlFeature());
&& ControlElementValues.MEASUREMENT.equals(controlRule.getControlElement())
&& ControlFeatureMeasurementValues.QUALITATIVE_VALUE.equals(controlRule.getControlFeature());
}
public static boolean isNumericalControlRule(ControlRuleDTO controlRule) {
return ControlFunctionValues.MIN_MAX.equals(controlRule.getFunction())
&& ControlElementValues.MEASUREMENT.equals(controlRule.getControlElement())
&& ControlFeatureMeasurementValues.NUMERICAL_VALUE.equals(controlRule.getControlFeature());
&& ControlElementValues.MEASUREMENT.equals(controlRule.getControlElement())
&& ControlFeatureMeasurementValues.NUMERICAL_VALUE.equals(controlRule.getControlFeature());
}
/**
......@@ -844,8 +895,8 @@ public class ReefDbBeans extends QuadrigeBeans {
*/
public static boolean isSurveyValidated(SurveyDTO survey) {
return survey.getValidationDate() != null
|| SynchronizationStatusValues.SYNCHRONIZED.equals(survey.getSynchronizationStatus())
|| SynchronizationStatusValues.READY_TO_SYNCHRONIZE.equals(survey.getSynchronizationStatus());
|| SynchronizationStatusValues.SYNCHRONIZED.equals(survey.getSynchronizationStatus())
|| SynchronizationStatusValues.READY_TO_SYNCHRONIZE.equals(survey.getSynchronizationStatus());
}
/**
......@@ -858,8 +909,8 @@ public class ReefDbBeans extends QuadrigeBeans {
public static FilterDTO getFilterOfType(ExtractionDTO extraction, ExtractionFilterValues extractionFilter) {
return CollectionUtils.isNotEmpty(extraction.getFilters())
? findByProperty(extraction.getFilters(), FilterDTO.PROPERTY_FILTER_TYPE_ID, extractionFilter.getFilterTypeId())
: null;
? findByProperty(extraction.getFilters(), FilterDTO.PROPERTY_FILTER_TYPE_ID, extractionFilter.getFilterTypeId())
: null;
}
/**
......@@ -881,11 +932,28 @@ public class ReefDbBeans extends QuadrigeBeans {
return null;
}
public static void setFilterElements(ExtractionDTO extraction, ExtractionFilterValues extractionFilter, List<? extends QuadrigeBean> elements) {
FilterDTO filter = getFilterOfType(extraction, extractionFilter);
if (filter == null) {
filter = ReefDbBeanFactory.newFilterDTO();
filter.setFilterTypeId(extractionFilter.getFilterTypeId());
extraction.addFilters(filter);
}
filter.setElements(elements);
}
public static List<String> getIdsAsString(Collection<? extends QuadrigeBean> beans) {
if (beans == null)
return null;
return beans.stream().map(bean -> bean instanceof CodeOnly ? ((CodeOnly)bean).getCode() : bean.getId().toString()).collect(Collectors.toList());
return beans.stream().map(bean -> bean instanceof CodeOnly ? ((CodeOnly) bean).getCode() : bean.getId().toString()).collect(Collectors.toList());
}
public static List<ExtractionPeriodDTO> getExtractionPeriods(ExtractionDTO extraction) {
FilterDTO periodFilter = getFilterOfType(extraction, ExtractionFilterValues.PERIOD);
if (periodFilter == null || periodFilter.getElements() == null) return new ArrayList<>();
return periodFilter.getElements().stream().map((Function<QuadrigeBean, ExtractionPeriodDTO>) ExtractionPeriodDTO.class::cast).collect(Collectors.toList());
}
}
reefdb-core/src/main/java/fr/ifremer/reefdb/dto/enums/ExtractionFilterValues.java
View file @ 7c593026
......@@ -36,7 +36,7 @@ public enum ExtractionFilterValues {
PERIOD(null, n("reefdb.core.enums.extractionFilter.period"), false),
PROGRAM(FilterTypeId.PROGRAM, n("reefdb.core.enums.extractionFilter.program"), false),
LOCATION(FilterTypeId.MONITORING_LOCATION, n("reefdb.core.enums.extractionFilter.location"), false),
// CAMPAIGN(FilterTypeId.CAMPAIGN, n("reefdb.core.enums.extractionFilter.campaign"), false),
CAMPAIGN(FilterTypeId.CAMPAIGN, n("reefdb.core.enums.extractionFilter.campaign"), false),
TAXON(FilterTypeId.TAXON_NAME, n("reefdb.core.enums.extractionFilter.taxon"), false),
TAXON_GROUP(FilterTypeId.TAXON_GROUP, n("reefdb.core.enums.extractionFilter.taxonGroup"), false),
DEPARTMENT(FilterTypeId.DEPARTMENT, n("reefdb.core.enums.extractionFilter.department"), false),
......
reefdb-core/src/main/java/fr/ifremer/reefdb/service/extraction/ExtractionPerformServiceImpl.java
View file @ 7c593026
......@@ -10,29 +10,33 @@ package fr.ifremer.reefdb.service.extraction;
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.google.common.base.Joiner;
import com.google.common.collect.Maps;
import fr.ifremer.quadrige3.core.ProgressionCoreModel;
import fr.ifremer.quadrige3.core.dao.technical.Assert;
import fr.ifremer.quadrige3.core.dao.technical.Dates;
import fr.ifremer.quadrige3.core.dao.technical.Times;
import fr.ifremer.quadrige3.core.dao.technical.xmlQuery.XMLQuery;
import fr.ifremer.reefdb.config.ReefDbConfiguration;
import fr.ifremer.reefdb.dao.administration.strategy.ReefDbStrategyDao;
import fr.ifremer.reefdb.dao.referential.pmfm.ReefDbPmfmDao;
import fr.ifremer.reefdb.dao.system.extraction.ReefDbExtractionResultDao;
import fr.ifremer.reefdb.dao.technical.Daos;
import fr.ifremer.reefdb.decorator.DecoratorService;
import fr.ifremer.reefdb.dto.ReefDbBeanFactory;
import fr.ifremer.reefdb.dto.ReefDbBeans;
import fr.ifremer.reefdb.dto.configuration.filter.FilterDTO;
import fr.ifremer.reefdb.dto.configuration.programStrategy.PmfmStrategyDTO;
import fr.ifremer.reefdb.dto.configuration.programStrategy.ProgramDTO;
import fr.ifremer.reefdb.dto.enums.ExtractionFilterValues;
import fr.ifremer.reefdb.dto.enums.ExtractionOutputType;
import fr.ifremer.reefdb.dto.referential.GroupingTypeDTO;
......@@ -47,20 +51,26 @@ import fr.ifremer.reefdb.service.ReefDbServiceLocator;
import fr.ifremer.reefdb.service.ReefDbTechnicalException;
import fr.ifremer.reefdb.service.administration.program.ProgramStrategyService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jdom2.Element;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static org.nuiton.i18n.I18n.t;
/**
* New extraction perform service
* Base Mantis #49970
*
* @author peck7 on 14/05/2019.
*/
@Service("reefdbExtractionPerformService")
......@@ -68,15 +78,14 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
private static final Log LOG = LogFactory.getLog(ExtractionPerformServiceImpl.class);
private static final String EXTRACTION_START_DATE = "EXTRACTION_START_DATE";
private static final String EXTRACTION_END_DATE = "EXTRACTION_END_DATE";
private static final String RAW_TABLE_NAME_PATTERN = "R%s";
private static final String COMMON_TABLE_NAME_PATTERN = "C%s";
private static final String SPLIT_TABLE_NAME_PATTERN = "S%s";
private static final String PMFM_TABLE_NAME_PATTERN = "P%s_%s";
private static final String TABLE_NAME_PREFIX = "EXT_";
private static final String BASE_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "B%s";
private static final String RAW_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "R%s";
private static final String PMFM_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "P%s_%s";
private static final String COMMON_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "C%s";
private static final String RESULT_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "E%s";
private static final String LS = System.lineSeparator();
private static final String XML_QUERY_PATH = "xmlQuery/extraction";
@Resource
protected ReefDbConfiguration config;
......@@ -96,6 +105,9 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
@Resource(name = "reefDbPmfmDao")
protected ReefDbPmfmDao pmfmDao;
@Resource(name = "reefDbStrategyDao")
protected ReefDbStrategyDao strategyDao;
@Override
public void performExtraction(ExtractionDTO extraction, ExtractionOutputType outputType, File outputFile, ProgressionCoreModel progressionModel) {
Assert.notNull(extraction);
......@@ -108,8 +120,11 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
// ensure all filters are loaded
extractionService.loadFilteredElements(extraction);
// init progression model
progressionModel.setMessage("");
progressionModel.setTotal(9);
progressionModel.setTotal(8);
long startTime = System.currentTimeMillis();
if (LOG.isInfoEnabled()) {
LOG.info(String.format("Beginning a %s extraction (id=%s) with:", outputType, extraction.getId()));
......@@ -126,1161 +141,562 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
try {
try {
// count root data to extract
Long count = countRootData(extraction);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s surveys have to be extract", count));
}
if (count == 0) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noData.error"));
}
// create the concat functions used to concat distinct strings (ex: DEP_NM)
createConcatDistinctFunction("MEAS_CM", "VARCHAR(2000)"/*, " - "*/);
createConcatDistinctFunction("DEP_NM", "VARCHAR(255)"/*, ","*/);
createConcatDistinctFunction("MEAS_ID", "INTEGER"/*, "|"*/);
// Extraction context
ExtractionContextDTO context = ReefDbBeanFactory.newExtractionContextDTO();
context.setExtraction(extraction);
context.setUniqueId(System.currentTimeMillis());
context.setBaseTableName(String.format(BASE_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setRawTableName(String.format(RAW_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setCommonTableName(String.format(COMMON_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setSplitTableName(String.format(SPLIT_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setResultTableName(String.format(RESULT_TABLE_NAME_PATTERN, context.getUniqueId()));
// create the first temp table
createRawTable(context, outputType);
// insert raw data in temp table
int nbRowsInserted = insertRawData(context, outputType);
// create the concat functions used to concat distinct strings (ex: DEP_NM)
createConcatDistinctFunction("STRING", "VARCHAR(2000)");
createConcatDistinctFunction("INTEGER", "INTEGER");
// create the distinct functions used for quality informations (Mantis #51297)
createDistinctFunction("MEAS_QUAL_FLAG_NM", "VARCHAR(2000)");
createDistinctFunction("MEAS_QUALIF_CM", "VARCHAR(2000)");
// STEP 1 : Create the base table
long nbRowsInserted = createBaseTable(context, outputType);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of raw data inserted into %s", nbRowsInserted, context.getRawTableName()));
LOG.debug(String.format("%s surveys/sampling operations have to be extract (temp table : %s)", nbRowsInserted, context.getBaseTableName()));
} else {
LOG.info(String.format("%s surveys/sampling operations have to be extract", nbRowsInserted));
}
if (nbRowsInserted == 0) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noData.error"));
}
// build pmfm metadata
List<ExtractionPmfmInfoDTO> pmfmInfos = getPmfmInfo(context);
// STEP 2 : Create the raw table with full measurements data in line
nbRowsInserted = createRawTable(context, outputType);
progressionModel.increments(1);
// Mantis #29139: remove all individual measurement on survey
pmfmInfos = ReefDbBeans.filterCollection(pmfmInfos, input -> !(input.isSurvey() && input.isIndividual()));
if (CollectionUtils.isEmpty(pmfmInfos)) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noPmfm.error"));
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of raw data (temp table : %s)", nbRowsInserted, context.getRawTableName()));
} else {
LOG.info(String.format("%s rows of raw data", nbRowsInserted));
}
context.setPmfmInfos(pmfmInfos);
// if a pmfm, taxon or taxon group filter is defined, will remove raw line not corresponding to filter (avoid blank cells in split table)
int nbRowsRemoved = cleanRawData(context, outputType);
// STEP 3 : Build PMFM metadata
buildPmfmInformation(context);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
if (nbRowsRemoved > 0) {
LOG.debug(String.format("%s rows removed from raw data which not corresponding to PMFMU filter", nbRowsRemoved));
}
LOG.debug(String.format("list of pmfmu ids to split : %s", ReefDbBeans.collectProperties(context.getPmfmInfos(), ExtractionPmfmInfoDTO.PROPERTY_PMFM_ID)));
}
// Second clean operation: remove surveys from hermetic programs if user is not allowed (Mantis #47660)
int nbRowsRemoved2 = cleanHermeticData(context);
// STEP 4 : Clean raw data
int nbRowsRemoved = cleanRawData(context, outputType);
progressionModel.increments(1);
if (LOG.isDebugEnabled() && nbRowsRemoved2 > 0) {
LOG.debug(String.format("%s rows removed from raw data which not corresponding to user rights (hermetic programs)", nbRowsRemoved2));
}
// if all the rows have been removed because of filters, there is no more data to extract
if (nbRowsRemoved + nbRowsRemoved2 == nbRowsInserted) {
// if all the rows have been removed because of cleaning, there is no more data to extract
if (nbRowsRemoved == nbRowsInserted) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noPmfm.error"));
}
// create pmfm tables
createPmfmTables(context, outputType);
// insert raw data in pmfm tables
insertPmfmData(context, outputType);
// STEP 5 : Create pmfm tables
createPmfmTables(context);
progressionModel.increments(1);
// create common table
createCommonTable(context, outputType);
Map<String, String> fieldNamesByAlias = Maps.newHashMap();
Map<String, String> decimalFormats = Maps.newHashMap();
Map<String, String> dateFormats = Maps.newHashMap();
// insert common part of raw table into common table
nbRowsInserted = insertCommonData(context, outputType);
// STEP 6 : Create common table (todo ? with non individual measurements)
createCommonTable(context, outputType);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of common raw data inserted into %s", nbRowsInserted, context.getCommonTableName()));
}
// create split data table
createSplitTable(context, outputType);
// insert data from raw table to result table = split pmfms in columns
nbRowsInserted = insertSplitData(context, outputType);
// STEP 7 : Create result table
nbRowsInserted = createResultTable(context, outputType, fieldNamesByAlias, decimalFormats, dateFormats);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of split data inserted into %s", nbRowsInserted, context.getSplitTableName()));
LOG.debug(String.format("%s rows to write (result table : %s)", nbRowsInserted, context.getResultTableName()));
} else {
LOG.info(String.format("%s rows to write", nbRowsInserted));
}
// final query and write to csv file
writeExtraction(context, outputType, outputFile);
// STEP 8 : Final query and write to csv file
writeExtraction(context, outputType, fieldNamesByAlias, decimalFormats, dateFormats, outputFile);
progressionModel.increments(1);
if (LOG.isInfoEnabled()) {
LOG.info(String.format("Extraction %s performed. result file is : %s", outputType, outputFile.getAbsolutePath()));
long time = System.currentTimeMillis() - startTime;
LOG.info(String.format("Extraction %s performed in %s. result file is : %s", outputType, Times.durationToString(time), outputFile.getAbsolutePath()));
}
} catch (ReefDbBusinessException e) {
throw e; // throw directly
} catch (Exception e) {
throw new ReefDbTechnicalException(t("reefdb.service.extraction.error"), e);
}
} finally {
// drop concat functions
dropConcatDistinctFunction("MEAS_CM");
dropConcatDistinctFunction("DEP_NM");
dropConcatDistinctFunction("MEAS_ID");
dropConcatDistinctFunction("STRING");
dropConcatDistinctFunction("INTEGER");
}
}
private void createConcatDistinctFunction(String functionName, String valueType/*, String separator*/) {
private long createBaseTable(ExtractionContextDTO context, ExtractionOutputType outputType) {
XMLQuery xmlQuery = createXMLQuery("createBaseTable");
xmlQuery.bind("baseTableName", context.getBaseTableName());
xmlQuery.bind("orderItemTypeCode", getOrderItemTypeCode(context.getExtraction()));
dropConcatDistinctFunction(functionName);
// active groups depending the output type
setGroups(xmlQuery, outputType);
// new function with external java method
String query = "CREATE FUNCTION CONCAT_DISTINCT_" + functionName + "(IN_ARRAY " + valueType + " ARRAY, SEPARATOR VARCHAR(10)) RETURNS LONGVARCHAR" + LS +
"LANGUAGE JAVA DETERMINISTIC NO SQL EXTERNAL NAME 'CLASSPATH:fr.ifremer.reefdb.dto.ReefDbBeans.getUnifiedSQLString';";
// old function with internal hsqldb method
// String query = "CREATE FUNCTION CONCAT_DISTINCT_" + functionName + "(IN_ARRAY " + valueType + " ARRAY) RETURNS LONGVARCHAR" + LS +
// "BEGIN ATOMIC" + LS +
// "DECLARE ARRAY_IT INTEGER DEFAULT 1;" + LS +
// "DECLARE VAR_ARRAY " + valueType + " ARRAY;" + LS +
// "SET VAR_ARRAY = IN_ARRAY;" + LS +
// "WHILE ARRAY_IT < CARDINALITY(VAR_ARRAY) DO" + LS +
// "IF POSITION_ARRAY(VAR_ARRAY[ARRAY_IT] IN VAR_ARRAY FROM ARRAY_IT + 1) != 0 THEN" + LS +
// "SET VAR_ARRAY[ARRAY_IT] = NULL;" + LS +
// "END IF;" + LS +
// "SET ARRAY_IT = ARRAY_IT + 1;" + LS +
// "END WHILE;" + LS +
// "RETURN (SELECT GROUP_CONCAT(RESULT.VALUE SEPARATOR '" + separator + "') FROM UNNEST(VAR_ARRAY) AS RESULT(VALUE));" + LS +
// "END;";
//
extractionResultDao.queryUpdate(query, null);
// add mandatory period filter
List<ExtractionPeriodDTO> periodFilters = ReefDbBeans.getExtractionPeriods(context.getExtraction());
Element periodFilter = xmlQuery.getFirstTag(XMLQuery.TAG_WHERE, XMLQuery.ATTR_GROUP, "periodFilter");
Assert.notNull(periodFilter);
for (int i = 0; i < periodFilters.size(); i++) {
XMLQuery periodFilterQuery = createXMLQuery("injectionPeriodFilter");
if (i > 0) {
periodFilterQuery.getDocumentQuery().getRootElement().setAttribute(XMLQuery.ATTR_OPERATOR, "OR");
}
String periodAlias = "PERIOD" + i;
periodFilterQuery.replaceAllBindings("PERIOD", periodAlias);
periodFilter.addContent(periodFilterQuery.getDocument().getRootElement().detach());
xmlQuery.bind(periodAlias + "_startDate", Dates.formatDate(periodFilters.get(i).getStartDate(), "dd/MM/yyyy"));
xmlQuery.bind(periodAlias + "_endDate", Dates.formatDate(periodFilters.get(i).getEndDate(), "dd/MM/yyyy"));
}
// add program filter
List<String> programCodes = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.PROGRAM);
xmlQuery.setGroup("programFilter", CollectionUtils.isNotEmpty(programCodes));
xmlQuery.bind("progCodes", Daos.getInStatementFromStringCollection(programCodes));
// add monitoring location filter
List<Integer> locationIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.LOCATION);
xmlQuery.setGroup("locationFilter", CollectionUtils.isNotEmpty(locationIds));
xmlQuery.bind("monLocIds", Daos.getInStatementFromIntegerCollection(locationIds));
// add campaign filter
List<Integer> campaignIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.CAMPAIGN);
xmlQuery.setGroup("campaignFilter", CollectionUtils.isNotEmpty(campaignIds));
xmlQuery.bind("campaignIds", Daos.getInStatementFromIntegerCollection(campaignIds));
// add department filter
List<Integer> departmentIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.DEPARTMENT);
xmlQuery.setGroup("departmentFilter", CollectionUtils.isNotEmpty(departmentIds));
xmlQuery.bind("depIds", Daos.getInStatementFromIntegerCollection(departmentIds));
// add pmfm filter
List<Integer> pmfmIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.PMFM);
xmlQuery.setGroup("pmfmFilter", CollectionUtils.isNotEmpty(pmfmIds));
xmlQuery.bind("pmfmIds", Daos.getInStatementFromIntegerCollection(pmfmIds));
}
// execute insertion
execute(xmlQuery);
private String getConcatDistinctFunction(String functionName) {
// if no program filter, gather them from base table
if (CollectionUtils.isEmpty(programCodes)) {
gatherProgramCodes(context);
}
return "CONCAT_DISTINCT_" + functionName;
return countFrom(context.getBaseTableName());
}
private void dropConcatDistinctFunction(String functionName) {
extractionResultDao.queryUpdate("DROP FUNCTION CONCAT_DISTINCT_" + functionName + " IF EXISTS", null);
private void gatherProgramCodes(ExtractionContextDTO context) {
// gather program codes from base table
XMLQuery xmlQuery = createXMLQuery("programCodes");
xmlQuery.bind("baseTableName", context.getBaseTableName());
List<String> programCodes = extractionResultDao.queryStringList(xmlQuery.getSQLQueryAsString(), null);
// set into program filter
ReefDbBeans.setFilterElements(context.getExtraction(), ExtractionFilterValues.PROGRAM,
programCodes.stream().map(programCode -> {
ProgramDTO program = ReefDbBeanFactory.newProgramDTO();
program.setCode(programCode);
return program;
}).collect(Collectors.toList()));
}
private Long countRootData(ExtractionDTO extraction) {
private long createRawTable(ExtractionContextDTO context, ExtractionOutputType outputType) {
String countQuery = "SELECT COUNT(*)" + LS +
getFromClauseMinimal() +
getWhereClause(extraction);
XMLQuery xmlQuery = createXMLQuery("createRawTable");
xmlQuery.bind("rawTableName", context.getRawTableName());
xmlQuery.bind("baseTableName", context.getBaseTableName());
Map<String, Object> queryBindings = getQueryBindings(extraction);
// active groups depending the output type
setGroups(xmlQuery, outputType);
return extractionResultDao.queryCount(countQuery, queryBindings);
}
// add pampa configuration
xmlQuery.bind("alternativeTaxonOriginPampa", config.getAlternativeTaxonOriginPAMPA());
private List<ExtractionPmfmInfoDTO> getPmfmInfo(ExtractionContextDTO context) {
// retrieve pmfm ids from filter
List<Integer> pmfmIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.PMFM);
// query the raw table to retrieve pmfm metadata
String pmfmQuery = "SELECT DISTINCT PMFM_ID, IS_SURVEY_MEAS, CASE WHEN MEAS_INDIV_ID IS NULL THEN FALSE ELSE TRUE END FROM " + context.getRawTableName();
if (CollectionUtils.isNotEmpty(pmfmIds)) {
pmfmQuery += " WHERE PMFM_ID IN (" + Daos.getInStatementFromIntegerCollection(pmfmIds) + ")";
}
return extractionResultDao.query(pmfmQuery, null, (resultSet, i) -> {
ExtractionPmfmInfoDTO pmfmInfo = ReefDbBeanFactory.newExtractionPmfmInfoDTO();
pmfmInfo.setPmfmId(resultSet.getInt(1));
pmfmInfo.setSurvey(resultSet.getBoolean(2));
pmfmInfo.setIndividual(resultSet.getBoolean(3));
// compute alias
String safePmfmId = pmfmInfo.getPmfmId() < 0 ? "M" : "" + pmfmInfo.getPmfmId();
pmfmInfo.setAlias((pmfmInfo.isSurvey() ? "SU" : "SO") + (pmfmInfo.isIndividual() ? "I" : "") + safePmfmId);
return pmfmInfo;
});
}
private void createRawTable(ExtractionContextDTO context, ExtractionOutputType outputType) {
StringBuilder queryBuilder = new StringBuilder();
queryBuilder.append("DECLARE LOCAL TEMPORARY TABLE ").append(context.getRawTableName()).append(" (");
queryBuilder.append("SURVEY_ID INTEGER, ");
queryBuilder.append("SURVEY_PROG_CD VARCHAR(40), ");
queryBuilder.append("REC_DEP_ID INTEGER, ");
queryBuilder.append("MON_LOC_NM VARCHAR(100), ");
queryBuilder.append("SURVEY_DT TIMESTAMP, ");
queryBuilder.append("SAMPLING_OPER_ID INTEGER, ");
queryBuilder.append("SAMPLING_OPER_LB VARCHAR(50), ");
queryBuilder.append("MON_LOC_ID INTEGER, ");
queryBuilder.append("MON_LOC_LB VARCHAR(50), ");
queryBuilder.append("ORDER_ITEM_NM VARCHAR(100), ");
queryBuilder.append("SURVEY_POSITION_LONG VARCHAR(20), ");
queryBuilder.append("SURVEY_POSITION_LAT VARCHAR(20), ");
queryBuilder.append("SAMPLING_EQUIPMENT_NM VARCHAR(100), ");
queryBuilder.append("SAMPLING_OPER_DEP_NM VARCHAR(255), ");
if (outputType != ExtractionOutputType.PAMPA) {
queryBuilder.append("SURVEY_QUSERS VARCHAR(1000), ");
}
queryBuilder.append("SURVEY_LB VARCHAR(50), ");
queryBuilder.append("SAMPLING_OPER_SIZE DOUBLE, ");
queryBuilder.append("SAMPLING_OPER_SIZE_UNIT_SYMBOL VARCHAR(100), ");
if (outputType == ExtractionOutputType.COMPLETE) {
queryBuilder.append("ORDER_ITEM_TYPE_NM VARCHAR(100), ");
queryBuilder.append("ORDER_ITEM_CD VARCHAR(40), ");
queryBuilder.append("MON_LOC_ORDER_ITEM_NUMBER DOUBLE, ");
queryBuilder.append("SURVEY_TIME DOUBLE, ");
queryBuilder.append("SURVEY_CM VARCHAR(2000), ");
queryBuilder.append("SURVEY_CONTROL_DT TIMESTAMP, ");
queryBuilder.append("SURVEY_VALID_DT TIMESTAMP, ");
queryBuilder.append("SURVEY_QUAL_FLAG VARCHAR(100), ");
queryBuilder.append("SURVEY_QUALIF_DT TIMESTAMP, ");
queryBuilder.append("SURVEY_QUALIF_CM VARCHAR(2000), ");
queryBuilder.append("SURVEY_INHERITED_POSITION VARCHAR(1), ");
queryBuilder.append("SAMPLING_OPER_TIME INTEGER, ");
queryBuilder.append("SAMPLING_OPER_DEPTH DOUBLE, ");
queryBuilder.append("SAMPLING_OPER_MIN_DEPTH DOUBLE, ");
queryBuilder.append("SAMPLING_OPER_MAX_DEPTH DOUBLE, ");
queryBuilder.append("SAMPLING_OPER_INHERITED_POSITION VARCHAR(1), ");
queryBuilder.append("SAMPLING_OPER_DEPTH_UNIT_SYMBOL VARCHAR(100), ");
queryBuilder.append("SAMPLING_OPER_DEPTH_UNIT VARCHAR(100), ");
queryBuilder.append("SAMPLING_OPER_CM VARCHAR(2000), ");
queryBuilder.append("SAMPLING_EQUIPMENT_SIZE DOUBLE, ");
queryBuilder.append("SAMPLING_EQUIPMENT_SIZE_UNIT_SYMBOL VARCHAR(100), ");
queryBuilder.append("SAMPLING_OPER_NUMBER_INDIV DOUBLE, ");
queryBuilder.append("SAMPLING_OPER_CONTROL_DT TIMESTAMP, ");
queryBuilder.append("SAMPLING_OPER_VALID_DT TIMESTAMP, ");
queryBuilder.append("SAMPLING_OPER_QUAL_FLAG VARCHAR(100), ");
queryBuilder.append("SAMPLING_OPER_QUALIF_DT TIMESTAMP, ");
queryBuilder.append("SAMPLING_OPER_QUALIF_CM VARCHAR(2000), ");
queryBuilder.append("SAMPLING_OPER_POSITION_LONG VARCHAR(20), ");
queryBuilder.append("SAMPLING_OPER_POSITION_LAT VARCHAR(20), ");
} else if (outputType == ExtractionOutputType.PAMPA) {
queryBuilder.append("SAMPLING_OPER_TIME INTEGER, ");
queryBuilder.append("SURVEY_BOTTOM_DEPTH DOUBLE, ");
queryBuilder.append("SURVEY_QUSERS VARCHAR(1000), ");
}
queryBuilder.append("MEAS_ID INTEGER, ");
queryBuilder.append("TAXON_MEAS_ID INTEGER, ");
queryBuilder.append("IS_SURVEY_MEAS BOOLEAN, ");
queryBuilder.append("MEAS_INDIV_ID INTEGER,");
queryBuilder.append("PMFM_ID INTEGER, ");
queryBuilder.append("DEP_NM VARCHAR(255), ");
queryBuilder.append("MEAS_NUMER_VALUE DOUBLE, ");
queryBuilder.append("QUAL_VALUE_NM VARCHAR(100), ");
queryBuilder.append("TAXON_NAME_ID INTEGER, ");
queryBuilder.append("TAXON_NAME_NM VARCHAR(255), ");
queryBuilder.append("TAXON_GROUP_ID INTEGER, ");
queryBuilder.append("TAXON_GROUP_NM VARCHAR(100), ");
queryBuilder.append("MEAS_CM VARCHAR(2000), ");
if (outputType == ExtractionOutputType.PAMPA) {
queryBuilder.append("TAXON_PAMPA VARCHAR(40), ");
} else {
queryBuilder.append("INPUT_TAXON_NAME_NM VARCHAR(255), ");
queryBuilder.append("INPUT_TAXON_NAME_ID INTEGER, ");
}
String query = trimAtLastSequence(queryBuilder.toString(), ",").concat(")");
extractionResultDao.queryUpdate(query, null);
execute(xmlQuery);
return countFrom(context.getRawTableName());
}
private void createPmfmTables(ExtractionContextDTO context, ExtractionOutputType outputType) {
if (!context.isPmfmInfosEmpty()) {
for (ExtractionPmfmInfoDTO pmfmInfo : context.getPmfmInfos()) {
// Generate pmfm table name
String pmfmTableName = String.format(PMFM_TABLE_NAME_PATTERN, context.getUniqueId(), pmfmInfo.getAlias());
pmfmInfo.setTableName(pmfmTableName);
StringBuilder queryBuilder = new StringBuilder();
queryBuilder.append("DECLARE LOCAL TEMPORARY TABLE ").append(pmfmTableName).append(" (");
queryBuilder.append(pmfmInfo.isSurvey() ? "SURVEY_ID INTEGER, " : "SAMPLING_OPER_ID INTEGER, ");
queryBuilder.append("MEAS_ID INTEGER, ");
queryBuilder.append("TAXON_MEAS_ID INTEGER, ");
queryBuilder.append("IS_SURVEY_MEAS BOOLEAN, ");
queryBuilder.append("MEAS_INDIV_ID INTEGER, ");
queryBuilder.append("PMFM_ID INTEGER, ");
queryBuilder.append("DEP_NM VARCHAR(255), ");
queryBuilder.append("MEAS_NUMER_VALUE DOUBLE, ");
queryBuilder.append("QUAL_VALUE_NM VARCHAR(100), ");
queryBuilder.append("TAXON_NAME_NM VARCHAR(255), ");
queryBuilder.append("TAXON_GROUP_NM VARCHAR(100), ");
queryBuilder.append("MEAS_CM VARCHAR(2000), ");
if (outputType == ExtractionOutputType.PAMPA) {
queryBuilder.append("TAXON_PAMPA VARCHAR(40), ");
} else {
queryBuilder.append("INPUT_TAXON_NAME_NM VARCHAR(255), ");
queryBuilder.append("INPUT_TAXON_NAME_ID INTEGER, ");
}
String query = trimAtLastSequence(queryBuilder.toString(), ",").concat(")");
extractionResultDao.queryUpdate(query, null);
}
}
}
private void buildPmfmInformation(ExtractionContextDTO context) {
private void createCommonTable(ExtractionContextDTO context, ExtractionOutputType outputType) {
// Gather pmfm from raw table
List<ExtractionPmfmInfoDTO> pmfmInfos = getPmfmInfo(context);
StringBuilder queryBuilder = new StringBuilder();
queryBuilder.append("DECLARE LOCAL TEMPORARY TABLE ").append(context.getCommonTableName()).append(" (");
queryBuilder.append("SURVEY_ID INTEGER, ");
queryBuilder.append("SURVEY_PROG_CD VARCHAR(40), ");
queryBuilder.append("MON_LOC_NM VARCHAR(100), ");
queryBuilder.append("SURVEY_DT TIMESTAMP, ");
queryBuilder.append("SAMPLING_OPER_ID INTEGER, ");
queryBuilder.append("SAMPLING_OPER_LB VARCHAR(50), ");
queryBuilder.append("MON_LOC_ID INTEGER, ");
queryBuilder.append("MON_LOC_LB VARCHAR(50), ");
queryBuilder.append("ORDER_ITEM_NM VARCHAR(100), ");
queryBuilder.append("SURVEY_POSITION_LONG VARCHAR(20), ");
queryBuilder.append("SURVEY_POSITION_LAT VARCHAR(20), ");
queryBuilder.append("SAMPLING_EQUIPMENT_NM VARCHAR(100), ");
queryBuilder.append("SAMPLING_OPER_DEP_NM VARCHAR(255), ");
if (outputType != ExtractionOutputType.PAMPA) {
queryBuilder.append("SURVEY_QUSERS VARCHAR(1000), ");
}
queryBuilder.append("SURVEY_LB VARCHAR(50), ");
queryBuilder.append("SAMPLING_OPER_SIZE DOUBLE, ");
queryBuilder.append("SAMPLING_OPER_SIZE_UNIT_SYMBOL VARCHAR(100), ");