Commit 4c9a1e33 authored by PECQUOT's avatar PECQUOT
Browse files

[enh] New extraction perform service using xmlQuery (version 1.1+) (Mantis #49970)


[enh] Pmfmu labels are transcribed with specific label for extraction, if exists, or with "screen" label (Mantis #50725)
Signed-off-by: PECQUOT's avatarlp1ee9d <ludovic.pecquot@e-is.pro>
parent e7166511
## Sprint 83 - v3.9.2
- Pas de mise à jour de modèle
- Une nouvelle option a été ajoutée pour gérer les transcodages des référentiels suivant :
reefdb.transcribingItemType.label.pmfmExtraction
Sa valeur par défaut est `REEFDB-PMFM.PMFM_EXTRACTION`. Couplée avec l'option `reefdb.transcribingItemType.label.pmfmNm`, elle permet de transcoder les libellé des PSFMU lors des extractions avec la priorité suivante:
1- libellé transcodé pour extraction
2- libellé transcodé pour écran de saisie
3- libellé interne du psfmu (issu du transcodage pour écran de saisie)
4- libellé du paramètre
- Pour simplifier la configuration du serveur de synchronisation, l'option `quadrige3.synchro.import.referential.transcribingItemType.label.includes`
qui doit contenir tous les libellés des types de transcodage à importer, accepte désormais le caractère '%'.
quadrige3.synchro.import.referential.transcribingItemType.label.includes=REEFDB% (tous les types de transcodage commençant par REEFDB seront importés)
## Sprint 82 - v3.9.1
- Pas de mise à jour de modèle
## Sprint 81 - v3.9.0
- Pas de mise à jour de modèle
......
......@@ -10,7 +10,7 @@
<groupId>fr.ifremer.reefdb</groupId>
<artifactId>reefdb</artifactId>
<version>3.9.1</version>
<version>3.9.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Reef DB</name>
......@@ -171,8 +171,8 @@
<maven.compiler.debug>true</maven.compiler.debug>
<!-- Quadrige3 Core version -->
<quadrige3-core.version>3.6.5</quadrige3-core.version>
<!-- <quadrige3-core.version>3.6.6-SNAPSHOT</quadrige3-core.version>-->
<!-- <quadrige3-core.version>3.6.5</quadrige3-core.version>-->
<quadrige3-core.version>3.6.6-SNAPSHOT</quadrige3-core.version>
<!-- Last ReefDb launcher version -->
<launcherVersion>3.0.3</launcherVersion>
......
......@@ -4,7 +4,7 @@
<parent>
<groupId>fr.ifremer.reefdb</groupId>
<artifactId>reefdb</artifactId>
<version>3.9.1</version>
<version>3.9.2-SNAPSHOT</version>
</parent>
<artifactId>reefdb-core</artifactId>
......
......@@ -967,6 +967,10 @@ public final class ReefDbConfiguration extends QuadrigeCoreConfiguration {
return applicationConfig.getOption(ReefDbConfigurationOption.TRANSCRIBING_ITEM_TYPE_LB_PMFM_NM.getKey());
}
public String getTranscribingItemTypeLbForPmfmExtraction() {
return applicationConfig.getOption(ReefDbConfigurationOption.TRANSCRIBING_ITEM_TYPE_LB_PMFM_EXTRACTION.getKey());
}
/**
* <p>getAlternativeTaxonOriginTaxRef.</p>
*
......
......@@ -24,7 +24,6 @@ package fr.ifremer.reefdb.config;
*/
import fr.ifremer.quadrige3.core.dao.referential.UnitId;
import fr.ifremer.quadrige3.core.dao.referential.transcribing.TranscribingItemTypeLb;
import fr.ifremer.quadrige3.core.dao.technical.QuadrigeEnumerationDef;
import org.nuiton.config.ConfigOptionDef;
import org.nuiton.version.Version;
......@@ -303,7 +302,13 @@ public enum ReefDbConfigurationOption implements ConfigOptionDef {
TRANSCRIBING_ITEM_TYPE_LB_PMFM_NM(
"reefdb.transcribingItemType.label.pmfmNm",
n("reefdb.config.option.transcribingItemType.label.pmfmNm.description"),
String.format("${%s}", TranscribingItemTypeLb.REEFDB_PMFM_NM.getKey()), // 1
String.format("${%s%s}", QuadrigeEnumerationDef.CONFIG_OPTION_PREFIX, "TranscribingItemTypeLb.REEFDB_PMFM_NM"),
String.class,
false),
TRANSCRIBING_ITEM_TYPE_LB_PMFM_EXTRACTION(
"reefdb.transcribingItemType.label.pmfmExtraction",
n("reefdb.config.option.transcribingItemType.label.pmfmExtraction.description"),
String.format("${%s%s}", QuadrigeEnumerationDef.CONFIG_OPTION_PREFIX, "TranscribingItemTypeLb.REEFDB_PMFM_EXTRACTION"),
String.class,
false),
/* constants used to calculate transition length */
......
......@@ -24,6 +24,7 @@ package fr.ifremer.reefdb.dao.system.extraction;
*/
import fr.ifremer.quadrige3.core.dao.technical.csv.CSVDao;
import fr.ifremer.reefdb.dto.referential.pmfm.PmfmDTO;
import org.springframework.jdbc.core.RowMapper;
import java.util.List;
......@@ -91,4 +92,10 @@ public interface ReefDbExtractionResultDao extends CSVDao {
* @return a int.
*/
int queryUpdate(String query, Map<String, Object> queryBindings);
String getPmfmNameForExtraction(PmfmDTO pmfm);
String getPmfmUnitNameForExtraction(PmfmDTO pmfm);
}
......@@ -23,7 +23,11 @@ package fr.ifremer.reefdb.dao.system.extraction;
* #L%
*/
import fr.ifremer.quadrige3.core.dao.referential.transcribing.TranscribingItemExtendDao;
import fr.ifremer.quadrige3.core.dao.technical.Assert;
import fr.ifremer.quadrige3.core.dao.technical.csv.CSVDaoImpl;
import fr.ifremer.reefdb.config.ReefDbConfiguration;
import fr.ifremer.reefdb.dto.referential.pmfm.PmfmDTO;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -32,9 +36,11 @@ import org.springframework.dao.DataRetrievalFailureException;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Repository;
import javax.annotation.Resource;
import javax.sql.DataSource;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
/**
......@@ -50,6 +56,12 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
protected final Properties connectionProperties;
@Resource
protected ReefDbConfiguration config;
@Resource(name = "transcribingItemDao")
private TranscribingItemExtendDao transcribingItemDao;
/**
* <p>Constructor for ReefDbExtractionResultDaoImpl.</p>
*
......@@ -114,6 +126,25 @@ public class ReefDbExtractionResultDaoImpl extends CSVDaoImpl implements ReefDbE
return queryUpdate(connectionProperties, query, paramMap);
}
@Override
public String getPmfmNameForExtraction(PmfmDTO pmfm) {
Assert.notNull(pmfm);
return Optional
.ofNullable(transcribingItemDao.getTranscribingItemById(config.getTranscribingItemTypeLbForPmfmExtraction(), pmfm.getId()))
.orElse(
Optional.ofNullable(transcribingItemDao.getTranscribingItemById(config.getTranscribingItemTypeLbForPmfmNm(), pmfm.getId()))
.orElse(
Optional.ofNullable(pmfm.getName())
.orElse(pmfm.getParameter().getName())
)
); }
@Override
public String getPmfmUnitNameForExtraction(PmfmDTO pmfm) {
Assert.notNull(pmfm);
return pmfm.getUnit() == null || config.getExtractionUnitIdsToIgnore().contains(pmfm.getUnit().getId()) ? "" : pmfm.getUnit().getName();
}
/**
* {@inheritDoc}
*/
......
......@@ -287,6 +287,7 @@ public class DecoratorServiceImpl extends fr.ifremer.quadrige3.core.service.deco
}
}
@Deprecated
private class PMFMExtractionDecorator extends PMFMNameDecorator {
static final String SEPARATOR = "-";
......
......@@ -10,29 +10,33 @@ package fr.ifremer.reefdb.service.extraction;
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.google.common.base.Joiner;
import com.google.common.collect.Maps;
import fr.ifremer.quadrige3.core.ProgressionCoreModel;
import fr.ifremer.quadrige3.core.dao.technical.Assert;
import fr.ifremer.quadrige3.core.dao.technical.Dates;
import fr.ifremer.quadrige3.core.dao.technical.Times;
import fr.ifremer.quadrige3.core.dao.technical.xmlQuery.XMLQuery;
import fr.ifremer.reefdb.config.ReefDbConfiguration;
import fr.ifremer.reefdb.dao.administration.strategy.ReefDbStrategyDao;
import fr.ifremer.reefdb.dao.referential.pmfm.ReefDbPmfmDao;
import fr.ifremer.reefdb.dao.system.extraction.ReefDbExtractionResultDao;
import fr.ifremer.reefdb.dao.technical.Daos;
import fr.ifremer.reefdb.decorator.DecoratorService;
import fr.ifremer.reefdb.dto.ReefDbBeanFactory;
import fr.ifremer.reefdb.dto.ReefDbBeans;
import fr.ifremer.reefdb.dto.configuration.filter.FilterDTO;
import fr.ifremer.reefdb.dto.configuration.programStrategy.PmfmStrategyDTO;
import fr.ifremer.reefdb.dto.configuration.programStrategy.ProgramDTO;
import fr.ifremer.reefdb.dto.enums.ExtractionFilterValues;
import fr.ifremer.reefdb.dto.enums.ExtractionOutputType;
import fr.ifremer.reefdb.dto.referential.GroupingTypeDTO;
......@@ -47,20 +51,26 @@ import fr.ifremer.reefdb.service.ReefDbServiceLocator;
import fr.ifremer.reefdb.service.ReefDbTechnicalException;
import fr.ifremer.reefdb.service.administration.program.ProgramStrategyService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jdom2.Element;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static org.nuiton.i18n.I18n.t;
/**
* New extraction perform service
* Base Mantis #49970
*
* @author peck7 on 14/05/2019.
*/
@Service("reefdbExtractionPerformService")
......@@ -68,13 +78,14 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
private static final Log LOG = LogFactory.getLog(ExtractionPerformServiceImpl.class);
private static final String EXTRACTION_START_DATE = "EXTRACTION_START_DATE";
private static final String EXTRACTION_END_DATE = "EXTRACTION_END_DATE";
private static final String TABLE_NAME_PREFIX = "EXT_";
private static final String BASE_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "B%s";
private static final String RAW_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "R%s";
private static final String PMFM_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "P%s_%s";
private static final String COMMON_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "C%s";
private static final String RESULT_TABLE_NAME_PATTERN = TABLE_NAME_PREFIX + "E%s";
private static final String RAW_TABLE_NAME_PATTERN = "R%s";
private static final String COMMON_TABLE_NAME_PATTERN = "C%s";
private static final String SPLIT_TABLE_NAME_PATTERN = "S%s";
private static final String PMFM_TABLE_NAME_PATTERN = "P%s_%s";
private static final String XML_QUERY_PATH = "xmlQuery/extraction";
private static final String LS = System.lineSeparator();
......@@ -96,6 +107,9 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
@Resource(name = "reefDbPmfmDao")
protected ReefDbPmfmDao pmfmDao;
@Resource(name = "reefDbStrategyDao")
protected ReefDbStrategyDao strategyDao;
@Override
public void performExtraction(ExtractionDTO extraction, ExtractionOutputType outputType, File outputFile, ProgressionCoreModel progressionModel) {
Assert.notNull(extraction);
......@@ -108,8 +122,11 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
// ensure all filters are loaded
extractionService.loadFilteredElements(extraction);
// init progression model
progressionModel.setMessage("");
progressionModel.setTotal(9);
progressionModel.setTotal(8);
long startTime = System.currentTimeMillis();
if (LOG.isInfoEnabled()) {
LOG.info(String.format("Beginning a %s extraction (id=%s) with:", outputType, extraction.getId()));
......@@ -126,1161 +143,539 @@ public class ExtractionPerformServiceImpl implements ExtractionPerformService {
try {
try {
// count root data to extract
Long count = countRootData(extraction);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s surveys have to be extract", count));
}
if (count == 0) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noData.error"));
}
// create the concat functions used to concat distinct strings (ex: DEP_NM)
createConcatDistinctFunction("MEAS_CM", "VARCHAR(2000)"/*, " - "*/);
createConcatDistinctFunction("DEP_NM", "VARCHAR(255)"/*, ","*/);
createConcatDistinctFunction("MEAS_ID", "INTEGER"/*, "|"*/);
// Extraction context
ExtractionContextDTO context = ReefDbBeanFactory.newExtractionContextDTO();
context.setExtraction(extraction);
context.setUniqueId(System.currentTimeMillis());
context.setBaseTableName(String.format(BASE_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setRawTableName(String.format(RAW_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setCommonTableName(String.format(COMMON_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setResultTableName(String.format(SPLIT_TABLE_NAME_PATTERN, context.getUniqueId()));
context.setResultTableName(String.format(RESULT_TABLE_NAME_PATTERN, context.getUniqueId()));
// create the first temp table
createRawTable(context, outputType);
// create the concat functions used to concat distinct strings (ex: DEP_NM)
createConcatDistinctFunction("STRING", "VARCHAR(2000)");
createConcatDistinctFunction("INTEGER", "INTEGER");
// insert raw data in temp table
int nbRowsInserted = insertRawData(context, outputType);
// STEP 1 : Create the base table
long nbRowsInserted = createBaseTable(context, outputType);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of raw data inserted into %s", nbRowsInserted, context.getRawTableName()));
LOG.debug(String.format("%s surveys/sampling operations have to be extract (temp table : %s)", nbRowsInserted, context.getBaseTableName()));
} else {
LOG.info(String.format("%s surveys/sampling operations have to be extract", nbRowsInserted));
}
if (nbRowsInserted == 0) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noData.error"));
}
// build pmfm metadata
List<ExtractionPmfmInfoDTO> pmfmInfos = getPmfmInfo(context);
// STEP 2 : Create the raw table with full measurements data in line
nbRowsInserted = createRawTable(context, outputType);
progressionModel.increments(1);
// Mantis #29139: remove all individual measurement on survey
pmfmInfos = ReefDbBeans.filterCollection(pmfmInfos, input -> !(input.isSurvey() && input.isIndividual()));
if (CollectionUtils.isEmpty(pmfmInfos)) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noPmfm.error"));
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of raw data (temp table : %s)", nbRowsInserted, context.getRawTableName()));
} else {
LOG.info(String.format("%s rows of raw data", nbRowsInserted));
}
context.setPmfmInfos(pmfmInfos);
// if a pmfm, taxon or taxon group filter is defined, will remove raw line not corresponding to filter (avoid blank cells in split table)
int nbRowsRemoved = cleanRawData(context, outputType);
// STEP 3 : Build PMFM metadata
buildPmfmInformation(context);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
if (nbRowsRemoved > 0) {
LOG.debug(String.format("%s rows removed from raw data which not corresponding to PMFMU filter", nbRowsRemoved));
}
LOG.debug(String.format("list of pmfmu ids to split : %s", ReefDbBeans.collectProperties(context.getPmfmInfos(), ExtractionPmfmInfoDTO.PROPERTY_PMFM_ID)));
}
// Second clean operation: remove surveys from hermetic programs if user is not allowed (Mantis #47660)
int nbRowsRemoved2 = cleanHermeticData(context);
// STEP 4 : Clean raw data
int nbRowsRemoved = cleanRawData(context, outputType);
progressionModel.increments(1);
if (LOG.isDebugEnabled() && nbRowsRemoved2 > 0) {
LOG.debug(String.format("%s rows removed from raw data which not corresponding to user rights (hermetic programs)", nbRowsRemoved2));
}
// if all the rows have been removed because of filters, there is no more data to extract
if (nbRowsRemoved + nbRowsRemoved2 == nbRowsInserted) {
// if all the rows have been removed because of cleaning, there is no more data to extract
if (nbRowsRemoved == nbRowsInserted) {
throw new ReefDbBusinessException(t("reefdb.service.extraction.noPmfm.error"));
}
// create pmfm tables
createPmfmTables(context, outputType);
// insert raw data in pmfm tables
insertPmfmData(context, outputType);
// STEP 5 : Create pmfm tables
createPmfmTables(context);
progressionModel.increments(1);
// create common table
createCommonTable(context, outputType);
Map<String, String> fieldNamesByAlias = Maps.newHashMap();
Map<String, String> decimalFormats = Maps.newHashMap();
Map<String, String> dateFormats = Maps.newHashMap();
// insert common part of raw table into common table
nbRowsInserted = insertCommonData(context, outputType);
// STEP 6 : Create common table (todo ? with non individual measurements)
createCommonTable(context, outputType);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of common raw data inserted into %s", nbRowsInserted, context.getCommonTableName()));
}
// create split data table
createSplitTable(context, outputType);
// insert data from raw table to result table = split pmfms in columns
nbRowsInserted = insertSplitData(context, outputType);
// STEP 7 : Create result table
nbRowsInserted = createResultTable(context, outputType, fieldNamesByAlias, decimalFormats, dateFormats);
progressionModel.increments(1);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s rows of split data inserted into %s", nbRowsInserted, context.getResultTableName()));
LOG.debug(String.format("%s rows to write (result table : %s)", nbRowsInserted, context.getResultTableName()));
} else {
LOG.info(String.format("%s rows to write", nbRowsInserted));
}
// final query and write to csv file
writeExtraction(context, outputType, outputFile);
// STEP 8 : Final query and write to csv file
writeExtraction(context, outputType, fieldNamesByAlias, decimalFormats, dateFormats, outputFile);
progressionModel.increments(1);
if (LOG.isInfoEnabled()) {
LOG.info(String.format("Extraction %s performed. result file is : %s", outputType, outputFile.getAbsolutePath()));
long time = System.currentTimeMillis() - startTime;
LOG.info(String.format("Extraction %s performed in %s. result file is : %s", outputType, Times.durationToString(time), outputFile.getAbsolutePath()));
}
} catch (ReefDbBusinessException e) {
throw e; // throw directly
} catch (Exception e) {
throw new ReefDbTechnicalException(t("reefdb.service.extraction.error"), e);
}
} finally {
// drop concat functions
dropConcatDistinctFunction("MEAS_CM");
dropConcatDistinctFunction("DEP_NM");
dropConcatDistinctFunction("MEAS_ID");
dropConcatDistinctFunction("STRING");
dropConcatDistinctFunction("INTEGER");
}
}
private void createConcatDistinctFunction(String functionName, String valueType/*, String separator*/) {
dropConcatDistinctFunction(functionName);
// new function with external java method
String query = "CREATE FUNCTION CONCAT_DISTINCT_" + functionName + "(IN_ARRAY " + valueType + " ARRAY, SEPARATOR VARCHAR(10)) RETURNS LONGVARCHAR" + LS +
"LANGUAGE JAVA DETERMINISTIC NO SQL EXTERNAL NAME 'CLASSPATH:fr.ifremer.reefdb.dto.ReefDbBeans.getUnifiedSQLString';";
// old function with internal hsqldb method
// String query = "CREATE FUNCTION CONCAT_DISTINCT_" + functionName + "(IN_ARRAY " + valueType + " ARRAY) RETURNS LONGVARCHAR" + LS +
// "BEGIN ATOMIC" + LS +
// "DECLARE ARRAY_IT INTEGER DEFAULT 1;" + LS +
// "DECLARE VAR_ARRAY " + valueType + " ARRAY;" + LS +
// "SET VAR_ARRAY = IN_ARRAY;" + LS +
// "WHILE ARRAY_IT < CARDINALITY(VAR_ARRAY) DO" + LS +
// "IF POSITION_ARRAY(VAR_ARRAY[ARRAY_IT] IN VAR_ARRAY FROM ARRAY_IT + 1) != 0 THEN" + LS +
// "SET VAR_ARRAY[ARRAY_IT] = NULL;" + LS +
// "END IF;" + LS +
// "SET ARRAY_IT = ARRAY_IT + 1;" + LS +
// "END WHILE;" + LS +
// "RETURN (SELECT GROUP_CONCAT(RESULT.VALUE SEPARATOR '" + separator + "') FROM UNNEST(VAR_ARRAY) AS RESULT(VALUE));" + LS +
// "END;";
//
extractionResultDao.queryUpdate(query, null);
private long createBaseTable(ExtractionContextDTO context, ExtractionOutputType outputType) {
XMLQuery xmlQuery = createXMLQuery("createBaseTable");
xmlQuery.bind("baseTableName", context.getBaseTableName());
xmlQuery.bind("orderItemTypeCode", getOrderItemTypeCode(context.getExtraction()));
}
// active groups depending the output type
setGroups(xmlQuery, outputType);
private String getConcatDistinctFunction(String functionName) {
// add mandatory period filter
List<ExtractionPeriodDTO> periodFilters = ReefDbBeans.getExtractionPeriods(context.getExtraction());
Element periodFilter = xmlQuery.getFirstTag(XMLQuery.TAG_WHERE, XMLQuery.ATTR_GROUP, "periodFilter");
Assert.notNull(periodFilter);
for (int i = 0; i < periodFilters.size(); i++) {
XMLQuery periodFilterQuery = createXMLQuery("injectionPeriodFilter");
if (i > 0) {
periodFilterQuery.getDocumentQuery().getRootElement().setAttribute(XMLQuery.ATTR_OPERATOR, "OR");
}
String periodAlias = "PERIOD" + i;
periodFilterQuery.replaceAllBindings("PERIOD", periodAlias);
periodFilter.addContent(periodFilterQuery.getDocument().getRootElement().detach());
xmlQuery.bind(periodAlias + "_startDate", Dates.formatDate(periodFilters.get(i).getStartDate(), "dd/MM/yyyy"));
xmlQuery.bind(periodAlias + "_endDate", Dates.formatDate(periodFilters.get(i).getEndDate(), "dd/MM/yyyy"));
}
// add program filter
List<String> programCodes = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.PROGRAM);
xmlQuery.setGroup("programFilter", CollectionUtils.isNotEmpty(programCodes));
xmlQuery.bind("progCodes", Daos.getInStatementFromStringCollection(programCodes));
// add monitoring location filter
List<Integer> locationIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.LOCATION);
xmlQuery.setGroup("locationFilter", CollectionUtils.isNotEmpty(locationIds));
xmlQuery.bind("monLocIds", Daos.getInStatementFromIntegerCollection(locationIds));
// add campaign filter
List<Integer> campaignIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.CAMPAIGN);
xmlQuery.setGroup("campaignFilter", CollectionUtils.isNotEmpty(campaignIds));
xmlQuery.bind("campaignIds", Daos.getInStatementFromIntegerCollection(campaignIds));
// add department filter
List<Integer> departmentIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.DEPARTMENT);
xmlQuery.setGroup("departmentFilter", CollectionUtils.isNotEmpty(departmentIds));
xmlQuery.bind("depIds", Daos.getInStatementFromIntegerCollection(departmentIds));
// add pmfm filter
List<Integer> pmfmIds = ReefDbBeans.getFilterElementsIds(context.getExtraction(), ExtractionFilterValues.PMFM);
xmlQuery.setGroup("pmfmFilter", CollectionUtils.isNotEmpty(pmfmIds));
xmlQuery.bind("pmfmIds", Daos.getInStatementFromIntegerCollection(pmfmIds));
return "CONCAT_DISTINCT_" + functionName;
}
// execute insertion
execute(xmlQuery);
private void dropConcatDistinctFunction(String functionName) {
// if no program filter, gather them from base table
if (CollectionUtils.isEmpty(programCodes)) {
gatherProgramCodes(context);
}
extractionResultDao.queryUpdate("DROP FUNCTION CONCAT_DISTINCT_" + functionName + " IF EXISTS", null);
return countFrom(context.getBaseTableName());
}
private Long countRootData(ExtractionDTO extraction) {
String countQuery = "SELECT COUNT(*)" + LS +
getFromClauseMinimal() +
getWhereClause(extraction);
Map<String, Object> queryBindings = getQueryBindings(extraction);
return extractionResultDao.queryCount(countQuery, queryBindings);
private void gatherProgramCodes(ExtractionContextDTO context) {
// gather program codes from base table
XMLQuery xmlQuery = createXMLQuery("programCodes");
xmlQuery.bind("baseTableName", context.getBaseTableName());
List<String> programCodes = extractionResultDao.queryStringList(xmlQuery.getSQLQueryAsString(), null);
// set into program filter
ReefDbBeans.setFilt