Commit 3089771d authored by LAVENIER's avatar LAVENIER
Browse files

[fix] Root data: Allow filter on data quality status (modified, controlled, validated, qualified)

[fix] Trip: Allow filter on recorderPersonIds
parent fbf87dc1
......@@ -28,10 +28,9 @@ import net.sumaris.core.exception.DataNotFoundException;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.dao.technical.Daos;
import net.sumaris.core.extraction.dao.technical.ExtractionBaseDaoImpl;
import net.sumaris.core.extraction.dao.technical.XMLQuery;
import net.sumaris.core.extraction.dao.technical.xml.XMLQuery;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.specification.administration.StratSpecification;
import net.sumaris.core.extraction.vo.ExtractionFilterCriterionVO;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.extraction.vo.administration.ExtractionStrategyContextVO;
import net.sumaris.core.extraction.vo.administration.ExtractionStrategyFilterVO;
......
......@@ -34,6 +34,7 @@ import net.sumaris.core.dao.technical.schema.SumarisTableMetadata;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.dao.technical.schema.SumarisTableMetadatas;
import net.sumaris.core.extraction.dao.technical.xml.XMLQuery;
import net.sumaris.core.extraction.vo.ExtractionContextVO;
import net.sumaris.core.extraction.vo.ExtractionFilterVO;
import net.sumaris.core.model.referential.IItemReferentialEntity;
......@@ -59,7 +60,6 @@ import java.util.stream.Stream;
public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
protected static final String XML_QUERY_PATH = "xmlQuery";
protected static final String XSL_ORACLE_FILENAME = "xmlQuery/queryOracle.xsl";
@Autowired
protected ExtractionConfiguration configuration;
......@@ -176,14 +176,7 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
* @return
*/
protected XMLQuery createXMLQuery() {
XMLQuery xmlQuery = applicationContext.getBean("xmlQuery", XMLQuery.class);
if (this.databaseType == DatabaseType.oracle) {
xmlQuery.setXSLFileName(XSL_ORACLE_FILENAME);
}
else if (this.databaseType == DatabaseType.postgresql){
xmlQuery.setLowercase(true);
}
return xmlQuery;
return new XMLQuery(databaseType);
}
protected void dropTables(@NonNull ExtractionContextVO context) {
......@@ -257,4 +250,22 @@ public abstract class ExtractionBaseDaoImpl extends HibernateDaoSupport {
return query;
}
/**
* Enable/Disable group, depending on the DBMS
* @param xmlQuery
*/
protected void setDbms(XMLQuery xmlQuery) {
if (this.databaseType != null) {
switch (this.databaseType) {
case hsqldb:
case oracle:
xmlQuery.setDbms(this.databaseType.name());
break;
case postgresql:
xmlQuery.setDbms("pgsql");
break;
}
}
}
}
/*
* Created on 29 juil. 2004
*/
package net.sumaris.core.extraction.dao.technical;
/*-
* #%L
* Quadrige3 Core :: Shared
* %%
* Copyright (C) 2017 - 2018 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.base.Preconditions;
import fr.ifremer.common.xmlquery.HSQLDBSingleXMLQuery;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.dao.technical.xml.XPaths;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.jdom2.Attribute;
import org.jdom2.Element;
import org.jdom2.filter.Filters;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author ludovic.pecquot@e-is.pro
*/
@Component("xmlQuery")
@Scope("prototype")
public class XMLQuery extends HSQLDBSingleXMLQuery {
private String xslFileName;
private boolean lowercase;
public XMLQuery() {
super();
xslFileName = super.getXSLFileName();
this.lowercase = false;
}
public XMLQuery(boolean lowercase) {
super();
xslFileName = super.getXSLFileName();
this.lowercase = lowercase;
}
@Override
protected String getXSLFileName() {
return xslFileName;
}
protected void setXSLFileName(String xslFileName) {
this.xslFileName = xslFileName;
}
// let default values here for HSQLDB
public boolean isLowercase() {
return lowercase;
}
public void setLowercase(boolean lowercase) {
this.lowercase = lowercase;
}
/**
* Get column names, with type="hidden"
*
* @return
*/
public Set<String> getHiddenColumnNames() {
try {
List<Element> selectElements = XPaths.compile("//query/select[contains(@type, 'hidden')]", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(attribute -> attribute.getValue())
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
public Set<String> getVisibleColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"hidden".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getNotNumericColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"number".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getColumnNames(final Predicate<Element> filter) {
Preconditions.checkNotNull(filter);
try {
List<Element> selectElements = XPaths.compile("//query/select", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
// Apply filter
.filter(filter::evaluate)
// Get alias
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(attribute -> attribute.getValue())
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
/**
* Return if option="DISTINCT" has been set on the query
*
* @return
*/
public boolean hasDistinctOption() {
Attribute optionAtr = getFirstQueryTag().getAttribute("option");
return optionAtr != null && "distinct".equalsIgnoreCase(optionAtr.getValue());
}
public String getSQLQueryAsString(){
String query = super.getSQLQueryAsString();
if (this.lowercase){
query = query.toLowerCase();
}
return query;
}
}
/*
* #%L
* SUMARiS
* %%
* Copyright (C) 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
/*
* Created on 29 juil. 2004
*/
package net.sumaris.core.extraction.dao.technical.xml;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import fr.ifremer.common.xmlquery.*;
import lombok.NonNull;
import net.sumaris.core.dao.technical.DatabaseType;
import net.sumaris.core.exception.SumarisTechnicalException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.jdom2.Attribute;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.filter.Filters;
import org.jdom2.output.Format;
import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author ludovic.pecquot@e-is.pro
*/
public class XMLQuery {
private DatabaseType dbms;
private AsbtractSingleXMLQuery delegate;
public XMLQuery(@NonNull DatabaseType dbms) {
super();
this.dbms = dbms;
switch (dbms) {
case hsqldb:
delegate = new HSQLDBSingleXMLQuery();
delegate.bind("true", "1");
delegate.bind("false", "0");
break;
case postgresql:
delegate = new PgsqlSingleXMLQuery();
delegate.bind("true", "True");
delegate.bind("false", "False");
break;
case oracle:
delegate = new OracleSingleXMLQuery();
delegate.bind("true", "1");
delegate.bind("false", "0");
break;
default:
throw new IllegalArgumentException("Not XMLQuery instance found for database type: " + dbms.name());
}
delegate.setDbms(dbms.name());
}
/**
* Get column names, with type="hidden"
*
* @return
*/
public Set<String> getHiddenColumnNames() {
try {
List<Element> selectElements = XPaths.compile("//query/select[contains(@type, 'hidden')]", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(Attribute::getValue)
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
public Set<String> getVisibleColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"hidden".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getNotNumericColumnNames() {
return getColumnNames(element -> {
Attribute typeAttr = element.getAttribute("type");
return typeAttr == null || !"number".equalsIgnoreCase(typeAttr.getValue());
});
}
public Set<String> getColumnNames(final Predicate<Element> filter) {
Preconditions.checkNotNull(filter);
try {
List<Element> selectElements = XPaths.compile("//query/select", Filters.element())
.evaluate(getDocument());
if (CollectionUtils.isEmpty(selectElements)) return null;
return selectElements.stream()
// Apply filter
.filter(filter::evaluate)
// Get alias
.map(element -> element.getAttribute("alias"))
.filter(Objects::nonNull)
.map(Attribute::getValue)
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toCollection(LinkedHashSet::new));
} catch (Exception e) {
throw new SumarisTechnicalException(e);
}
}
/**
* Return if option="DISTINCT" has been set on the query
*
* @return
*/
public boolean hasDistinctOption() {
Attribute optionAtr = getFirstQueryTag().getAttribute("option");
return optionAtr != null && "distinct".equalsIgnoreCase(optionAtr.getValue());
}
public Document getDocument() {
return delegate.getDocument();
}
public Element getFirstQueryTag() {
return delegate.getFirstQueryTag();
}
/* -- delegated functions -- */
public boolean isLowercase() {
return delegate.isLowercase();
}
public void setLowercase(boolean lowercase) {
delegate.setLowercase(lowercase);
}
public void manageRootElement() throws XMLQueryException {
delegate.manageRootElement();
}
public String getSQLQueryAsString() {
String query = delegate.getSQLQueryAsString();
// Oracle replacements
if (dbms == DatabaseType.oracle) {
return query
// Replace true/false => 1/0
.replaceAll("(?i)=[/s]*true", "=1")
.replaceAll("(?i)=[/s]*false", "=0")
// null columns must have a type (avoid error ORA-01723)
.replaceAll("(?i)null (\"[^\"]+\"[,\n\r]+)", "CAST('' AS VARCHAR2(1)) $1")
;
}
return query;
}
public Map<String, String> getSqlParameters() {
return delegate.getSqlParameters();
}
public void setSqlParameters(Map<String, String> sqlParameters) {
delegate.setSqlParameters(sqlParameters);
}
public String getSort() {
return delegate.getSort();
}
public void setSort(String sort) {
delegate.setSort(sort);
}
public String getSortDirection() {
return delegate.getSortDirection();
}
public void setSortDirection(String sortDirection) {
delegate.setSortDirection(sortDirection);
}
public Document getDocumentQuery() {
return delegate.getDocumentQuery();
}
public void setQuery(String pXml) throws XMLQueryException {
delegate.setQuery(pXml);
}
public void setQuery(URL pFileURL) throws XMLQueryException {
delegate.setQuery(pFileURL);
}
public void setQuery(String pXml, boolean pManageRootElement) throws XMLQueryException {
delegate.setQuery(pXml, pManageRootElement);
}
public void setQuery(File pQueryFile) throws XMLQueryException {
delegate.setQuery(pQueryFile);
}
public void setQuery(File pQueryFile, boolean pManageRootElement) throws XMLQueryException {
delegate.setQuery(pQueryFile, pManageRootElement);
}
public void setQuery(URL fileURL, boolean pManageRootElement) throws XMLQueryException {
delegate.setQuery(fileURL, pManageRootElement);
}
public void addSelect(File pXmlFile) throws XMLQueryException {
delegate.addSelect(pXmlFile);
}
public void addSelect(URL pXmlFileURL) throws XMLQueryException {
delegate.addSelect(pXmlFileURL);
}
public void addSelect(String pXmlFilter) throws XMLQueryException {
delegate.addSelect(pXmlFilter);
}
public void addSubSelect(File pXmlFile) throws XMLQueryException {
delegate.addSubSelect(pXmlFile);
}
public void addSubSelect(URL pXmlFileURL) throws XMLQueryException {
delegate.addSubSelect(pXmlFileURL);
}
public void addSubSelect(String pXmlFilter) throws XMLQueryException {
delegate.addSubSelect(pXmlFilter);
}
public void addSelect(String pQueryName, File pXmlFile) throws XMLQueryException {
delegate.addSelect(pQueryName, pXmlFile);
}
public void addSelect(String pQueryName, URL pXmlFileURL) throws XMLQueryException {
delegate.addSelect(pQueryName, pXmlFileURL);
}
public void addSelect(String pQueryName, String pXmlFilter) throws XMLQueryException {
delegate.addSelect(pQueryName, pXmlFilter);
}
public void addSubSelect(String pQueryName, File pXmlFile) throws XMLQueryException {
delegate.addSubSelect(pQueryName, pXmlFile);
}
public void addSubSelect(String pQueryName, URL pXmlFileURL) throws XMLQueryException {
delegate.addSubSelect(pQueryName, pXmlFileURL);
}
public void addSubSelect(String pQueryName, String pXmlFilter) throws XMLQueryException {
delegate.addSubSelect(pQueryName, pXmlFilter);
}
public void addFrom(File pXmlFile) throws XMLQueryException {
delegate.addFrom(pXmlFile);
}
public void addFrom(URL pXmlFileURL) throws XMLQueryException {
delegate.addFrom(pXmlFileURL);
}
public void addFrom(File pXmlFile, String pTag, String pAttributeName, String pAttributeValue) throws XMLQueryException {
delegate.addFrom(pXmlFile, pTag, pAttributeName, pAttributeValue);
}
public void addFrom(URL pXmlFileURL, String pTag, String pAttributeName, String pAttributeValue) throws XMLQueryException {
delegate.addFrom(pXmlFileURL, pTag, pAttributeName, pAttributeValue);
}
public void addFrom(String pXmlFilter) throws XMLQueryException {
delegate.addFrom(pXmlFilter);
}
public void addWhere(File pFilterFile) throws XMLQueryException {
delegate.addWhere(pFilterFile);
}
public void addWhere(URL pFilterFileURL) throws XMLQueryException {
delegate.addWhere(pFilterFileURL);
}
public void addWhere(String pXmlFilter) throws XMLQueryException {
delegate.addWhere(pXmlFilter);
}
public void addWhere(String pQueryName, File pFilterFile) throws XMLQueryException {
delegate.addWhere(pQueryName, pFilterFile);
}
public void addWhere(String pQueryName, URL pFilterFileURL) throws XMLQueryException {
delegate.addWhere(pQueryName, pFilterFileURL);
}
public void addWhere(String pQueryName, String pXmlFilter) throws XMLQueryException {