Commit 597d88ec authored by COTONNEC's avatar COTONNEC
Browse files

[enh] add postgresql unit tests for extractions

parent 08073bff
package net.sumaris.core.extraction;
/*-
* #%L
* SUMARiS :: Sumaris Client Core
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2018 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import net.sumaris.core.extraction.dao.technical.DaosPgsqlTest;
import net.sumaris.core.extraction.dao.technical.DaosTest;
import net.sumaris.core.extraction.service.*;
import org.junit.ClassRule;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.springframework.test.context.ActiveProfiles;
/**
* Created by Ludovic on 02/02/2016.
*/
@RunWith(Suite.class)
@ActiveProfiles("pgsql")
@Suite.SuiteClasses({
// DAO
DaosPgsqlTest.class,
// Service
ExtractionServicePgsqlTest.class,
ExtractionProductServicePgsqlTest.class,
AggregationServicePgsqlTest.class
})
public class AllPgsqlTests {
@ClassRule
public static InitTests initTests = new InitTests("pgsql");
}
......@@ -49,6 +49,12 @@ public class InitTests extends net.sumaris.core.test.InitTests {
log.error(ex.getLocalizedMessage(), ex);
}
}
public InitTests() {
}
public InitTests(String datasourcePlatform) {
this.datasourcePlatform = datasourcePlatform;
}
@Override
public String getTargetDbDirectory() {
......@@ -60,6 +66,11 @@ public class InitTests extends net.sumaris.core.test.InitTests {
return TestConfiguration.MODULE_NAME;
}
@Override
protected String getConfigFileName(){
return TestConfiguration.CONFIG_FILE_PREFIX + "-" + this.datasourcePlatform + ".properties";
}
@Override
protected void beforeInsert(Connection conn) throws SQLException {
super.beforeInsert(conn);
......
......@@ -29,7 +29,8 @@ public class TestConfiguration extends net.sumaris.core.test.TestConfiguration {
public static final String MODULE_NAME = "sumaris-core-extraction";
public static final String DATASOURCE_PLATFORM = "hsqldb";
public static final String CONFIG_FILE_PREFIX = MODULE_NAME + "-test";
// public static final String CONFIG_FILE_PREFIX = MODULE_NAME + "-test";
public static final String CONFIG_FILE_PREFIX = "application";
public static final String CONFIG_FILE_NAME = CONFIG_FILE_PREFIX + ".properties";
public static final String I18N_BUNDLE_NAME = MODULE_NAME + "-i18n";
......
package net.sumaris.core.extraction.dao.technical;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.collect.ImmutableMap;
import net.sumaris.core.config.SumarisConfiguration;
import net.sumaris.core.dao.technical.DatabaseType;
import net.sumaris.core.extraction.DatabaseResource;
import net.sumaris.core.util.Dates;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.ClassRule;
import org.junit.Test;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Date;
/**
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>*
*/
public class DaosPgsqlTest {
@ClassRule
public static final DatabaseResource dbResource = DatabaseResource.writeDb("pgsql");
@Test
public void getSqlToDate() {
Date date = Dates.getFirstDayOfYear(2019);
String sql = Daos.getSqlToDate(date);
Assert.assertNotNull(sql);
Assert.assertEquals("TO_DATE('2019-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS')", sql);
}
@Test
public void getSelectHashCodeString() throws Exception {
SumarisConfiguration config = SumarisConfiguration.getInstance();
DatabaseType dbType = Daos.getDatabaseType(config.getJdbcURL());
Assume.assumeNotNull(dbType);
// Get the string
String hashCodeString = Daos.getSelectHashCodeString(dbType, "%s");
Assert.assertNotNull(hashCodeString);
try (Connection conn = Daos.createConnection(config.getConnectionProperties())) {
// Test not empty value
{
final String[] expressions = new String[] {
"aaaaaa",
"test ex",
"abcdef",
"abcdefg",
"abcdefgh",
"abcdefghi",
"abcdefghijkl",
"abcdefghijklmnopqrs",
"abcdefghijklmnopqrstuvxyz",
"abcdefghijklmnopqrstuvxyz".toUpperCase(),
"0123456789",
"#{[|^@]}ễ²¡÷׿“~´\"éç'@^\\´& ",
};
for (String expression: expressions) {
Integer result = getDaoHashCode(conn, expression);
Assert.assertNotNull(result);
// Hsqldb function is equivalent to Java String.hashCode()
if (dbType == DatabaseType.hsqldb) {
Assert.assertEquals(String.format("Invalid hashCode for expresion '%s'. Expected: %s, Actual: %s", expression,
expression.hashCode(), result),
expression.hashCode(), result.intValue());
}
}
}
// Test empty value
{
Integer result = getDaoHashCode(conn, "");
Assert.assertNotNull(result);
Assert.assertEquals(0, result.intValue());
}
// Test null value
Assert.assertNull(getDaoHashCode(conn,null));
}
}
@Test
public void sqlReplaceColumnNames(){
String sql = "SELECT\n" +
" TO_DATE(S.DATE || ' ' || S.TIME, 'YYYY-MM-DD HH24:MI') AS FISHING_TIME\n" +
" SUM(S.FISHING_TIME) AS TRIP_COUNT_BY_FISHING_TIME\n" +
" S.FISHING_TIME_NOT_REPLACED\n" +
"FROM TRIP T\n" +
"WHERE\n" +
" 0 < S.FISHING_TIME";
String expectedSql = "SELECT\n" +
" TO_DATE(S.FISHING_DATE || ' ' || S.TIME, 'YYYY-MM-DD HH24:MI') AS FISHING_DURATION\n" +
" SUM(S.FISHING_DURATION) AS TRIP_COUNT_BY_FISHING_TIME\n" +
" S.FISHING_TIME_NOT_REPLACED\n" +
"FROM TRIP T\n" +
"WHERE\n" +
" 0 < S.FISHING_DURATION";
String actualSql = Daos.sqlReplaceColumnNames(sql, ImmutableMap.of("date", "fishing_date",
"fishing_time", "fishing_duration"));
Assert.assertEquals(expectedSql, actualSql);
}
/* -- protected method -- */
protected Integer getDaoHashCode(Connection conn, String expression) throws SQLException {
DatabaseType dbType = Daos.getDatabaseType(conn.getMetaData().getURL());
Assume.assumeNotNull(dbType);
// Get the string
String hashCodeString = Daos.getSelectHashCodeString(dbType, "%s");
Assert.assertNotNull(hashCodeString);
String quotedExpression = expression == null ? "null" : "'" + expression.replaceAll("'", "''") + "'";
String sql = "SELECT "
+ String.format(hashCodeString, quotedExpression)
+ " HASH FROM status WHERE id=0";
try (PreparedStatement ps = Daos.prepareQuery(conn, sql)) {
try (ResultSet rs = ps.executeQuery()) {
Assert.assertTrue(rs.next());
Integer result = rs.getObject("HASH", Integer.class);
return result;
}
}
}
}
......@@ -51,7 +51,7 @@ import java.util.List;
*/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = {ServiceTestConfiguration.class})
@TestPropertySource(locations="classpath:sumaris-core-extraction-test.properties")
//@TestPropertySource(locations={"classpath:application-test.properties", "classpath:application-pgsql.properties"})
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
public abstract class AbstractServiceTest {
......
package net.sumaris.core.extraction.service;
/*-
* #%L
* SUMARiS:: Core Extraction
* %%
* Copyright (C) 2018 - 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.google.common.collect.ImmutableList;
import net.sumaris.core.extraction.DatabaseResource;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.extraction.specification.data.trip.AggRdbSpecification;
import net.sumaris.core.extraction.specification.data.trip.AggRjbTripSpecification;
import net.sumaris.core.extraction.specification.data.trip.AggSurvivalTestSpecification;
import net.sumaris.core.extraction.specification.data.trip.RdbSpecification;
import net.sumaris.core.extraction.util.ExtractionProducts;
import net.sumaris.core.extraction.vo.*;
import net.sumaris.core.model.referential.StatusEnum;
import net.sumaris.core.model.technical.extraction.ExtractionCategoryEnum;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
import net.sumaris.core.model.technical.extraction.rdb.ProductRdbStation;
import net.sumaris.core.vo.administration.user.DepartmentVO;
import net.sumaris.core.vo.technical.extraction.AggregationStrataVO;
import org.junit.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ActiveProfiles;
import java.io.File;
import java.io.IOException;
/**
* @author peck7 on 17/12/2018.
*/
@ActiveProfiles("pgsql")
public class AggregationServicePgsqlTest extends AbstractServiceTest {
@ClassRule
public static final DatabaseResource dbResource = DatabaseResource.writeDb("pgsql");
@Autowired
private AggregationService aggregationService;
private String yearRawData;
private String yearRdbProduct;
@Before
public void setUp() throws Exception {
yearRawData = String.valueOf(fixtures.getYearRawData());
yearRdbProduct = String.valueOf(fixtures.getYearRdbProduct());
}
@Test
public void aggregateLiveRdb() throws IOException {
AggregationTypeVO type = new AggregationTypeVO();
type.setCategory(ExtractionCategoryEnum.LIVE);
type.setLabel(LiveFormatEnum.RDB.name());
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSpatialColumnName(ProductRdbStation.COLUMN_STATISTICAL_RECTANGLE);
strata.setTimeColumnName(ProductRdbStation.COLUMN_YEAR);
File outputFile = aggregationService.executeAndDump(type, null, strata);
File root = unpack(outputFile, type);
// HH.csv
File stationFile = new File(root, AggRdbSpecification.HH_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(stationFile) > 1);
// SL.csv
File speciesListFile = new File(root, AggRdbSpecification.SL_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(speciesListFile) > 1);
// HL.csv
// File speciesLengthFile = new File(root, AggRdbSpecification.HL_SHEET_NAME + ".csv");
// Assert.assertTrue(countLineInCsvFile(speciesLengthFile) > 1);
}
@Test
public void aggregateProductRdb() throws IOException {
AggregationTypeVO type = new AggregationTypeVO();
type.setCategory(ExtractionCategoryEnum.PRODUCT);
type.setLabel(fixtures.getRdbProductLabel(0));
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSpatialColumnName(ProductRdbStation.COLUMN_STATISTICAL_RECTANGLE);
strata.setTimeColumnName(ProductRdbStation.COLUMN_YEAR);
File outputFile = aggregationService.executeAndDump(type, null, strata);
File root = unpack(outputFile, type);
// HH.csv
File stationFile = new File(root, AggRdbSpecification.HH_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(stationFile) > 1);
// SL.csv
File speciesListFile = new File(root, AggRdbSpecification.SL_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(speciesListFile) > 1);
// HL.csv
// Fixme BLA: Cannot link HL rowsto SL rows, so the generated HL is empty
// - tests data mistake: P01_RDB_SPECIES_LENGTH should have same columns as SL rows, to be able to link to SL
//File speciesLengthFile = new File(root, AggRdbSpecification.HL_SHEET_NAME + ".csv");
//Assert.assertTrue(countLineInCsvFile(speciesLengthFile) > 1);
}
@Test
public void aggregateSurvivalTest() throws IOException {
AggregationTypeVO type = new AggregationTypeVO();
type.setCategory(ExtractionCategoryEnum.LIVE);
type.setLabel(LiveFormatEnum.SURVIVAL_TEST.name());
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSpatialColumnName(ProductRdbStation.COLUMN_STATISTICAL_RECTANGLE);
strata.setTimeColumnName(ProductRdbStation.COLUMN_YEAR);
File outputFile = aggregationService.executeAndDump(type, null, strata);
Assert.assertTrue(outputFile.exists());
File root = unpack(outputFile, type);
// ST.csv
File survivalTestFile = new File(root, AggSurvivalTestSpecification.ST_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(survivalTestFile) > 1);
// RL.csv
File releaseFile = new File(root, AggSurvivalTestSpecification.RL_SHEET_NAME + ".csv");
Assert.assertTrue(releaseFile.exists()); // No release DATA in the test DB
}
@Test
public void aggregateFree1() throws IOException {
AggregationTypeVO type = new AggregationTypeVO();
type.setCategory(ExtractionCategoryEnum.LIVE);
type.setLabel(LiveFormatEnum.FREE1.getLabel());
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSpatialColumnName(ProductRdbStation.COLUMN_STATISTICAL_RECTANGLE);
strata.setTimeColumnName(AggRdbSpecification.COLUMN_QUARTER);
ExtractionFilterVO filter = ExtractionFilterVO.builder()
.sheetName(AggRjbTripSpecification.HH_SHEET_NAME)
.criteria(ImmutableList.of(
ExtractionFilterCriterionVO.builder()
.sheetName(AggRjbTripSpecification.HH_SHEET_NAME)
.name(ProductRdbStation.COLUMN_YEAR)
.operator(ExtractionFilterOperatorEnum.EQUALS.getSymbol())
.value(yearRawData)
.build()
))
.build();
File outputFile = aggregationService.executeAndDump(type, filter, strata);
Assert.assertTrue(outputFile.exists());
File root = unpack(outputFile, type);
// HH.csv
File stationFile = new File(root, AggRjbTripSpecification.HH_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(stationFile) > 1);
// SL.csv
File speciesListFile = new File(root, AggRjbTripSpecification.SL_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(speciesListFile) > 1);
// HL.csv
// File speciesLengthFile = new File(root, AggRjbTripSpecification.HL_SHEET_NAME + ".csv");
// Assert.assertTrue(countLineInCsvFile(speciesLengthFile) > 1);
}
@Test
@Ignore
// FIXME: add BATCH on RJB species, with individual count only (no weights) in ADAP XML data
public void aggregateRjbTrip() throws IOException {
AggregationTypeVO type = new AggregationTypeVO();
type.setCategory(ExtractionCategoryEnum.LIVE);
type.setLabel(LiveFormatEnum.RJB_TRIP.name());
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSpatialColumnName(ProductRdbStation.COLUMN_STATISTICAL_RECTANGLE);
strata.setTimeColumnName(ProductRdbStation.COLUMN_YEAR);
ExtractionFilterVO filter = ExtractionFilterVO.builder()
.sheetName(AggRjbTripSpecification.HH_SHEET_NAME)
.criteria(ImmutableList.of(
ExtractionFilterCriterionVO.builder()
.name(ProductRdbStation.COLUMN_TRIP_CODE)
.operator(ExtractionFilterOperatorEnum.EQUALS.getSymbol())
.value("1379") // a ADAP RJB trip
.build()
))
.build();
File outputFile = aggregationService.executeAndDump(type, filter, strata);
Assert.assertTrue(outputFile.exists());
File root = unpack(outputFile, type);
// HH.csv
File stationFile = new File(root, AggRjbTripSpecification.HH_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(stationFile) > 1);
// SL.csv
File speciesListFile = new File(root, AggRjbTripSpecification.SL_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(speciesListFile) > 1);
// HL.csv
File speciesLengthFile = new File(root, AggRjbTripSpecification.HL_SHEET_NAME + ".csv");
Assert.assertTrue(countLineInCsvFile(speciesLengthFile) > 1);
}
@Test
public void executeAndRead() {
AggregationTypeVO type = createAggType(ExtractionCategoryEnum.LIVE, LiveFormatEnum.SURVIVAL_TEST);
ExtractionFilterVO filter = new ExtractionFilterVO();
filter.setSheetName(AggRdbSpecification.SL_SHEET_NAME);
ExtractionFilterCriterionVO criterion = new ExtractionFilterCriterionVO() ;
filter.setCriteria(ImmutableList.of(criterion));
criterion.setSheetName(AggRdbSpecification.HH_SHEET_NAME);
criterion.setName(AggRdbSpecification.COLUMN_YEAR);
criterion.setOperator("=");
criterion.setValue(yearRawData);
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSheetName(AggRdbSpecification.SL_SHEET_NAME);
strata.setSpatialColumnName(AggRdbSpecification.COLUMN_AREA);
strata.setTimeColumnName(AggRdbSpecification.COLUMN_MONTH);
AggregationResultVO result = aggregationService.executeAndRead(type, filter, strata, 0,100, null, null);
Assert.assertNotNull(result);
Assert.assertNotNull(result.getRows());
Assert.assertTrue(result.getRows().size() > 0);
Assert.assertNotNull(result.getSpaceStrata());
Assert.assertTrue(result.getSpaceStrata().size() > 0);
Assert.assertTrue(result.getSpaceStrata().contains(AggRdbSpecification.COLUMN_AREA));
Assert.assertNotNull(result.getTimeStrata());
Assert.assertTrue(result.getTimeStrata().size() > 0);
Assert.assertTrue(result.getTimeStrata().contains(AggRdbSpecification.COLUMN_MONTH));
Assert.assertTrue(result.getTimeStrata().contains(AggRdbSpecification.COLUMN_QUARTER));
Assert.assertTrue(result.getTimeStrata().contains(AggRdbSpecification.COLUMN_YEAR));
}
@Test
public void saveThenGetAggBySpace() {
AggregationTypeVO type = createAggType(ExtractionCategoryEnum.LIVE, LiveFormatEnum.SURVIVAL_TEST);
type.setLabel(ExtractionProducts.getProductLabel(LiveFormatEnum.SURVIVAL_TEST, System.currentTimeMillis()));
// Save
AggregationTypeVO savedType = aggregationService.save(type, null);
Assert.assertNotNull(savedType);
Assert.assertNotNull(savedType.getId());
ExtractionFilterVO filter = new ExtractionFilterVO();
filter.setSheetName(AggRdbSpecification.HH_SHEET_NAME);
ExtractionFilterCriterionVO criterion = new ExtractionFilterCriterionVO() ;
filter.setCriteria(ImmutableList.of(criterion));
criterion.setSheetName(AggRdbSpecification.HH_SHEET_NAME);
criterion.setName(AggRdbSpecification.COLUMN_YEAR);
criterion.setOperator("=");
criterion.setValue(yearRawData);
AggregationStrataVO strata = new AggregationStrataVO();
strata.setSpatialColumnName(AggRdbSpecification.COLUMN_AREA);
AggregationResultVO result = aggregationService.getAggBySpace(savedType, filter, strata, 0,100, null, null);
Assert.assertNotNull(result);
Assert.assertNotNull(result.getRows());
Assert.assertTrue(result.getRows().size() > 0);
}
@Test
public void readTechProductRdb() {