Commit b6e8eac4 authored by LAVENIER's avatar LAVENIER
Browse files

[enh] Denormalization: add elevated weight, and individual count

[enh] Liquibase: add UnitConversion
parent 5e75ad3d
# SUMARiS options
sumaris.name=SUMARiS
sumaris.version=@project.version@
sumaris.log.file=${user.home}/.config/${sumaris.name}/${sumaris.name}.log
# Spring: Common properties
# see https://docs.spring.io/spring-boot/docs/current/reference/html/common-application-properties.html
spring.main.banner-mode=off
# Disable web
spring.main.web-application-type=none
# DataSource configuration
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
spring.datasource.username=sa
spring.datasource.password=
spring.datasource.url=jdbc:hsqldb:hsql://localhost/sumaris
spring.datasource.platform=@env@
spring.datasource.hikari.connectionTestQuery=SELECT 1 FROM STATUS WHERE ID=1
spring.datasource.hikari.initializationFailTimeout=-1
# General JPA properties
spring.jpa.database-platform=net.sumaris.core.dao.technical.hibernate.spatial.HSQLSpatialDialect
spring.jpa.open-in-view=false
# Hibernate Specific properties
spring.jpa.hibernate.ddl-auto=none
spring.jpa.properties.hibernate.format_sql=false
spring.jpa.properties.hibernate.hbm2ddl.auto=none
#Naming strategy
spring.jpa.hibernate.naming.physical-strategy=net.sumaris.core.dao.technical.hibernate.HibernatePhysicalNamingStrategy
spring.jpa.hibernate.naming.implicit-strategy=net.sumaris.core.dao.technical.hibernate.HibernateImplicitNamingStrategy
# Liquibase
spring.liquibase.enabled=false
# Extraction options
# Embedded ActiveMQ Configuration
spring.activemq.pool.enabled=true
spring.activemq.broker-url=vm://embedded?broker.persistent=true
# Spring JMS Settings
spring.jms.jndi-name=
# Logging Levels
logging.level.ROOT=info
logging.level.net.sumaris=info
logging.level.net.sumaris.core.extraction=debug
logging.level.org.springframework=warn
logging.level.org.nuiton=warn
logging.level.org.nuiton.i18n=error
logging.level.net.sf.ehcache=warn
logging.level.net.sf.ehcache.hibernate=info
logging.level.org.apache.commons.beanutils=warn
logging.level.org.apache.jena=warn
logging.level.org.hibernate=warn
#logging.level.org.hibernate.SQL=debug
logging.level.org.hibernate.spatial=warn
logging.level.org.hibernate.engine.jdbc.spi.SqlExceptionHelper=error
logging.level.org.hibernate.engine.internal.StatefulPersistenceContext=error
logging.level.org.hibernate.engine.StatefulPersistenceContext.ProxyWarnLog=error
logging.level.org.jboss.logging=warn
logging.level.hsqldb.db=warn
logging.level.liquibase=error
logging.level.liquibase-hibernate=warn
logging.level.com.zaxxer.hikari=warn
package net.sumaris.core.extraction.action;
/*
* #%L
* SIH-Adagio :: Shared
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2012 - 2014 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.action.ActionUtils;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.exception.UnknownFormatException;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.service.AggregationService;
import net.sumaris.core.extraction.service.ExtractionProductService;
import net.sumaris.core.extraction.service.ExtractionService;
import net.sumaris.core.extraction.util.ExtractionFormats;
import net.sumaris.core.extraction.vo.AggregationTypeVO;
import net.sumaris.core.extraction.vo.ExtractionTypeVO;
import net.sumaris.core.model.referential.StatusEnum;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
import net.sumaris.core.service.ServiceLocator;
import net.sumaris.core.service.data.ProductService;
import net.sumaris.core.util.Beans;
import net.sumaris.core.util.Files;
import net.sumaris.core.util.StringUtils;
import net.sumaris.core.vo.technical.extraction.ExtractionProductFetchOptions;
import net.sumaris.core.vo.technical.extraction.ExtractionProductFilterVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
import org.apache.commons.collections4.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
/**
* <p>DatabaseChangeLogAction class.</p>
*
*/
@Slf4j
public class AgggregationAction {
/**
* <p>execute.</p>
*/
public void execute() {
ExtractionConfiguration config = ExtractionConfiguration.instance();
AggregationService aggregationService = ServiceLocator.instance().getService("aggregationService", AggregationService.class);
ExtractionProductService productService = ServiceLocator.instance().getService("extractionProductService", ExtractionProductService.class);
String formatLabel = config.getExtractionCliOutputFormat();
AggregationTypeVO type = null;
try {
IExtractionFormat format = ProductFormatEnum.valueOf(formatLabel);
type = aggregationService.getTypeByFormat(format);
} catch (UnknownFormatException | IllegalArgumentException e) {
String availableProducts = productService.findByFilter(ExtractionProductFilterVO.builder()
.statusIds(new Integer[]{StatusEnum.ENABLE.getId()})
.build(),
ExtractionProductFetchOptions.DOCUMENTATION)
.stream()
.map(p -> " - " + p.getLabel())
.collect(Collectors.joining("\n"));
log.error("Unknown aggregation product '{}'.\nExisting products:\n{}",
formatLabel,
availableProducts);
System.exit(1);
}
log.info("Starting {} aggregation {{}}...",
StringUtils.capitalize(type.getCategory().name().toLowerCase()),
type.getLabel());
ActionUtils.logConnectionProperties();
}
/**
* <p>refresh.</p>
*/
public void refresh() {
ExtractionConfiguration config = ExtractionConfiguration.instance();
AggregationService aggregationService = ServiceLocator.instance().getService("aggregationService", AggregationService.class);
ExtractionProductService productService = ServiceLocator.instance().getService("extractionProductService", ExtractionProductService.class);
List<ExtractionProductVO> products = productService.findByFilter(ExtractionProductFilterVO.builder()
.statusIds(new Integer[]{StatusEnum.ENABLE.getId(), StatusEnum.TEMPORARY.getId()})
.build(),
ExtractionProductFetchOptions.builder().build());
if (CollectionUtils.isEmpty(products)) {
log.info("No product found in database. Nothing to refresh");
return;
}
log.info("Refreshing aggregation products...");
ActionUtils.logConnectionProperties();
for (ExtractionProductVO product: products) {
log.info("Updating product {{}}...", product.getLabel());
aggregationService.refresh(product.getId());
try {
Thread.sleep(10000); // Waiting 10s, to avoid HSQLDB 'generale error'
}
catch (InterruptedException e) {
// Stop
return;
}
}
}
}
package net.sumaris.core.extraction.action;
/*
* #%L
* SIH-Adagio :: Shared
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2012 - 2014 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.action.ActionUtils;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.exception.UnknownFormatException;
import net.sumaris.core.extraction.service.ExtractionService;
import net.sumaris.core.extraction.util.ExtractionFormats;
import net.sumaris.core.extraction.vo.ExtractionTypeVO;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
import net.sumaris.core.service.ServiceLocator;
import net.sumaris.core.util.Files;
import net.sumaris.core.util.StringUtils;
import java.io.File;
import java.io.IOException;
/**
* <p>DatabaseChangeLogAction class.</p>
*
*/
@Slf4j
public class ExtractionAction {
/**
* <p>execute.</p>
*/
public void execute() {
ExtractionConfiguration config = ExtractionConfiguration.instance();
ExtractionService service = ServiceLocator.instance().getService("extractionService", ExtractionService.class);
String formatLabel = config.getExtractionCliOutputFormat();
IExtractionFormat format = null;
try {
format = ExtractionFormats.getFormatFromLabel(formatLabel);
} catch (UnknownFormatException e) {
log.error("Unknown format: " + formatLabel);
return;
}
log.info("Starting {} extraction {{}}...",
StringUtils.capitalize(format.getCategory().name().toLowerCase()),
format.getLabel());
ActionUtils.logConnectionProperties();
// Check output file
File outputFile = ActionUtils.checkAndGetOutputFile(false, ExtractionAction.class);
// Execute the extraction
long now = System.currentTimeMillis();
File tempFile;
try {
ExtractionTypeVO type = service.getByFormat(format);
tempFile = service.executeAndDump(type, null);
if (!tempFile.exists()) {
log.error("No data");
return;
}
} catch (IOException e) {
log.error("Error during extraction: " + e.getMessage(), e);
return;
}
// Move temp file to expected output file
try {
Files.moveFile(tempFile, outputFile);
log.info("{} extraction {{}} finished, in {}s - output file: {}",
StringUtils.capitalize(format.getCategory().name().toLowerCase()),
format.getLabel(),
(System.currentTimeMillis() - now) / 1000,
outputFile.getAbsolutePath());
}
catch (IOException e) {
log.error("Error while creating output file: " + e.getMessage(), e);
return;
}
}
}
/*
* #%L
* SUMARiS
* %%
* Copyright (C) 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
package net.sumaris.core.extraction.config;
import net.sumaris.core.config.SumarisConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConditionalOnProperty(
prefix = "sumaris.extraction",
name = {"enabled"},
matchIfMissing = true
)
public class ExtractionAutoConfiguration {
@Bean
public ExtractionConfiguration extractionConfiguration(SumarisConfiguration configuration) {
ExtractionConfiguration instance = new ExtractionConfiguration(configuration);
ExtractionConfiguration.setInstance(instance);
return instance;
}
}
/*
* #%L
* SUMARiS
* %%
* Copyright (C) 2019 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
package net.sumaris.core.extraction.config;
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.config.SumarisConfiguration;
import net.sumaris.core.config.SumarisConfigurationOption;
import org.nuiton.config.ApplicationConfig;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.File;
@Slf4j
public class ExtractionConfiguration {
private static ExtractionConfiguration INSTANCE;
public static ExtractionConfiguration instance() {
if (INSTANCE == null) {
SumarisConfiguration delegate = SumarisConfiguration.getInstance();
Preconditions.checkNotNull(delegate, "SumarisConfiguration not initialized!");
INSTANCE = new ExtractionConfiguration(delegate);
}
return INSTANCE;
}
public static void setInstance(ExtractionConfiguration instance) {
INSTANCE = instance;
}
private final SumarisConfiguration delegate;
@Autowired
public ExtractionConfiguration(SumarisConfiguration configuration){
this.delegate = configuration;
setInstance(this);
}
public String getExtractionCliOutputFormat() {
return getApplicationConfig().getOption(ExtractionConfigurationOption.EXTRACTION_CLI_OUTPUT_FORMAT.getKey());
}
public ApplicationConfig getApplicationConfig() {
return delegate.getApplicationConfig();
}
}
package net.sumaris.core.extraction.config;
/*-
* #%L
* Quadrige3 Core :: Quadrige3 Server Core
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2017 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import net.sumaris.core.extraction.action.AgggregationAction;
import net.sumaris.core.extraction.action.ExtractionAction;
import org.nuiton.config.ConfigActionDef;
/**
* <p>
* BatchesServerConfigurationAction class.
* </p>
*/
public enum ExtractionConfigurationAction implements ConfigActionDef {
EXTRACTION(ExtractionAction.class.getName() + "#execute", "Execute an extraction", "--extraction"),
AGGREGATION(AgggregationAction.class.getName() + "#execute", "Execute an aggregation", "--aggregation"),
AGGREGATION_REFRESH(AgggregationAction.class.getName() + "#refresh", "Refresh aggregation products", "--aggregation-refresh");
public final String action;
public final String description;
public final String[] aliases;
ExtractionConfigurationAction(String action, String description, String... aliases) {
this.action = action;
this.description = description;
this.aliases = aliases;
}
/**
* {@inheritDoc}
*/
@Override
public String getAction() {
return action;
}
/**
* {@inheritDoc}
*/
@Override
public String[] getAliases() {
return aliases;
}
@Override
public String getDescription() {
return description;
}
}
package net.sumaris.core.extraction.config;
/*-
* #%L
* SUMARiS :: Sumaris Core Shared
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2018 SUMARiS Consortium
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import net.sumaris.core.config.LaunchModeEnum;
import net.sumaris.core.dao.technical.hibernate.spatial.HSQLSpatialDialect;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import org.nuiton.config.ConfigOptionDef;
import org.nuiton.version.Version;
import java.io.File;
import java.net.URL;
import java.util.Locale;
import static org.nuiton.i18n.I18n.n;
/**
* All application configuration options.
*
* @author Benoit Lavenier <benoit.lavenier@e-is.pro>
* @since 1.0
*/
public enum ExtractionConfigurationOption implements ConfigOptionDef {
// ------------------------------------------------------------------------//
// -- READ-WRITE OPTIONS ---------------------------------------------------//
// ------------------------------------------------------------------------//
/*
* Application options
*/
EXTRACTION_ENABLED(
"sumaris.extraction.enabled",
n("sumaris.config.option.extraction.enabled.description"),
Boolean.TRUE.toString(),
Boolean.class,
false),
EXTRACTION_CLI_OUTPUT_FORMAT(
"sumaris.extraction.cli.output.format",
n("sumaris.config.option.extraction.cli.output.format.description"),
LiveFormatEnum.RDB.getLabel(),
String.class,
false)
;
/** Configuration key. */
private final String key;
/** I18n key of option description */
private final String description;