Commit b103f560 authored by LAVENIER's avatar LAVENIER
Browse files

[enh] Liquibase: add ProcessingFrequency

[enh] Extraction: add ExtractionProduct.processingFrequency
[enh] Allow to disable use of gravatar icon, for users
parent f42a6a60
......@@ -31,25 +31,14 @@ import net.sumaris.core.extraction.exception.UnknownFormatException;
import net.sumaris.core.extraction.format.ProductFormatEnum;
import net.sumaris.core.extraction.service.AggregationService;
import net.sumaris.core.extraction.service.ExtractionProductService;
import net.sumaris.core.extraction.service.ExtractionService;
import net.sumaris.core.extraction.util.ExtractionFormats;
import net.sumaris.core.extraction.vo.AggregationTypeVO;
import net.sumaris.core.extraction.vo.ExtractionTypeVO;
import net.sumaris.core.model.referential.StatusEnum;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
import net.sumaris.core.service.ServiceLocator;
import net.sumaris.core.service.data.ProductService;
import net.sumaris.core.util.Beans;
import net.sumaris.core.util.Files;
import net.sumaris.core.util.StringUtils;
import net.sumaris.core.vo.technical.extraction.ExtractionProductFetchOptions;
import net.sumaris.core.vo.technical.extraction.ExtractionProductFilterVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
import org.apache.commons.collections4.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
/**
......@@ -90,43 +79,6 @@ public class AgggregationAction {
StringUtils.capitalize(type.getCategory().name().toLowerCase()),
type.getLabel());
ActionUtils.logConnectionProperties();
}
/**
* <p>refresh.</p>
*/
public void refresh() {
ExtractionConfiguration config = ExtractionConfiguration.instance();
AggregationService aggregationService = ServiceLocator.instance().getService("aggregationService", AggregationService.class);
ExtractionProductService productService = ServiceLocator.instance().getService("extractionProductService", ExtractionProductService.class);
List<ExtractionProductVO> products = productService.findByFilter(ExtractionProductFilterVO.builder()
.statusIds(new Integer[]{StatusEnum.ENABLE.getId(), StatusEnum.TEMPORARY.getId()})
.build(),
ExtractionProductFetchOptions.builder().build());
if (CollectionUtils.isEmpty(products)) {
log.info("No product found in database. Nothing to refresh");
return;
}
log.info("Refreshing aggregation products...");
ActionUtils.logConnectionProperties();
for (ExtractionProductVO product: products) {
log.info("Updating product {{}}...", product.getLabel());
aggregationService.refresh(product.getId());
try {
Thread.sleep(10000); // Waiting 10s, to avoid HSQLDB 'generale error'
}
catch (InterruptedException e) {
// Stop
return;
}
}
}
}
package net.sumaris.core.extraction.action;
/*
* #%L
* SIH-Adagio :: Shared
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2012 - 2014 Ifremer
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.action.ActionUtils;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.service.AggregationService;
import net.sumaris.core.extraction.service.ExtractionProductService;
import net.sumaris.core.extraction.service.ExtractionService;
import net.sumaris.core.model.referential.StatusEnum;
import net.sumaris.core.model.technical.extraction.ExtractionProduct;
import net.sumaris.core.model.technical.history.ProcessingFrequency;
import net.sumaris.core.model.technical.history.ProcessingFrequencyEnum;
import net.sumaris.core.service.ServiceLocator;
import net.sumaris.core.vo.technical.extraction.ExtractionProductFetchOptions;
import net.sumaris.core.vo.technical.extraction.ExtractionProductFilterVO;
import net.sumaris.core.vo.technical.extraction.ExtractionProductVO;
import org.apache.commons.collections4.CollectionUtils;
import java.util.List;
/**
* <p>DatabaseChangeLogAction class.</p>
*
*/
@Slf4j
public class ProductAction {
/**
* <p>Update a product (execute extraction or aggregation).</p>
*/
public void update() {
// Get beans
ExtractionConfiguration config = ExtractionConfiguration.instance();
ExtractionProductService productService = ServiceLocator.instance().getService("extractionProductService", ExtractionProductService.class);
ExtractionService extractionService = ServiceLocator.instance().getService("extractionService", ExtractionService.class);
AggregationService aggregationService = ServiceLocator.instance().getService("aggregationService", AggregationService.class);
// Get execution frequency
ProcessingFrequencyEnum frequency = config.getExtractionCliFrequency();
if (frequency == ProcessingFrequencyEnum.NEVER) {
log.error("Products with frequency '{}' cannot be updated!", frequency);
}
long now = System.currentTimeMillis();
log.info("Updating products... {frequency: '{}'}", frequency);
ActionUtils.logConnectionProperties();
// Get products to refresh
List<ExtractionProductVO> products = productService.findByFilter(ExtractionProductFilterVO.builder()
// Filter on public or private products
.statusIds(new Integer[]{StatusEnum.ENABLE.getId(), StatusEnum.TEMPORARY.getId()})
// With the expected frequency
.searchJoin(ExtractionProduct.Fields.PROCESSING_FREQUENCY)
.searchAttribute(ProcessingFrequency.Fields.LABEL)
.searchText(frequency.getLabel())
.build(),
ExtractionProductFetchOptions.builder().build());
if (CollectionUtils.isEmpty(products)) {
log.info("No product found.");
return;
}
for (ExtractionProductVO product: products) {
log.info("Updating product {{}}...", product.getLabel());
try {
aggregationService.updateProduct(product.getId());
Thread.sleep(10000); // Waiting 10s, to let DB drop tables (asynchronously)
}
catch (InterruptedException e) {
// Stop
return;
}
}
log.info("Updating products... {frequency: '{}'}", frequency);
}
}
......@@ -26,6 +26,8 @@ import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import net.sumaris.core.config.SumarisConfiguration;
import net.sumaris.core.config.SumarisConfigurationOption;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.model.technical.history.ProcessingFrequencyEnum;
import org.nuiton.config.ApplicationConfig;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -61,6 +63,31 @@ public class ExtractionConfiguration {
return getApplicationConfig().getOption(ExtractionConfigurationOption.EXTRACTION_CLI_OUTPUT_FORMAT.getKey());
}
public ProcessingFrequencyEnum getExtractionCliFrequency() {
String value = getApplicationConfig().getOption(ExtractionConfigurationOption.EXTRACTION_CLI_FREQUENCY.getKey());
try {
return ProcessingFrequencyEnum.valueOf(value);
}
catch (IllegalArgumentException e) {
throw new SumarisTechnicalException(String.format("Invalid frequency '%s'. Available values: %s",
value,
ProcessingFrequencyEnum.values()
));
}
}
public boolean enableExtractionProduct() {
return getApplicationConfig().getOptionAsBoolean(ExtractionConfigurationOption.EXTRACTION_PRODUCT_ENABLE.getKey());
}
public boolean enableTechnicalTablesUpdate() {
return getApplicationConfig().getOptionAsBoolean(SumarisConfigurationOption.ENABLE_TECHNICAL_TABLES_UPDATE.getKey());
}
public File getTempDirectory() {
return getApplicationConfig().getOptionAsFile(SumarisConfigurationOption.TMP_DIRECTORY.getKey());
}
public ApplicationConfig getApplicationConfig() {
return delegate.getApplicationConfig();
}
......
......@@ -26,6 +26,7 @@ package net.sumaris.core.extraction.config;
import net.sumaris.core.extraction.action.AgggregationAction;
import net.sumaris.core.extraction.action.ExtractionAction;
import net.sumaris.core.extraction.action.ProductAction;
import org.nuiton.config.ConfigActionDef;
/**
......@@ -37,7 +38,7 @@ public enum ExtractionConfigurationAction implements ConfigActionDef {
EXTRACTION(ExtractionAction.class.getName() + "#execute", "Execute an extraction", "--extraction"),
AGGREGATION(AgggregationAction.class.getName() + "#execute", "Execute an aggregation", "--aggregation"),
AGGREGATION_REFRESH(AgggregationAction.class.getName() + "#refresh", "Refresh aggregation products", "--aggregation-refresh");
EXTRACTION_PRODUCT_UPDATE(ProductAction.class.getName() + "#update", "Update extraction products", "--extraction-product-update");
public final String action;
public final String description;
......
......@@ -24,15 +24,9 @@ package net.sumaris.core.extraction.config;
* #L%
*/
import net.sumaris.core.config.LaunchModeEnum;
import net.sumaris.core.dao.technical.hibernate.spatial.HSQLSpatialDialect;
import net.sumaris.core.extraction.format.LiveFormatEnum;
import net.sumaris.core.model.technical.history.ProcessingFrequencyEnum;
import org.nuiton.config.ConfigOptionDef;
import org.nuiton.version.Version;
import java.io.File;
import java.net.URL;
import java.util.Locale;
import static org.nuiton.i18n.I18n.n;
......@@ -64,7 +58,21 @@ public enum ExtractionConfigurationOption implements ConfigOptionDef {
n("sumaris.config.option.extraction.cli.output.format.description"),
LiveFormatEnum.RDB.getLabel(),
String.class,
false)
false),
EXTRACTION_CLI_FREQUENCY(
"sumaris.extraction.cli.frequency",
n("sumaris.config.option.extraction.cli.frequency.description"),
ProcessingFrequencyEnum.DAILY.name(),
String.class,
false),
EXTRACTION_PRODUCT_ENABLE(
"sumaris.extraction.product.enable",
n("sumaris.config.option.extraction.product.enable.description"),
Boolean.FALSE.toString(),
Boolean.class,
false)
;
/** Configuration key. */
......
......@@ -57,12 +57,19 @@ public interface AggregationService {
@Transactional(readOnly = true)
AggregationTypeVO getTypeById(int id, ExtractionProductFetchOptions fetchOptions);
/**
* Refresh a product (execute the aggregation, using the filter stored in the product)
* @param productId
*/
@Transactional
void updateProduct(int productId);
/**
* Do an aggregate
*
* @param type
* @param filter
* @param strata
*/
@Transactional
AggregationContextVO execute(AggregationTypeVO type,
......@@ -81,12 +88,14 @@ public interface AggregationService {
@Nullable AggregationStrataVO strata,
int offset, int size, String sort, SortDirection direction);
@Transactional(readOnly = true)
AggregationTechResultVO getAggByTech(AggregationTypeVO format,
ExtractionFilterVO filter,
AggregationStrataVO strata,
String sort,
SortDirection direction);
@Transactional(readOnly = true)
MinMaxVO getAggMinMaxByTech(AggregationTypeVO format,
ExtractionFilterVO filter,
AggregationStrataVO strata);
......@@ -104,9 +113,6 @@ public interface AggregationService {
@Nullable String sort,
@Nullable SortDirection direction);
@Transactional
void refresh(int id);
@Transactional
AggregationTypeVO save(AggregationTypeVO type, @Nullable ExtractionFilterVO filter);
......
......@@ -46,6 +46,7 @@ import net.sumaris.core.extraction.vo.trip.rdb.AggregationRdbTripContextVO;
import net.sumaris.core.model.referential.StatusEnum;
import net.sumaris.core.model.technical.extraction.ExtractionCategoryEnum;
import net.sumaris.core.model.technical.extraction.IExtractionFormat;
import net.sumaris.core.model.technical.history.ProcessingFrequencyEnum;
import net.sumaris.core.util.Beans;
import net.sumaris.core.util.StringUtils;
import net.sumaris.core.vo.technical.extraction.*;
......@@ -305,9 +306,9 @@ public class AggregationServiceImpl implements AggregationService {
}
@Override
public void refresh(int id) {
public void updateProduct(int productId) {
ExtractionProductVO target = productService.findById(id, ExtractionProductFetchOptions.FOR_UPDATE).orElse(null);
ExtractionProductVO target = productService.findById(productId, ExtractionProductFetchOptions.FOR_UPDATE).orElse(null);
Collection<String> existingTablesToDrop = Lists.newArrayList(target.getTableNames());
// Read filter
......@@ -351,40 +352,46 @@ public class AggregationServiceImpl implements AggregationService {
@CacheEvict(cacheNames = ExtractionCacheNames.AGGREGATION_TYPE_BY_FORMAT, allEntries = true)
}
)
public AggregationTypeVO save(AggregationTypeVO type, @Nullable ExtractionFilterVO filter) {
Preconditions.checkNotNull(type);
Preconditions.checkNotNull(type.getLabel());
Preconditions.checkNotNull(type.getName());
public AggregationTypeVO save(AggregationTypeVO source, @Nullable ExtractionFilterVO filter) {
Preconditions.checkNotNull(source);
Preconditions.checkNotNull(source.getLabel());
Preconditions.checkNotNull(source.getName());
Collection<String> existingTablesToDrop = Lists.newArrayList();
// Load the product
ExtractionProductVO target = null;
if (type.getId() != null) {
target = productService.findById(type.getId(), ExtractionProductFetchOptions.FOR_UPDATE).orElse(null);
if (source.getId() != null) {
target = productService.findById(source.getId(), ExtractionProductFetchOptions.FOR_UPDATE).orElse(null);
}
boolean isNew = target == null;
if (isNew) {
target = new ExtractionProductVO();
target.setLabel(type.getLabel().toUpperCase());
target.setLabel(source.getLabel().toUpperCase());
// Check label != format
Preconditions.checkArgument(!Objects.equals(type.getLabel(), type.getRawFormatLabel()), "Invalid label. Expected pattern: <type_name>-NNN");
Preconditions.checkArgument(!Objects.equals(source.getLabel(), source.getRawFormatLabel()), "Invalid label. Expected pattern: <type_name>-NNN");
}
else {
// Check label was not changed
String previousLabel = target.getLabel();
Preconditions.checkArgument(previousLabel.equalsIgnoreCase(type.getLabel()), "Cannot change a product label");
Preconditions.checkArgument(previousLabel.equalsIgnoreCase(source.getLabel()), "Cannot change a product label");
filter = filter != null ? filter : readFilterString(target.getFilter());
}
// Check if need aggregate (ig new or if filter changed)
ProcessingFrequencyEnum frequency = source.getProcessingFrequencyId() != null
? ProcessingFrequencyEnum.valueOf(source.getProcessingFrequencyId())
: ProcessingFrequencyEnum.NEVER;
String filterAsString = writeFilterAsString(filter);
boolean aggregate = isNew || !Objects.equals(target.getFilter(), filterAsString);
// Run the aggregation (if need) before saving
if (aggregate) {
boolean needExecution = (isNew || !Objects.equals(target.getFilter(), filterAsString))
&& (frequency == ProcessingFrequencyEnum.MANUALLY);
// Execute the aggregation (if need) before saving
if (needExecution) {
// Should clean existing table
if (!isNew) {
......@@ -393,8 +400,8 @@ public class AggregationServiceImpl implements AggregationService {
// Prepare a executable type (with label=format)
AggregationTypeVO executableType = new AggregationTypeVO();
executableType.setLabel(type.getRawFormatLabel());
executableType.setCategory(type.getCategory());
executableType.setLabel(source.getRawFormatLabel());
executableType.setCategory(source.getCategory());
// Execute the aggregation
AggregationContextVO context = execute(executableType, filter, null);
......@@ -403,29 +410,30 @@ public class AggregationServiceImpl implements AggregationService {
toProductVO(context, target);
// Copy some properties from the given type
target.setName(type.getName());
target.setUpdateDate(type.getUpdateDate());
target.setDescription(type.getDescription());
target.setDocumentation(type.getDocumentation());
target.setStatusId(type.getStatusId());
target.setRecorderDepartment(type.getRecorderDepartment());
target.setRecorderPerson(type.getRecorderPerson());
target.setName(source.getName());
target.setUpdateDate(source.getUpdateDate());
target.setDescription(source.getDescription());
target.setDocumentation(source.getDocumentation());
target.setStatusId(source.getStatusId());
target.setRecorderDepartment(source.getRecorderDepartment());
target.setRecorderPerson(source.getRecorderPerson());
}
// Not need new aggregation: update entity before saving
else {
Preconditions.checkArgument(StringUtils.equalsIgnoreCase(target.getLabel(), type.getLabel()), "Cannot update the label of an existing product");
target.setName(type.getName());
target.setUpdateDate(type.getUpdateDate());
target.setDescription(type.getDescription());
target.setDocumentation(type.getDocumentation());
target.setComments(type.getComments());
target.setStatusId(type.getStatusId());
target.setUpdateDate(type.getUpdateDate());
target.setIsSpatial(type.getIsSpatial());
Preconditions.checkArgument(StringUtils.equalsIgnoreCase(target.getLabel(), source.getLabel()), "Cannot update the label of an existing product");
target.setName(source.getName());
target.setUpdateDate(source.getUpdateDate());
target.setDescription(source.getDescription());
target.setDocumentation(source.getDocumentation());
target.setComments(source.getComments());
target.setStatusId(source.getStatusId());
target.setUpdateDate(source.getUpdateDate());
target.setIsSpatial(source.getIsSpatial());
}
target.setStratum(type.getStratum());
target.setStratum(source.getStratum());
target.setFilter(filterAsString);
target.setProcessingFrequencyId(frequency.getId());
// Save the product
target = productService.save(target);
......
......@@ -40,6 +40,7 @@ import net.sumaris.core.event.config.ConfigurationReadyEvent;
import net.sumaris.core.event.config.ConfigurationUpdatedEvent;
import net.sumaris.core.exception.DataNotFoundException;
import net.sumaris.core.exception.SumarisTechnicalException;
import net.sumaris.core.extraction.config.ExtractionConfiguration;
import net.sumaris.core.extraction.dao.ExtractionDao;
import net.sumaris.core.extraction.dao.administration.ExtractionStrategyDao;
import net.sumaris.core.extraction.dao.technical.Daos;
......@@ -106,7 +107,7 @@ import java.util.stream.Collectors;
public class ExtractionServiceImpl implements ExtractionService {
@Autowired
protected SumarisConfiguration configuration;
protected ExtractionConfiguration configuration;
@Autowired
protected DataSource dataSource;
......@@ -146,9 +147,7 @@ public class ExtractionServiceImpl implements ExtractionService {
private boolean includeProductTypes;
private Map<IExtractionFormat,
ExtractionDao<? extends ExtractionContextVO,
? extends ExtractionFilterVO>>
private Map<IExtractionFormat, ExtractionDao<? extends ExtractionContextVO, ? extends ExtractionFilterVO>>
daosByFormat = Maps.newHashMap();
......@@ -171,7 +170,7 @@ public class ExtractionServiceImpl implements ExtractionService {
@EventListener({ConfigurationReadyEvent.class, ConfigurationUpdatedEvent.class})
protected void onConfigurationReady(ConfigurationEvent event) {
includeProductTypes = configuration.enableExtractionProduct();
includeProductTypes = configuration.enableExtractionProduct();
if (configuration.enableTechnicalTablesUpdate()) {
initRectangleLocations();
}
......@@ -204,7 +203,7 @@ public class ExtractionServiceImpl implements ExtractionService {
@Override
public List<ExtractionTypeVO> findByFilter(ExtractionTypeFilterVO filter) {
ImmutableList.Builder<ExtractionTypeVO> builder = ImmutableList.builder();
filter = filter != null ? filter : new ExtractionTypeFilterVO();
filter = ExtractionTypeFilterVO.nullToEmpty(filter);
// Exclude types with a DISABLE status, by default
if (ArrayUtils.isEmpty(filter.getStatusIds())) {
......@@ -689,7 +688,7 @@ public class ExtractionServiceImpl implements ExtractionService {
protected Map<String, String> getAliasByColumnMap(Set<String> tableNames) {
return tableNames.stream()
.collect(Collectors.toMap(
columnName -> columnName.toUpperCase(),
String::toUpperCase,
StringUtils::underscoreToChangeCase));
}
......@@ -761,7 +760,7 @@ public class ExtractionServiceImpl implements ExtractionService {
else {
ExtractionDao dao = daosByFormat.get(context.getFormat());
Preconditions.checkNotNull(dao);
log.info("Cleaning extraction #{}-{}", context.getLabel(), context.getId());
log.info("Cleaning extraction #{}-{}", context.getRawFormatLabel(), context.getId());
dao.clean(context);
}
}
......
......@@ -25,6 +25,7 @@ package net.sumaris.core.extraction.vo;
import lombok.*;
import lombok.experimental.FieldDefaults;
import lombok.experimental.FieldNameConstants;
import net.sumaris.core.model.technical.history.ProcessingFrequencyEnum;
import net.sumaris.core.vo.technical.extraction.AggregationStrataVO;
import java.util.Date;
......@@ -40,8 +41,12 @@ import java.util.List;
@FieldNameConstants
public class AggregationTypeVO extends ExtractionTypeVO {
String documentation;
/**
* The extraction filter used to create data. Useful to refresh the aggregation
*/
String filter;
Integer processingFrequencyId;
String documentation;
Date updateDate;
Date creationDate;
......
......@@ -54,10 +54,10 @@ public class ExtractionTypeVO implements IValueObject<Integer>,
String version;
String description;
String comments;
String docUrl;
String[] sheetNames;
Integer statusId;
Boolean isSpatial;
String docUrl;
@JsonIgnore
LiveFormatEnum liveFormat;
......
......@@ -22,13 +22,18 @@ package net.sumaris.core.extraction.vo.filter;
* #L%
*/
import lombok.AllArgsConstructor;