Commit 8839ec1e authored by PIOLLE's avatar PIOLLE

added MultiSpatioTemporal search

parent f274685b
......@@ -164,7 +164,7 @@ if '__main__' == __name__:
else:
# global selection by default
lonmin, latmin, lonmax, latmax = -180, -90., 180., 90
area = shapely.geometry.box([lonmin, latmin, lonmax, latmax])
area = shapely.geometry.box(lonmin, latmin, lonmax, latmax)
if args.start:
start = dateutil.parser.parse(args.start)
......@@ -219,7 +219,6 @@ if '__main__' == __name__:
if args.verbose:
print("Got %d results:\n" % search.total)
print(res)
if len(res["error"]) != 0:
print(res["error"])
......
......@@ -19,23 +19,23 @@ from naiad.queries.query import Query
LOGGER = logging.getLogger('naiad.queries.index')
GRANULE_INDEX = {
"settings" : {
"index" : {
"number_of_shards" : 2,
"number_of_replicas" : 1
"settings": {
"index": {
"number_of_shards": 2,
"number_of_replicas": 1
}
},
"mappings": {
"granule": {
# don't analyze string fields
"dynamic_templates" : [ {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : True
"dynamic_templates": [{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "string",
"index": "not_analyzed",
"omit_norms": True
}
}
}],
......
'''
Created on 30 juil. 2015
@author: sumba
'''
"""
"""
import datetime
import logging
import json
import math
import inspect
import copy
from typing import List, Tuple, Union
from collections import OrderedDict
import shapely.geometry
import elasticsearch
......@@ -423,7 +422,7 @@ class GranuleInfoSearch(SpatialQuery):
result["data"] = self.decode_hit_result(
res['hits']['hits'][0]
)
)
return result
......@@ -662,7 +661,7 @@ class SpatioTemporalSearch(SpatialQuery):
return res
class MultiSpatioTemporalSearch(SpatialQuery):
class BulkTemporalSearch(SpatialQuery):
"""Performs several MultiSpatioTemporalSearch at once.
Args:
......@@ -672,7 +671,7 @@ class MultiSpatioTemporalSearch(SpatialQuery):
"""
def __init__(self, products, searches,
fail_silently=False):
super(MultiSpatioTemporalSearch, self).__init__(products, fail_silently)
super(BulkTemporalSearch, self).__init__(products, fail_silently)
self.searches = searches
def run(self,
......@@ -763,3 +762,54 @@ class MultiSpatioTemporalSearch(SpatialQuery):
return results
class MultiSpatioTemporalQuery(BulkTemporalSearch):
"""Search for product over a sequence of spatio-temporal targets.
Can be used for instance to search over hurricane track locations/
Args:
products (str): comma separated list of product names on which to
perform the searches
"""
def __init__(
self,
products: List[str],
targets: List[Tuple['datetime.datetime', 'shapely.geometry']],
delta_time: Union[
'datetime.timedelta',
Tuple['datetime.timedelta', 'datetime.timedelta']],
fail_silently: bool = False):
self.targets = targets
self.delta_time = delta_time
searches = []
for t, area in self.targets:
if isinstance(delta_time, tuple):
dt_before, dt_after = delta_time
else:
dt_before = dt_after = delta_time
searches.append(SpatioTemporalSearch(
products,
area=area,
start=t-dt_before,
end=t+dt_after
))
super(MultiSpatioTemporalQuery, self).__init__(
products, searches, fail_silently)
def run(self,
server,
precise: bool = False,
return_tiles: bool = False):
res = super(MultiSpatioTemporalQuery, self).run(
server, precise=precise, return_tiles=return_tiles)
crossovers = OrderedDict([])
if len(res) > 0:
for i, target in enumerate(self.targets):
crossovers[target[0].strftime('%Y%m%dT%H%M%S')] = res[i]['data']
return crossovers
......@@ -30,10 +30,15 @@ class Server(object):
of hosts.
"""
def __init__(self, hosts=None, login=None, password=None, sniffing=False):
def __init__(
self,
hosts=None,
login=None,
password=None,
sniffing=False):
self.login = login
self.password = password
self._set_esmaster(hosts)
self._set_esmaster(hosts, sniffing=sniffing)
def _set_esmaster(self, hosts=None, sniffing=False):
......@@ -54,7 +59,7 @@ class Server(object):
eshosts = ast.literal_eval(eshosts)
else:
# use localhost by default
eshosts = None
eshosts = 'http://localhost:9200'
else:
eshosts = hosts
if not isinstance(eshosts, list) and eshosts is not None:
......@@ -70,7 +75,7 @@ class Server(object):
http_auth = (self.login, self.password)
else:
http_auth = None
print(http_auth)
if sniffing:
self.esmaster = Elasticsearch(
hosts=eshosts,
......@@ -115,6 +120,9 @@ class Server(object):
Args:
index (str): name of the index
"""
logging.debug(
"Testing if index {} is existing in {}".format(index, self.esmaster)
)
return self.esmaster.indices.exists(index)
def start_bulk_optimization(self, index):
......
......@@ -142,11 +142,11 @@ class Tile(object):
def hash(self):
"""return a unique hash code identifying the tile"""
m = hashlib.md5()
m.update(self.granule)
m.update(str(self.xmin))
m.update(str(self.ymin))
m.update(str(self.xmax))
m.update(str(self.ymax))
m.update(self.granule.encode('utf-8'))
m.update(str(self.xmin).encode('utf-8'))
m.update(str(self.ymin).encode('utf-8'))
m.update(str(self.xmax).encode('utf-8'))
m.update(str(self.ymax).encode('utf-8'))
return m.hexdigest()
def slice(self):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment