def setup():
try:
from elasticsearch import Elasticsearch, ElasticsearchException
except ImportError:
raise unittest.SkipTest("elasticsearch-py not installed.")
es = Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'])
try:
es.info()
except ElasticsearchException as e:
raise unittest.SkipTest("elasticsearch not running on %r" % settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], e)
global test_runner
global old_config
from django.test.runner import DiscoverRunner
test_runner = DiscoverRunner()
test_runner.setup_test_environment()
old_config = test_runner.setup_databases()
python类ElasticsearchException()的实例源码
def search(self, body, **kwargs):
assert 'index' not in kwargs
if isinstance(body, string_types):
pass
else:
body=elastify(body)
tries = 0
while True:
tries += 1
try:
return self.esstore.es.search(
index=self.index_base,
body=body,
**kwargs)
except ElasticsearchException as e:
if (tries <= 1):
self.esstore.yellow()
else:
raise ESError(e)
def load_records(task, dataset_name, query_idx, records):
"""Load a single batch of QUEUE_PAGE records from the given query."""
dataset = model.get_dataset(dataset_name)
items = []
for record in records:
for item in map_record(dataset.queries[query_idx], record):
items.append(item)
try:
index_items(items)
except ElasticsearchException as exc:
time.sleep(30)
raise task.retry(exc=exc, countdown=30, max_retries=5)
log.info("[%r] Indexed %s records as %s documents...",
dataset_name, len(records), len(items))
def setup():
log = logging.getLogger('haystack')
try:
import elasticsearch
if not ((1, 0, 0) <= elasticsearch.__version__ < (2, 0, 0)):
raise ImportError
from elasticsearch import Elasticsearch, ElasticsearchException
except ImportError:
log.error("elasticsearch-py not installed.", exc_info=True)
raise unittest.SkipTest("elasticsearch-py not installed.")
es = Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
try:
es.info()
except ElasticsearchException as e:
log.error("elasticsearch not running on %r" % \
settings.HAYSTACK_CONNECTIONS['default']['URL'], exc_info=True)
raise unittest.SkipTest("elasticsearch not running on %r" % \
settings.HAYSTACK_CONNECTIONS['default']['URL'], e)
def simple_search(self):
"""Perform URI-based request search.
"""
accepted_params = ('q', 'df', 'default_operator', 'from', 'size')
kwargs = {k: self.config[k] for k in accepted_params if self.config[k]}
wl = curator.IndexList(self.client)
indices = ','.join(wl.working_list())
try:
result = self.client.search(index=indices, **kwargs)
except elasticsearch.ElasticsearchException as e:
logger.error(e.message)
sys.exit(2)
if self._return_object:
return True, result
else:
self._pp_exit(result)
return None
def full_search(self):
"""Perform search using Query DSL.
"""
accepted_params = ('from', 'size')
kwargs = {k: self.config[k] for k in accepted_params if self.config[k]}
try:
result = self.client.search(index=self.config.index,
body=self.config.body, **kwargs)
except elasticsearch.ElasticsearchException as e:
logger.error(e.message)
sys.exit(2)
if self._return_object:
return True, result
else:
self._pp_exit(result)
return None
def getNumberAlerts(timeframe, clientDomain):
''' retrieves number of alerts from index in timeframe (minutes)'''
try:
res = es.search(index=esindex, body={
"query": {
"bool": {
"must": [
{
"match": {
"clientDomain": clientDomain
}
}
],
"filter": [
{
"range": {
"createTime": {
"gte": "now-"+str(timeframe)+"m"
}
}
}
]
}
},
"size": 0
})
return res['hits']['total']
except ElasticsearchException as err:
print('ElasticSearch error: %s' % err)
return False
########################
### Functions to SET data
########################
def queryAlertsWithoutIP(maxAlerts, clientDomain):
""" Get IP addresses from alerts in elasticsearch """
try:
res = es.search(index=esindex, body={
"query": {
"match": {
"clientDomain": clientDomain
}
},
"sort": {
"recievedTime": {
"order": "desc"
}
},
"size": maxAlerts,
"_source": [
"createTime",
"peerType",
"country",
"originalRequestString",
"location",
"targetCountry",
"countryName",
"locationDestination",
"recievedTime",
"username",
"password",
"login"
]
})
return res["hits"]["hits"]
except ElasticsearchException as err:
print('ElasticSearch error: %s' % err)
return False
def queryAlerts(maxAlerts, clientDomain):
""" Get IP addresses from alerts in elasticsearch """
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"query": {
"match": {
"clientDomain": clientDomain
}
},
"sort": {
"recievedTime": {
"order": "desc"
}
},
"size": maxAlerts,
"_source": [
"createTime",
"recievedTime",
"peerIdent",
"peerType",
"country",
"targetCountry",
"originalRequestString",
"location",
"sourceEntryIp"
]
})
return res["hits"]["hits"]
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def queryAlertsWithoutIP(maxAlerts, clientDomain):
""" Get IP addresses from alerts in elasticsearch """
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"query": {
"match": {
"clientDomain": clientDomain
}
},
"sort": {
"recievedTime": {
"order": "desc"
}
},
"size": maxAlerts,
"_source": [
"createTime",
"peerType",
"country",
"originalRequestString",
"location",
"targetCountry",
"countryName",
"locationDestination",
"recievedTime",
"username",
"password",
"login"
]
})
return res["hits"]["hits"]
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def queryAlertsCount(timeframe, clientDomain):
""" Get number of Alerts in timeframe in elasticsearch """
# check if timespan = d or number
if timeframe == "day":
span = "now/d"
elif timeframe.isdecimal():
span = "now-%sm" % timeframe
else:
app.logger.error('Non numeric value in retrieveAlertsCount timespan. Must be decimal number (in minutes) or string "day"')
return False
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"query": {
"bool": {
"must": [
{
"match": {
"clientDomain": clientDomain
}
}
],
"filter": [
{
"range": {
"recievedTime": {
"gte": str(span)
}
}
}
]
}
},
"size": 0
})
return res['hits']['total']
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def test_cli_logs_elasticsearch_exceptions(self):
indexer = mock.MagicMock()
indexer.bulk().__enter__().index_record.side_effect = elasticsearch.ElasticsearchException
with mock.patch('kinto_elasticsearch.command_reindex.logger') as logger:
with mock.patch('kinto_elasticsearch.command_reindex.get_paginated_records',
return_value=[[{}, {}]]) as get_paginated_records:
reindex_records(indexer,
mock.sentinel.storage,
mock.sentinel.bucket_id,
mock.sentinel.collection_id)
get_paginated_records.assert_called_with(mock.sentinel.storage,
mock.sentinel.bucket_id,
mock.sentinel.collection_id)
logger.exception.assert_called_with('Failed to index record')
def test_response_is_served_if_indexer_fails(self):
with mock.patch("kinto_elasticsearch.indexer.elasticsearch.helpers.bulk",
side_effect=elasticsearch.ElasticsearchException):
r = self.app.post_json("/buckets/bid/collections/cid/records",
{"data": {"hola": "mundo"}},
headers=self.headers)
assert r.status_code == 201
def test_search_response_is_empty_if_indexer_fails(self):
with mock.patch("kinto_elasticsearch.indexer.Indexer.search",
side_effect=elasticsearch.ElasticsearchException):
resp = self.app.post("/buckets/bid/collections/cid/search",
headers=self.headers)
result = resp.json
assert result == {}
def reindex_records(indexer, storage, bucket_id, collection_id):
total = 0
for records in get_paginated_records(storage, bucket_id, collection_id):
try:
with indexer.bulk() as bulk:
for record in records:
bulk.index_record(bucket_id,
collection_id,
record=record)
print(".", end="")
total += len(bulk.operations)
except elasticsearch.ElasticsearchException:
logger.exception("Failed to index record")
print("\n%s records reindexed." % total)
def create_index(self,index):
try:
logger.debug("[elastic-logs] Creating index %s ...", index)
self.es.indices.create(index)
except ElasticsearchException, exp:
logger.error("[elastic-logs] exception while creating index %s: %s", index, str(exp))
def is_index_exists(self,index):
if not self.is_connected == CONNECTED:
try:
if self.es.indices.exists(index):
return True
else:
return False
except ElasticsearchException, exp:
logger.error("[elastic-logs] exception while checking the existance of the index %s: %s", index, str(exp))
return True
def __init__(self, name, host, port, doc_type, index):
self.host = host
self.port = port
self.doc_type = doc_type
self.index = index
DatabaseInterface.__init__(self,
name=name,
db_type='ElasticSearch',
conn_exception=elasticsearch.ConnectionError,
execution_exception=elasticsearch.ElasticsearchException,
type_converter=self.TYPE_CONVERTER,
deserialize_query=True)
def wait_es(cls):
es = AsyncElasticsearch(
['{}:{}'.format(cls.es_host, cls.es_port)], loop=cls.loop)
for i in range(40):
try:
await es.ping()
except elasticsearch.ElasticsearchException:
await asyncio.sleep(0.5, loop=cls.loop)
else:
es.close()
break
else:
raise RuntimeError("es connection error")
def queryAlertStats(clientDomain):
""" Get combined statistics from elasticsearch """
try:
res = es.search(index=esindex, body={
"aggs": {
"communityfilter": {
"filter": {
"term": {
"clientDomain": clientDomain
}
},
"aggs": {
"ctr": {
"range": {
"field": "createTime",
"ranges": [
{
"key": "1d",
"from": "now-1440m"
},
{
"key": "1h",
"from": "now-60m"
},
{
"key": "5m",
"from": "now-5m"
},
{
"key": "1m",
"from": "now-1m"
}
]
}
}}}
},
"size": 0
})
return res['aggregations']['communityfilter']['ctr']['buckets']
except ElasticsearchException as err:
print('ElasticSearch error: %s' % err)
return False
def queryAlertsCountWithType(timeframe, clientDomain):
""" Get number of Alerts in timeframe in elasticsearch """
# check if timespan = d or number
if timeframe == "day":
span = "now/d"
elif timeframe.isdecimal():
span = "now-%sm" % timeframe
else:
print('Non numeric value in retrieveAlertsCountWithType timespan. Must be decimal number (in minutes) or string "day"')
return False
try:
res = es.search(index=esindex, body={
"query": {
"range": {
"createTime": {
"gte": str(span)
}
}
},
"aggs": {
"communityfilter": {
"filter": {
"term": {
"clientDomain": clientDomain
}
},
"aggs": {
"honeypotTypes": {
"terms": {
"field": "peerType.keyword"
}
}
}
}
},
"size": 0
})
return res
except ElasticsearchException as err:
print('ElasticSearch error: %s' % err)
return False
def authenticate(username, token):
""" Authenticate user from cache or in ES """
# check for user in cache
authtoken = getCache(username, "user")
if authtoken is not False:
if len(authtoken) == 128:
tokenhash = hashlib.sha512(token.encode('utf-8')).hexdigest()
if authtoken == tokenhash:
return True
elif len(authtoken) == 32:
tokenhash = hashlib.md5(token.encode('utf-8')).hexdigest()
if authtoken == tokenhash:
return True
else:
app.logger.error('authenticate(): Hash "{0}" for user "{1}" is not matching md5 or sha512 length! Needs to be checked in memcache!'.format(authtoken, username))
# query ES
else:
try:
res = es.search(index=app.config['WSUSERINDEX'], body={
"query": {
"term": {
"peerName.keyword": username
}
}
})
if res["hits"]["total"] > 1:
app.logger.error('authenticate(): More than one user "%s" in ES index "users" found!' % username)
elif res["hits"]["total"] < 1:
app.logger.error('authenticate(): No user "%s" in ES index "users" found!' % username)
elif res["hits"]["total"] == 1:
authtoken = res["hits"]["hits"][0]["_source"]["token"]
getOnly = res["hits"]["hits"][0]["_source"]["getOnly"]
community = res["hits"]["hits"][0]["_source"]["community"]
if len(authtoken) == 128:
tokenhash = hashlib.sha512(token.encode('utf-8')).hexdigest()
if authtoken == tokenhash:
# add user and token to cache for 24h
setCache(username, authtoken, (60 * 60 * 24), "user")
return True
elif len(authtoken) == 32:
tokenhash = hashlib.md5(token.encode('utf-8')).hexdigest()
if authtoken == tokenhash:
# add user and token to cache for 24h
setCache(username, authtoken, (60 * 60 * 24),"user")
return True
else:
app.logger.error('authenticate(): Hash "{0}" for user "{1}" is not matching md5 or sha512 length! Needs to be checked in ES index!'.format(authtoken, username))
return False
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def queryAlertsCountWithType(timeframe, clientDomain):
""" Get number of Alerts in timeframe in elasticsearch """
# check if timespan = d or number
if timeframe == "day":
span = "now/d"
elif timeframe.isdecimal():
span = "now-%sm" % timeframe
else:
app.logger.error('Non numeric value in retrieveAlertsCountWithType timespan. Must be decimal number (in minutes) or string "day"')
return False
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"query": {
"range": {
"recievedTime": {
"gte": str(span)
}
}
},
"aggs": {
"communityfilter": {
"filter": {
"term": {
"clientDomain": clientDomain
}
},
"aggs": {
"honeypotTypes": {
"terms": {
"field": "peerType.keyword"
}
}
}
}
},
"size": 0
})
return res
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def queryDatasetAlertsPerMonth(days, clientDomain):
# check if months is a number
if days is None:
span = "now-1M/d"
elif days.isdecimal():
span = "now-%sd/d" % days
else:
app.logger.error('Non numeric value in datasetAlertsPerMonth timespan. Must be decimal number in days')
return False
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"query": {
"range": {
"createTime": {
"gte": str(span)
}
}
},
"aggs": {
"communityfilter": {
"filter": {
"term": {
"clientDomain": clientDomain
}
},
"aggs": {
"range": {
"date_histogram": {
"field": "createTime",
"interval": "day"
}
}
}
}
},
"size": 0
})
return res["aggregations"]["communityfilter"]["range"]
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def queryAlertStats(clientDomain):
""" Get combined statistics from elasticsearch """
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"aggs": {
"communityfilter": {
"filter": {
"term": {
"clientDomain": clientDomain
}
},
"aggs": {
"ctr": {
"range": {
"field": "createTime",
"ranges": [
{
"key": "1d",
"from": "now-1440m"
},
{
"key": "1h",
"from": "now-60m"
},
{
"key": "5m",
"from": "now-5m"
},
{
"key": "1m",
"from": "now-1m"
}
]
}
}}}
},
"size": 0
})
return res['aggregations']['communityfilter']['ctr']['buckets']
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
def queryForSingleIP(maxAlerts, ip, clientDomain):
""" Get data for specific IP addresse from elasticsearch """
try:
ipaddress.IPv4Address(ip)
if not ipaddress.ip_address(ip).is_global:
app.logger.debug('No global IP address given on /querySingleIP: %s' % str(request.args.get('ip')))
return False
except:
app.logger.debug('No valid IP given on /querySingleIP: %s' % str(request.args.get('ip')))
return False
try:
res = es.search(index=app.config['ELASTICINDEX'], body={
"query": {
"bool": {
"must": [
{
"term": {
"sourceEntryIp": str(ip)
}
},
{
"term": {
"clientDomain": clientDomain
}
}
]
}
},
"size": maxAlerts,
"sort": {
"createTime": {
"order": "desc"
}
},
"_source": [
"createTime",
"peerType",
"targetCountry",
"originalRequestString"
]
})
return res["hits"]["hits"]
except ElasticsearchException as err:
app.logger.error('ElasticSearch error: %s' % err)
return False
# Formatting functions
def search_view(request, **kwargs):
bucket_id = request.matchdict['bucket_id']
collection_id = request.matchdict['collection_id']
# Limit the number of results to return, based on existing Kinto settings.
paginate_by = request.registry.settings.get("paginate_by")
max_fetch_size = request.registry.settings["storage_max_fetch_size"]
if paginate_by is None or paginate_by <= 0:
paginate_by = max_fetch_size
configured = min(paginate_by, max_fetch_size)
# If the size is specified in query, ignore it if larger than setting.
specified = None
if "body" in kwargs:
try:
body = json.loads(kwargs["body"].decode("utf-8"))
specified = body.get("size")
except json.decoder.JSONDecodeError:
pass
if specified is None or specified > configured:
kwargs.setdefault("size", configured)
# Access indexer from views using registry.
indexer = request.registry.indexer
try:
results = indexer.search(bucket_id, collection_id, **kwargs)
except elasticsearch.NotFoundError as e:
# If plugin was enabled after the creation of the collection.
indexer.create_index(bucket_id, collection_id)
results = indexer.search(bucket_id, collection_id, **kwargs)
except elasticsearch.RequestError as e:
# Malformed query.
if isinstance(e.info["error"], dict):
message = e.info["error"]["reason"]
details = e.info["error"]["root_cause"][0]
else:
message = e.info["error"]
details = None
response = http_error(httpexceptions.HTTPBadRequest(),
errno=ERRORS.INVALID_PARAMETERS,
message=message,
details=details)
raise response
except elasticsearch.ElasticsearchException as e:
# General failure.
logger.exception("Index query failed.")
results = {}
return results
def commit_logs(self):
"""
Periodically called (commit_period), this method prepares a bunch of queued logs (commit_colume) to insert them in the index
"""
if not self.logs_cache:
return
if not self.is_connected == CONNECTED:
if not self.open():
logger.warning("[elastic-logs] log commiting failed")
logger.warning("[elastic-logs] %d lines to insert in the index", len(self.logs_cache))
return
logger.debug("[elastic-logs] commiting ...")
logger.debug("[elastic-logs] %d lines to insert in the index (max insertion is %d lines)", len(self.logs_cache), self.commit_volume)
# Flush all the stored log lines
logs_to_commit = 1
now = time.time()
some_logs = []
while True:
try:
# result = self.db[self.logs_collection].insert_one(self.logs_cache.popleft())
some_logs.append(self.logs_cache.popleft())
logs_to_commit = logs_to_commit + 1
if logs_to_commit >= self.commit_volume:
break
except IndexError:
logger.debug("[elastic-logs] prepared all available logs for commit")
break
except Exception, exp:
logger.error("[elastic-logs] exception: %s", str(exp))
logger.debug("[elastic-logs] time to prepare %s logs for commit (%2.4f)", logs_to_commit, time.time() - now)
now = time.time()
try:
# Insert lines to commit
result = helpers.bulk(self.es,some_logs,self.commit_volume)
logger.debug("[elastic-logs] inserted %d logs.", result)
except ElasticsearchException, exp:
self.close()
logger.error("[elastic-logs] Error occurred when commiting: %s", exp)
logger.debug("[elastic-logs] time to insert %s logs (%2.4f)", logs_to_commit, time.time() - now)
def query_streets(name=None, locality=None, department=None, state=None,
road=None, max=None, fields=[]):
"""Busca calles según parámetros de búsqueda de una consulta.
Args:
name (str): Nombre de la calle para filtrar (opcional).
locality (str): Nombre de la localidad para filtrar (opcional).
department (str): Nombre de departamento para filtrar (opcional).
state (str): ID o nombre de provincia para filtrar (opcional).
road_type (str): Nombre del tipo de camino para filtrar (opcional).
max (int): Limita la cantidad de resultados (opcional).
fields (list): Campos a devolver en los resultados (opcional).
Returns:
list: Resultados de búsqueda de calles.
"""
index = 'calles-*' # Search in all indexes by default.
terms = []
if name:
condition = build_condition(NAME, get_abbreviated(name), fuzzy=True)
terms.append(condition)
if road:
condition = build_condition(ROAD_TYPE, road, fuzzy=True)
terms.append(condition)
if locality:
condition = build_condition(LOCALITY, locality, fuzzy=True)
terms.append(condition)
if department:
condition = build_condition(DEPT, department, fuzzy=True)
terms.append(condition)
if state:
target_state = query_entity(STATES, state, max=1)
if target_state: # Narrows search to specific index.
index = '-'.join([STREETS, target_state[0][ID]])
if LOCATION in fields:
fields.extend([GEOM, START_R, START_L, END_R, END_L, FULL_NAME])
query = {'query': {'bool': {'must': terms}} if terms else {"match_all": {}},
'size': max or 10, '_source': fields}
try:
result = Elasticsearch().search(index=index, body=query)
except ElasticsearchException as error:
return []
return [parse_es(hit) for hit in result['hits']['hits']]
def query_entity(index, entity_id=None, name=None, department=None, state=None,
max=None, order=None, fields=[], flatten=False):
"""Busca entidades políticas (localidades, departamentos, o provincias)
según parámetros de búsqueda de una consulta.
Args:
index (str): Nombre del índice sobre el cual realizar la búsqueda.
name (str): Nombre del tipo de entidad (opcional).
department (str): ID o nombre de departamento para filtrar (opcional).
state (str): ID o nombre de provincia para filtrar (opcional).
max (int): Limita la cantidad de resultados (opcional).
order (str): Campo por el cual ordenar los resultados (opcional).
fields (list): Campos a devolver en los resultados (opcional).
Returns:
list: Resultados de búsqueda de entidades.
"""
terms = []
sorts = {}
if entity_id:
condition = build_condition(ID, entity_id)
terms.append(condition)
if name:
condition = build_condition(NAME, name, fuzzy=True)
terms.append(condition)
if department:
if department.isdigit():
condition = build_condition(DEPT_ID, department)
else:
condition = build_condition(DEPT_NAME, department, fuzzy=True)
terms.append(condition)
if state:
if state.isdigit():
condition = build_condition(STATE_ID, state)
else:
if len(state.split()) == 1:
condition = build_condition(STATE_NAME, state, fuzzy=True)
else:
condition = build_condition(STATE_NAME, state,
kind='match_phrase_prefix')
terms.append(condition)
if order:
if ID in order: sorts[ID_KEYWORD] = {'order': 'asc'}
if NAME in order: sorts[NAME_KEYWORD] = {'order': 'asc'}
query = {'query': {'bool': {'must': terms}} if terms else {"match_all": {}},
'size': max or 10, 'sort': sorts, '_source': fields}
try:
result = Elasticsearch().search(index=index, body=query)
except ElasticsearchException as error:
return []
return [parse_entity(hit, flatten) for hit in result['hits']['hits']]
