def test_index_percolate_query(self):
"""Test that we index the percolate query"""
query = {"query": {"match": {"profile.first_name": "here"}}}
percolate_query = PercolateQueryFactory.create(query=query, original_query="original")
percolate_query_id = 123
percolate_query.id = percolate_query_id
# Don't save since that will trigger a signal which will update the index
with self.assertRaises(NotFoundError):
es.get_percolate_query(percolate_query_id)
index_percolate_queries([percolate_query])
assert es.get_percolate_query(percolate_query_id) == {
'_id': str(percolate_query_id),
'_index': es.get_default_backing_index(),
'_source': query,
'_type': PERCOLATE_DOC_TYPE,
'_version': 1,
'found': True,
}
python类NotFoundError()的实例源码
def test_delete_percolate_queries(self):
"""Test that we delete the percolate query from the index"""
query = {"query": {"match": {"profile.first_name": "here"}}}
with patch('search.signals.transaction', on_commit=lambda callback: callback()):
percolate_query = PercolateQueryFactory.create(query=query, original_query="original")
assert es.get_percolate_query(percolate_query.id) == {
'_id': str(percolate_query.id),
'_index': es.get_default_backing_index(),
'_source': query,
'_type': PERCOLATE_DOC_TYPE,
'_version': 1,
'found': True,
}
delete_percolate_query(percolate_query.id)
with self.assertRaises(NotFoundError):
es.get_percolate_query(percolate_query.id)
# If we delete it again there should be no exception
delete_percolate_query(percolate_query.id)
with self.assertRaises(NotFoundError):
es.get_percolate_query(percolate_query.id)
def execute(self):
idx = es.Index(self.index)
try:
idx.delete(ignore=404)
log.info("Deleted index %s", self.index)
except AttributeError:
log.warning("Could not delete index '%s', ignoring", self.index)
except NotFoundError:
log.warning("Could not delete index '%s', ignoring", self.index)
# create doc types
for dt in self.doc_types:
idx.doc_type(dt)
# create index
idx.create()
def fetch_postcode(postcode, es, es_index="postcode", es_type="postcode"):
if postcode is None:
return None
areas = ["hro", "wz11", "bua11", "pct", "lsoa11", "nuts", "msoa11", "laua",
"oa11", "ccg", "ward", "teclec", "gor", "ttwa", "pfa", "pcon",
"lep1", "cty", "eer", "ctry", "park", "lep2", "hlthau", "buasd11"]
try:
res = es.get(index=es_index, doc_type=es_type,
id=postcode, ignore=[404])
if res['found']:
return (res['_source'].get("location"),
{k: res['_source'].get(k) for
k in res['_source'] if k in areas})
except (NotFoundError, ValueError):
return None
def get(self, index, id, doc_type='_all', params=None):
result = None
if index in self.__documents_dict:
for document in self.__documents_dict[index]:
if document.get('_id') == id:
if doc_type == '_all':
result = document
break
else:
if document.get('_type') == doc_type:
result = document
break
if result:
result['found'] = True
else:
error_data = {
'_index': index,
'_type': doc_type,
'_id': id,
'found': False
}
raise NotFoundError(404, json.dumps(error_data))
return result
def suggest(self, body, index=None, params=None):
if index is not None and index not in self.__documents_dict:
raise NotFoundError(404, 'IndexMissingException[[{0}] missing]'.format(index))
result_dict = {}
for key, value in body.items():
text = value.get('text')
suggestion = int(text) + 1 if isinstance(text, int) else '{0}_suggestion'.format(text)
result_dict[key] = [
{
'text': text,
'length': 1,
'options': [
{
'text': suggestion,
'freq': 1,
'score': 1.0
}
],
'offset': 0
}
]
return result_dict
def _normalize_index_to_list(self, index):
# Ensure to have a list of index
if index is None:
searchable_indexes = self.__documents_dict.keys()
elif isinstance(index, str) or isinstance(index, unicode):
searchable_indexes = [index]
elif isinstance(index, list):
searchable_indexes = index
else:
# Is it the correct exception to use ?
raise ValueError("Invalid param 'index'")
# Check index(es) exists
for searchable_index in searchable_indexes:
if searchable_index not in self.__documents_dict:
raise NotFoundError(404, 'IndexMissingException[[{0}] missing]'.format(searchable_index))
return searchable_indexes
def test_delete_dashid(self):
(l_srcName, l_dstName) = self.create_indices()
with patch('sys.stdout', new=StringIO()) as fake_out, patch('sys.stderr', new=StringIO()) as fake_err:
l_kibtool = kibtool.KibTool(["./test_kibtool", "--kibfrom", l_srcName,
"--dashid", "dashboard-1",
"--delete"])
l_kibtool.execute()
self.assertEquals(fake_out.getvalue().strip(), "")
self.assertEquals(fake_err.getvalue().strip(), "")
try:
self.client.get(index=l_srcName, doc_type="dashboard", id="dashboard-1")
self.assertTrue(False, "dashboard-1 still present")
except exceptions.NotFoundError as e:
pass
l_dst = self.client.search(index=l_dstName, doc_type="*", body={"query": {"match_all": {}}})
self.assertEquals(l_dst["hits"]["total"], 0)
def test_delete_dash(self):
(l_srcName, l_dstName) = self.create_indices()
with patch('sys.stdout', new=StringIO()) as fake_out, patch('sys.stderr', new=StringIO()) as fake_err:
l_kibtool = kibtool.KibTool(["./test_kibtool", "--kibfrom", l_srcName,
"--dash", "dashboard 1",
"--delete"])
l_kibtool.execute()
self.assertEquals(fake_out.getvalue().strip(), "")
self.assertEquals(fake_err.getvalue().strip(), "")
try:
self.client.get(index=l_srcName, doc_type="dashboard", id="dashboard-1")
self.assertTrue(False, "dashboard-1 still present")
except exceptions.NotFoundError as e:
pass
l_src = self.client.get(index=l_srcName, doc_type="dashboard", id="dashboard-8")
l_srcIdx = l_src.pop("_index")
self.assertEquals(l_srcIdx, l_srcName)
l_dst = self.client.search(index=l_dstName, doc_type="*", body={"query": {"match_all": {}}})
self.assertEquals(l_dst["hits"]["total"], 0)
def _synchronisation_op(self, elasticsearch_doctype, pending_insertions):
self._logging(logging.INFO,
'Computing required operations to synchronize documents.')
for p in pending_insertions:
doc_dict = p.to_dict(True)
try:
elasticsearch_doctype.get(p.id)
update_op = doc_dict
update_op['_op_type'] = 'update'
update_op['doc'] = doc_dict['_source']
del update_op['_source']
sync_op = update_op
except NotFoundError:
add_op = doc_dict
add_op['_op_type'] = 'index'
sync_op = add_op
yield sync_op
def get(uuid: str, replica: str):
owner = request.token_info['email']
es_client = ElasticsearchClient.get(logger)
try:
response = es_client.get(index=Config.get_es_index_name(ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name,
id=uuid)
except NotFoundError as ex:
raise DSSException(requests.codes.not_found, "not_found", "Cannot find subscription!")
source = response['_source']
source['uuid'] = uuid
source['replica'] = replica
if source['owner'] != owner:
# common_error_handler defaults code to capitalized 'Forbidden' for Werkzeug exception. Keeping consistent.
raise DSSException(requests.codes.forbidden, "Forbidden", "Your credentials can't access this subscription!")
return jsonify(source), requests.codes.okay
def update_template(es, max_retry, template_path, template_name):
with open(template_path) as f:
body = f.read()
for i in range(max_retry, 0, -1):
try:
es.indices.put_template(name=template_name, body=body)
log.info("Updating template {!r} done".format(template_name))
return
except (ConnectionError, NotFoundError):
log.warning(
"Updating template {!r} failed. Waiting for {} sec".format(
template_name, i))
time.sleep(i)
log.error("Updating template {!r} definitely failed".format(template_name))
def test_update_old_threads(self):
""" Test updating the index with old threads.
If there was a thread that was previously in the index and has
since been deleted, then it should be removed from the index.
"""
thread = create_thread()
thread_pk = thread.pk
backend = ElasticSearch()
backend.add(thread)
thread.delete()
call_command('updateindex', stdout=self.out)
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
with self.assertRaises(NotFoundError):
es.get_source(
index='test',
doc_type='thread',
id=thread_pk)
def test_remove(self):
""" Test removing an object from the search index.
Removing an object from the search index should make it
inaccessible to elasticsearch.
"""
thread = create_thread()
self.backend.add(thread)
self.backend.remove(thread)
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
with self.assertRaises(NotFoundError):
es.get_source(
index=self.backend.index,
doc_type='thread',
id=thread.pk)
def test_remove_message(self):
""" Test removing a thread with messages.
If a thread has messages assocated with it, those messages
should be removed from the search backend when the thread
instance is removed.
"""
thread = create_thread()
message = create_message(thread=thread)
self.backend.add(thread)
self.backend.remove(thread)
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
with self.assertRaises(NotFoundError):
es.get_source(
index=self.backend.index,
doc_type='message',
id=message.pk)
def test_wipe(self):
""" Test wiping the search index.
Objects in the search index prior to the wipe should no longer
be searchable.
"""
thread = create_thread()
self.backend.add(thread)
self.backend.wipe()
with self.assertRaises(NotFoundError):
self.backend.es.get_source(
index=self.backend.index,
doc_type='thread',
id=thread.pk)
def get(self, name):
'''get a named filter.'''
try:
return self.conn.get(self.ES_INDEX_NAME, name, self.ES_DOC_TYPE)['_source']
except NotFoundError:
return None
def document_needs_updating(enrollment):
"""
Get the document from elasticsearch and see if it matches what's in the database
Args:
enrollment (ProgramEnrollment): A program enrollment
Returns:
bool: True if the document needs to be updated via reindex
"""
conn = get_conn()
try:
document = conn.get(index=get_default_alias(), doc_type=USER_DOC_TYPE, id=enrollment.id)
except NotFoundError:
return True
serialized_enrollment = serialize_program_enrolled_user(enrollment)
del serialized_enrollment['_id']
source = document['_source']
if serialized_enrollment != source:
# Convert OrderedDict to dict
reserialized_enrollment = json.loads(json.dumps(serialized_enrollment))
diff = make_patch(source, reserialized_enrollment).patch
serialized_diff = json.dumps(diff, indent=" ")
log.info("Difference found for enrollment %s: %s", enrollment, serialized_diff)
return True
return False
def _delete_item(document_id, doc_type, index):
"""
Helper function to delete a document
Args:
document_id (int): A document id
doc_type (str): A document type
index (str): An Elasticsearch index
"""
conn = get_conn(verify_index=index)
try:
conn.delete(index=index, doc_type=doc_type, id=document_id)
except NotFoundError:
# Item is already gone
pass
def setup(self):
"""
Defers loading until needed.
"""
# Get the existing mapping & cache it. We'll compare it
# during the ``update`` & if it doesn't match, we'll put the new
# mapping.
try:
self.existing_mapping = self.conn.indices.get_mapping(index=self.index_name)
except NotFoundError:
pass
except Exception:
if not self.silently_fail:
raise
unified_index = haystack.connections[self.connection_alias].get_unified_index()
self.content_field_name, field_mapping = self.build_schema(unified_index.all_searchfields())
current_mapping = {
'modelresult': {
'properties': field_mapping,
}
}
if current_mapping != self.existing_mapping:
try:
# Make sure the index is there first.
self.conn.indices.create(index=self.index_name, body=self.DEFAULT_SETTINGS, ignore=400)
self.conn.indices.put_mapping(index=self.index_name, doc_type='modelresult', body=current_mapping)
self.existing_mapping = current_mapping
except Exception:
if not self.silently_fail:
raise
self.setup_complete = True
def es_read(self, keys, index, doc_type):
"""
Read from an ElasticSearch index and return a DataFrame
:param keys: a list of keys to extract in elasticsearch
:param index: the ElasticSearch index to read
:param doc_type: the ElasticSearch doc_type to read
"""
self.successful_ = 0
self.failed_ = 0
# Collect records for all of the keys
records = []
for key in keys:
try:
record = self.client.get(index=index, doc_type=doc_type, id=key)
self.successful_ += 1
if '_source' in record:
records.append(record['_source'])
except NotFoundError as nfe:
print('Key not found: %s' % nfe)
self.failed_ += 1
# Prepare the records into a single DataFrame
df = None
if records:
df = pd.DataFrame(records).fillna(value=np.nan)
df = df.reindex_axis(sorted(df.columns), axis=1)
return df
def post(self, **kwargs):
"""Get statistics."""
data = request.get_json(force=False)
if data is None:
data = {}
result = {}
for query_name, config in data.items():
if config is None or not isinstance(config, dict) \
or (set(config.keys()) != {'stat', 'params'} and
set(config.keys()) != {'stat'}):
raise InvalidRequestInputError(
'Invalid Input. It should be of the form '
'{ STATISTIC_NAME: { "stat": STAT_TYPE, '
'"params": STAT_PARAMS \}}'
)
stat = config['stat']
params = config.get('params', {})
try:
query_cfg = current_stats.queries[stat]
except KeyError:
raise UnknownQueryError(stat)
permission = current_stats.permission_factory(stat, params)
if permission is not None and not permission.can():
message = ('You do not have a permission to query the '
'statistic "{}" with those '
'parameters'.format(stat))
if current_user.is_authenticated:
abort(403, message)
abort(401, message)
try:
query = query_cfg.query_class(**query_cfg.query_config)
result[query_name] = query.run(**params)
except ValueError as e:
raise InvalidRequestInputError(e.args[0])
except NotFoundError as e:
return None
return self.make_response(result)
def test_should_raise_notfounderror_when_nonindexed_id_is_used(self):
with self.assertRaises(NotFoundError):
self.es.get(index=self.index_name, id='1')
def test_should_raise_notfounderror_when_search_for_unexistent_index(self):
with self.assertRaises(NotFoundError):
self.es.search(index=self.index_name)
def test_should_raise_exception_when_delete_nonindexed_document(self):
with self.assertRaises(NotFoundError):
self.es.delete(index=self.index_name, doc_type=self.doc_type, id=1)
def test_should_raise_notfounderror_when_nonindexed_id_is_used_for_suggest(self):
with self.assertRaises(NotFoundError):
self.es.suggest(body={}, index=self.index_name)
def getObjects(self, p_luceneReq, p_type, p_ctor):
l_request = {
"fields": ["_id"],
"size": self.m_args.count,
"sort": {
"_id": {
"order": "asc"
}
},
"query": {
"query_string" : {
"query" : "title:\"" + KibTool.toLuceneSyntax(p_luceneReq) + "\" AND _type:" + p_type
}
}
}
if self.m_args.debug:
print("---", l_request)
try:
l_response = self.m_esfrom.search(index=self.m_args.kibfrom, doc_type=p_type, body=l_request)
except exceptions.NotFoundError:
print("*** Can't search in unknown index", self.m_args.kibfrom, file=sys.stderr)
sys.exit(1)
l_result = []
if 0 == l_response["hits"]["total"]:
print("*** No %s found for '%s' in index %s/%s" %
(p_type, p_luceneReq, self.m_args.esfrom, self.m_args.kibfrom), file=sys.stderr)
sys.exit(1)
elif self.m_args.count < l_response["hits"]["total"]:
print("*** Please use a greater --count (%d) to select all %ss" %
(l_response["hits"]["total"], p_type), file=sys.stderr)
sys.exit(1)
else:
for c_hit in l_response["hits"]["hits"]:
l_d = p_ctor(self.m_esfrom, self.m_args.kibfrom, c_hit["_id"])
l_result.append(l_d)
return l_result
def deleteFromEs(self):
try:
self.m_json = self.m_es.delete(index=self.m_index, doc_type=self.m_type, id=self.m_id)
except exceptions.NotFoundError as e:
return
def readFromEs(self):
if self.m_json:
return
try:
self.m_json = self.m_es.get(index=self.m_index, doc_type=self.m_type, id=self.m_id)
except exceptions.NotFoundError as e:
return
def get_entity_dto(self, entity_uri):
"""Returns an EntityDAO given an entity_id
"""
e_uuid = hashlib.md5(entity_uri.encode('utf-8')).hexdigest()
try:
entity = self.es.get(index=self.index, doc_type=self.type,
id=e_uuid)
return EntityDTO(entity['_source'])
except es_exceptions.NotFoundError:
return EntityDTO({})
