From 61f93390d94ddc27a67cd0e9e7c0bf614f8db5a5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 21 Nov 2022 07:20:29 +0000 Subject: [PATCH] Replace OpenSearch with Elasticsearch --- app/local_settings.example.py | 14 +++++++------- core/db/__init__.py | 4 ++-- core/db/elastic.py | 24 ++++++++++++------------ core/db/storage.py | 4 ++-- core/lib/meta.py | 4 ++-- docker/prod/requirements.prod.txt | 2 +- docker/requirements.dev.txt | 2 +- requirements.txt | 2 +- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/app/local_settings.example.py b/app/local_settings.example.py index 6593102..f0a1bbe 100644 --- a/app/local_settings.example.py +++ b/app/local_settings.example.py @@ -1,9 +1,9 @@ -# OpenSearch settings -OPENSEARCH_URL = "10.1.0.1" -OPENSEARCH_PORT = 9200 -OPENSEARCH_TLS = True -OPENSEARCH_USERNAME = "admin" -OPENSEARCH_PASSWORD = "secret" +# Elasticsearch settings +ELASTICSEARCH_URL = "10.1.0.1" +ELASTICSEARCH_PORT = 9200 +ELASTICSEARCH_TLS = True +ELASTICSEARCH_USERNAME = "admin" +ELASTICSEARCH_PASSWORD = "secret" # Manticore settings MANTICORE_URL = "http://example-db-1:9308" @@ -58,7 +58,7 @@ DRILLDOWN_DEFAULT_PARAMS = { # # Delay results by this many days # DELAY_DURATION = 10 -OPENSEARCH_BLACKLISTED = { +ELASTICSEARCH_BLACKLISTED = { } diff --git a/core/db/__init__.py b/core/db/__init__.py index a93eb33..c8029cd 100644 --- a/core/db/__init__.py +++ b/core/db/__init__.py @@ -217,7 +217,7 @@ class StorageBackend(object): # For every hit from ES for index, item in enumerate(list(response["hits"]["hits"])): # For every blacklisted type - for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys(): + for blacklisted_type in settings.ELASTICSEARCH_BLACKLISTED.keys(): # Check this field we are matching exists if "_source" in item.keys(): data_index = "_source" @@ -228,7 +228,7 @@ class StorageBackend(object): if blacklisted_type in item[data_index].keys(): content = item[data_index][blacklisted_type] # For every item in the blacklisted array for the type - for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[ + for blacklisted_item in settings.ELASTICSEARCH_BLACKLISTED[ blacklisted_type ]: if blacklisted_item == str(content): diff --git a/core/db/elastic.py b/core/db/elastic.py index 28ff60c..768e92c 100644 --- a/core/db/elastic.py +++ b/core/db/elastic.py @@ -2,8 +2,8 @@ # from datetime import datetime, timedelta from django.conf import settings -from opensearchpy import OpenSearch -from opensearchpy.exceptions import NotFoundError, RequestError +from elasticsearch import Elasticsearch +from elasticsearch.exceptions import NotFoundError, RequestError from core.db import StorageBackend @@ -19,18 +19,18 @@ class ElasticsearchBackend(StorageBackend): def initialise(self, **kwargs): """ - Inititialise the OpenSearch API endpoint. + Inititialise the Elastuicsearch API endpoint. """ - auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD) - client = OpenSearch( + auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD) + client = Elasticsearch( # fmt: off - hosts=[{"host": settings.OPENSEARCH_URL, - "port": settings.OPENSEARCH_PORT}], + hosts=[{"host": settings.ELASTICSEARCH_URL, + "port": settings.ELASTICSEARCH_PORT}], http_compress=False, # enables gzip compression for request bodies http_auth=auth, # client_cert = client_cert_path, # client_key = client_key_path, - use_ssl=settings.OPENSEARCH_TLS, + use_ssl=settings.ELASTICSEARCH_TLS, verify_certs=False, ssl_assert_hostname=False, ssl_show_warn=False, @@ -40,7 +40,7 @@ class ElasticsearchBackend(StorageBackend): def construct_query(self, query, size, use_query_string=True, tokens=False): """ - Accept some query parameters and construct an OpenSearch query. + Accept some query parameters and construct an Elasticsearch query. """ if not size: size = 5 @@ -116,10 +116,10 @@ class ElasticsearchBackend(StorageBackend): try: response = client.search(body=search_query, index=index) except RequestError as err: - print("OpenSearch error", err) + print("Elasticsearch error", err) return err except NotFoundError as err: - print("OpenSearch error", err) + print("Elasticsearch error", err) return err return response @@ -137,7 +137,7 @@ class ElasticsearchBackend(StorageBackend): tags=None, ): """ - API helper to alter the OpenSearch return format into something + API helper to alter the Elasticsearch return format into something a bit better to parse. Accept a HTTP request object. Run the query, and annotate the results with the other data we have. diff --git a/core/db/storage.py b/core/db/storage.py index 54fbf97..09b9cfc 100644 --- a/core/db/storage.py +++ b/core/db/storage.py @@ -7,9 +7,9 @@ def get_db(): return DruidBackend() elif settings.DB_BACKEND == "ELASTICSEARCH": - from core.db.elastic import ElasticSearchBackend + from core.db.elastic import ElasticsearchBackend - return OpensearchBackend() + return ElasticsearchBackend() elif settings.DB_BACKEND == "MANTICORE": from core.db.manticore import ManticoreBackend diff --git a/core/lib/meta.py b/core/lib/meta.py index 91a367b..a5ee778 100644 --- a/core/lib/meta.py +++ b/core/lib/meta.py @@ -3,7 +3,7 @@ from math import ceil from django.conf import settings from numpy import array_split -from core.db.opensearch import client, run_main_query +from core.db.elastic import client, run_main_query def construct_query(net, nicks): @@ -48,7 +48,7 @@ def get_meta(request, net, nicks, iter=True): request.user, query, custom_query=True, - index=settings.OPENSEARCH_INDEX_META, + index=settings.ELASTICSEARCH_INDEX_META, ) if "hits" in results.keys(): if "hits" in results["hits"]: diff --git a/docker/prod/requirements.prod.txt b/docker/prod/requirements.prod.txt index 6cac89f..7614fa7 100644 --- a/docker/prod/requirements.prod.txt +++ b/docker/prod/requirements.prod.txt @@ -2,7 +2,7 @@ wheel django django-crispy-forms crispy-bulma -#opensearch-py +elasticsearch stripe django-rest-framework numpy diff --git a/docker/requirements.dev.txt b/docker/requirements.dev.txt index f940731..4471d43 100644 --- a/docker/requirements.dev.txt +++ b/docker/requirements.dev.txt @@ -2,7 +2,7 @@ wheel django django-crispy-forms crispy-bulma -#opensearch-py +elasticsearch stripe django-rest-framework numpy diff --git a/requirements.txt b/requirements.txt index 1600408..a3b253b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ django pre-commit django-crispy-forms crispy-bulma -#opensearch-py +elasticsearch stripe django-rest-framework numpy