Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initialise ES client only on first search

This commit is contained in:
Mark Veidemanis 2023-01-16 07:20:37 +00:00
parent c08ecc036f
commit ef9734a34d
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
2 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/db/__init__.py
View File

@ -55,7 +55,7 @@ class StorageBackend(ABC):
self.log.info(f"Initialising storage backend {name}") self.log.info(f"Initialising storage backend {name}")
self.initialise_caching() self.initialise_caching()
self.initialise() # self.initialise()
@abstractmethod @abstractmethod
def initialise(self, **kwargs): def initialise(self, **kwargs):

22
core/db/elastic.py
View File

@ -2,7 +2,7 @@
# from datetime import datetime, timedelta # from datetime import datetime, timedelta
from django.conf import settings from django.conf import settings
from elasticsearch import Elasticsearch from elasticsearch import AsyncElasticsearch, Elasticsearch
from elasticsearch.exceptions import NotFoundError, RequestError from elasticsearch.exceptions import NotFoundError, RequestError
from core.db import StorageBackend, add_defaults from core.db import StorageBackend, add_defaults
@ -24,10 +24,12 @@ from core.lib.parsing import (
class ElasticsearchBackend(StorageBackend): class ElasticsearchBackend(StorageBackend):
def __init__(self): def __init__(self):
super().__init__("Elasticsearch") super().__init__("Elasticsearch")
self.client = None
self.async_client = None
def initialise(self, **kwargs): def initialise(self, **kwargs):
""" """
Inititialise the Elastuicsearch API endpoint. Inititialise the Elasticsearch API endpoint.
""" """
auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD) auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
client = Elasticsearch( client = Elasticsearch(
@ -35,6 +37,16 @@ class ElasticsearchBackend(StorageBackend):
) )
self.client = client self.client = client
async def async_initialise(self, **kwargs):
"""
Inititialise the Elasticsearch API endpoint in async mode.
"""
auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
client = AsyncElasticsearch(
settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
)
self.async_client = client
def construct_context_query( def construct_context_query(
self, index, net, channel, src, num, size, type=None, nicks=None self, index, net, channel, src, num, size, type=None, nicks=None
): ):
@ -186,6 +198,8 @@ class ElasticsearchBackend(StorageBackend):
Accept fields and size, for the fields we want to match and the Accept fields and size, for the fields we want to match and the
number of results to return. number of results to return.
""" """
if self.client is None:
self.initialise()
index = kwargs.get("index") index = kwargs.get("index")
try: try:
response = self.client.search(body=search_query, index=index) response = self.client.search(body=search_query, index=index)
@ -205,9 +219,11 @@ class ElasticsearchBackend(StorageBackend):
Accept fields and size, for the fields we want to match and the Accept fields and size, for the fields we want to match and the
number of results to return. number of results to return.
""" """
if self.async_client is None:
await self.async_initialise()
index = kwargs.get("index") index = kwargs.get("index")
try: try:
response = self.client.search(body=search_query, index=index) response = await self.async_client.search(body=search_query, index=index)
except RequestError as err: except RequestError as err:
print("Elasticsearch error", err) print("Elasticsearch error", err)
return err return err