diff --git a/core/lib/manticore.py b/core/lib/manticore.py index 8d3dd76..447012a 100644 --- a/core/lib/manticore.py +++ b/core/lib/manticore.py @@ -1,12 +1,22 @@ +import json +import logging +import random +import string +import time from datetime import datetime +from math import floor, log10 from pprint import pprint import manticoresearch from django.conf import settings +from siphashc import siphash +from core import r from core.lib.processing import annotate_results, filter_blacklisted, parse_results from core.views import helpers +logger = logging.getLogger(__name__) + def initialise_manticore(): """ @@ -22,6 +32,22 @@ def initialise_manticore(): api_client, client = initialise_manticore() +def initialise_caching(): + hash_key = r.get("cache_hash_key") + if not hash_key: + letters = string.ascii_lowercase + hash_key = "".join(random.choice(letters) for i in range(16)) + logger.debug(f"Created new hash key: {hash_key}") + r.set("cache_hash_key", hash_key) + else: + hash_key = hash_key.decode("ascii") + logger.debug(f"Decoded hash key: {hash_key}") + return hash_key + + +hash_key = initialise_caching() + + def construct_query(query, size, index, blank=False): """ Accept some query parameters and construct an OpenSearch query. @@ -42,9 +68,27 @@ def construct_query(query, size, index, blank=False): def run_query(client, user, search_query): + start = time.process_time() + query_normalised = json.dumps(search_query, sort_keys=True) + hash = siphash(hash_key, query_normalised) + cache_hit = r.get(f"query_cache.{user.id}.{hash}") + if cache_hit: + print("Cache hit") + response = json.loads(cache_hit) + time_took = (time.process_time() - start) * 1000 + # Round to 3 significant figures + time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1) + response["took"] = time_took_rounded + response["cache"] = True + return response response = client.search(search_query) response = response.to_dict() filter_blacklisted(user, response) + print("Writing to cache") + to_write_cache = json.dumps(response) + r.set(f"query_cache.{user.id}.{hash}", to_write_cache) + r.expire(f"query_cache.{user.id}.{hash}", 30) + print("Written to cache") return response @@ -166,7 +210,9 @@ def query_results( for source_iter in sources: add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}}) total_count += 1 - total_sources = len(settings.MANTICORE_MAIN_SOURCES) + len(settings.MANTICORE_SOURCES_RESTRICTED) + total_sources = len(settings.MANTICORE_MAIN_SOURCES) + len( + settings.MANTICORE_SOURCES_RESTRICTED + ) if not total_count == total_sources: add_top.append(add_top_tmp) @@ -293,4 +339,6 @@ def query_results( "card": results["hits"]["total"], "took": results["took"], } + if "cache" in results: + context["cache"] = results["cache"] return context