Fix annotating results and remove debugging code

This commit is contained in:
Mark Veidemanis 2022-11-23 18:39:36 +00:00
parent c49c8898eb
commit 1bdd332e6e
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
4 changed files with 10 additions and 17 deletions

View File

@ -267,8 +267,6 @@ class StorageBackend(ABC):
cache_hit = r.get(f"query_cache.{user.id}.{hash}") cache_hit = r.get(f"query_cache.{user.id}.{hash}")
if cache_hit: if cache_hit:
response = orjson.loads(cache_hit) response = orjson.loads(cache_hit)
print("CACHE HIT", response)
time_took = (time.process_time() - start) * 1000 time_took = (time.process_time() - start) * 1000
# Round to 3 significant figures # Round to 3 significant figures
time_took_rounded = round( time_took_rounded = round(
@ -301,8 +299,6 @@ class StorageBackend(ABC):
return context return context
else: else:
return response return response
# response = response.to_dict()
# print("RESP", response)
if "took" in response: if "took" in response:
if response["took"] is None: if response["took"] is None:
return None return None
@ -331,7 +327,7 @@ class StorageBackend(ABC):
def process_results(self, response, **kwargs): def process_results(self, response, **kwargs):
if kwargs.get("annotate"): if kwargs.get("annotate"):
annotate_results(response) annotate_results(response)
if kwargs.get("dedup"): if kwargs.get("reverse"):
response = response[::-1] response = response[::-1]
if kwargs.get("dedup"): if kwargs.get("dedup"):
if not kwargs.get("dedup_fields"): if not kwargs.get("dedup_fields"):

View File

@ -234,11 +234,6 @@ class DruidBackend(StorageBackend):
dedup_fields=dedup_fields, dedup_fields=dedup_fields,
reverse=reverse, reverse=reverse,
) )
# ss = orjson.dumps(list(response), option=orjson.OPT_INDENT_2)
# ss = ss.decode()
# print(ss)
# print("PARSED", results_parsed)
# return results_parsed
context = response context = response
return context return context

View File

@ -37,7 +37,7 @@ class ElasticsearchBackend(StorageBackend):
extra_should = [] extra_should = []
extra_should2 = [] extra_should2 = []
if num: if num:
extra_must.append({"equals": {"num": num}}) extra_must.append({"match_phrase": {"num": num}})
if net: if net:
extra_must.append({"match_phrase": {"net": net}}) extra_must.append({"match_phrase": {"net": net}})
if channel: if channel:
@ -321,10 +321,12 @@ class ElasticsearchBackend(StorageBackend):
search_query, search_query,
index=index, index=index,
) )
if "message" in response:
return response
# A/D/R - Annotate/Dedup/Reverse # A/D/R - Annotate/Dedup/Reverse
self.process_results( self.process_results(
response, response["object_list"],
annotate=annotate, annotate=annotate,
dedup=dedup, dedup=dedup,
dedup_fields=dedup_fields, dedup_fields=dedup_fields,

View File

@ -3,7 +3,7 @@ from datetime import datetime
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
def annotate_results(results_parsed): def annotate_results(results):
""" """
Accept a list of dict objects, search for the number of channels and users. Accept a list of dict objects, search for the number of channels and users.
Add them to the object. Add them to the object.
@ -11,7 +11,7 @@ def annotate_results(results_parsed):
""" """
# Figure out items with net (not discord) # Figure out items with net (not discord)
nets = set() nets = set()
for x in results_parsed["object_list"]: for x in results:
if "net" in x: if "net" in x:
nets.add(x["net"]) nets.add(x["net"])
@ -21,7 +21,7 @@ def annotate_results(results_parsed):
set( set(
[ [
x["nick"] x["nick"]
for x in results_parsed["object_list"] for x in results
if {"nick", "src", "net"}.issubset(x) if {"nick", "src", "net"}.issubset(x)
and x["src"] == "irc" and x["src"] == "irc"
and x["net"] == net and x["net"] == net
@ -32,7 +32,7 @@ def annotate_results(results_parsed):
set( set(
[ [
x["channel"] x["channel"]
for x in results_parsed["object_list"] for x in results
if {"channel", "src", "net"}.issubset(x) if {"channel", "src", "net"}.issubset(x)
and x["src"] == "irc" and x["src"] == "irc"
and x["net"] == net and x["net"] == net
@ -44,7 +44,7 @@ def annotate_results(results_parsed):
num_users = annotate_num_users(net, channels) num_users = annotate_num_users(net, channels)
# Annotate the number channels the user is on # Annotate the number channels the user is on
num_chans = annotate_num_chans(net, nicks) num_chans = annotate_num_chans(net, nicks)
for item in results_parsed["object_list"]: for item in results:
if "net" in item: if "net" in item:
if item["net"] == net: if item["net"] == net:
if "nick" in item: if "nick" in item: