Run meta on results from nicktrace in Insights

This commit is contained in:
Mark Veidemanis 2022-07-21 13:52:48 +01:00
parent c6c4cc0c24
commit 8710c66e05
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
5 changed files with 39 additions and 28 deletions

View File

@ -37,17 +37,20 @@ def get_meta(request, net, nicks, iter=True):
"""
# Split query into chunks
split_nicks = array_split(
nicks, ceil(len(nicks) / settings.META_MAX_CHUNK_SIZE)
)
split_nicks = array_split(nicks, ceil(len(nicks) / settings.META_MAX_CHUNK_SIZE))
meta = []
for nicks_chunked in split_nicks:
if len(nicks_chunked) == 0:
break
meta_tmp = []
query = construct_query(net, nicks_chunked)
results = run_main_query(client, request.user, query,
custom_query=True, index=settings.OPENSEARCH_INDEX_META)
results = run_main_query(
client,
request.user,
query,
custom_query=True,
index=settings.OPENSEARCH_INDEX_META,
)
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:

View File

@ -169,10 +169,9 @@ def query_results(request, size=None):
def query_single_result(request):
context = query_results(request, 1)
dedup_set = {item["nick"] for item in context["results"]}
if len(dedup_set) == 1:
if dedup_set:
context["item"] = context["results"][0]
else:
return (len(dedup_set), context)
return (1, context)

View File

@ -18,15 +18,6 @@
hx-target="#nicks"
hx-swap="outerHTML">
</div>
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'meta_insights' %}"
hx-trigger="load"
hx-target="#meta"
hx-swap="outerHTML">
</div>
<div id="info">
{% if item is not None %}
<div class="content" style="max-height: 30em; overflow: auto;">

View File

@ -1,5 +1,13 @@
{% load index %}
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}'
hx-post="{% url 'meta_insights' %}"
hx-trigger="load"
hx-target="#meta"
hx-swap="outerHTML">
</div>
<div id="nicks">
<div class="content" style="max-height: 30em; overflow: auto;">
<table class="table is-fullwidth is-hoverable">

View File

@ -1,3 +1,5 @@
from ast import literal_eval
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
from django.shortcuts import render
@ -87,28 +89,36 @@ class InsightsMeta(LoginRequiredMixin, APIView):
return HttpResponseForbidden()
if "net" not in request.data:
return HttpResponse("No net")
if "nick" not in request.data:
return HttpResponse("No nick")
if "nicks" not in request.data:
return HttpResponse("No nicks")
net = request.data["net"]
nick = request.data["nick"]
meta = get_meta(request, net, [nick])
nicks = request.data["nicks"]
nicks = literal_eval(nicks)
meta = get_meta(request, net, nicks)
unique_values = {}
# Create a map of unique values for each key for each nick
for x in meta:
nick = x["nick"]
if nick not in unique_values:
unique_values[nick] = {}
for k, v in x.items():
if k not in unique_values:
unique_values[k] = set()
unique_values[k].add(v)
if k not in unique_values[nick]:
unique_values[nick][k] = set()
if k in ["ts", "time", "date", "id"]:
continue
unique_values[nick][k].add(v)
meta_dedup = []
for x in meta:
nick = x["nick"]
meta_l2 = {}
for k, v in x.items():
if v in unique_values[k]:
if v in unique_values[nick][k]:
meta_l2[k] = v
unique_values[k].remove(v)
unique_values[nick][k].remove(v)
if not set(meta_l2.keys()).issubset(set(["ts", "time", "date", "id"])):
meta_dedup.append(meta_l2)
context = {"net": net, "nick": nick, "meta": meta_dedup}
context = {"net": net, "nicks": nicks, "meta": meta_dedup}
return render(request, self.template_name, context)