Fix some Manticore queries
parent
3b8735be72
commit
87324de666
GPG Key ID:
5ACFCEED46C0904F
@ -0,0 +1,164 @@
|
||||
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
|
||||
from django.conf import settings
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def annotate_results(results_parsed):
|
||||
"""
|
||||
Accept a list of dict objects, search for the number of channels and users.
|
||||
Add them to the object.
|
||||
Mutate it in place. Does not return anything.
|
||||
"""
|
||||
# Figure out items with net (not discord)
|
||||
nets = set()
|
||||
for x in results_parsed:
|
||||
if "net" in x:
|
||||
nets.add(x["net"])
|
||||
|
||||
for net in nets:
|
||||
# Annotate the online attribute from Threshold
|
||||
nicks = list(
|
||||
set(
|
||||
[
|
||||
x["nick"]
|
||||
for x in results_parsed
|
||||
if {"nick", "src", "net"}.issubset(x)
|
||||
and x["src"] == "irc"
|
||||
and x["net"] == net
|
||||
]
|
||||
)
|
||||
)
|
||||
channels = list(
|
||||
set(
|
||||
[
|
||||
x["channel"]
|
||||
for x in results_parsed
|
||||
if {"channel", "src", "net"}.issubset(x)
|
||||
and x["src"] == "irc"
|
||||
and x["net"] == net
|
||||
]
|
||||
)
|
||||
)
|
||||
online_info = annotate_online(net, nicks)
|
||||
# Annotate the number of users in the channel
|
||||
num_users = annotate_num_users(net, channels)
|
||||
# Annotate the number channels the user is on
|
||||
num_chans = annotate_num_chans(net, nicks)
|
||||
for item in results_parsed:
|
||||
if "net" in item:
|
||||
if item["net"] == net:
|
||||
if "nick" in item:
|
||||
if item["nick"] in online_info:
|
||||
item["online"] = online_info[item["nick"]]
|
||||
if "channel" in item:
|
||||
if item["channel"] in num_users:
|
||||
item["num_users"] = num_users[item["channel"]]
|
||||
if "nick" in item:
|
||||
if item["nick"] in num_chans:
|
||||
item["num_chans"] = num_chans[item["nick"]]
|
||||
|
||||
|
||||
def filter_blacklisted(user, response):
|
||||
"""
|
||||
Low level filter to take the raw OpenSearch response and remove
|
||||
objects from it we want to keep secret.
|
||||
Does not return, the object is mutated in place.
|
||||
"""
|
||||
response["redacted"] = 0
|
||||
response["exemption"] = None
|
||||
if user.is_superuser:
|
||||
response["exemption"] = True
|
||||
# is_anonymous = isinstance(user, AnonymousUser)
|
||||
# For every hit from ES
|
||||
for index, item in enumerate(list(response["hits"]["hits"])):
|
||||
# For every blacklisted type
|
||||
for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
|
||||
# Check this field we are matching exists
|
||||
if "_source" in item.keys():
|
||||
data_index = "_source"
|
||||
elif "fields" in item.keys():
|
||||
data_index = "fields"
|
||||
else:
|
||||
return False
|
||||
if blacklisted_type in item[data_index].keys():
|
||||
content = item[data_index][blacklisted_type]
|
||||
# For every item in the blacklisted array for the type
|
||||
for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
|
||||
blacklisted_type
|
||||
]:
|
||||
if blacklisted_item == str(content):
|
||||
# Remove the item
|
||||
if item in response["hits"]["hits"]:
|
||||
# Let the UI know something was redacted
|
||||
if (
|
||||
"exemption"
|
||||
not in response["hits"]["hits"][index][data_index]
|
||||
):
|
||||
response["redacted"] += 1
|
||||
# Anonymous
|
||||
if user.is_anonymous:
|
||||
# Just set it to none so the index is not off
|
||||
response["hits"]["hits"][index] = None
|
||||
else:
|
||||
if not user.has_perm("core.bypass_blacklist"):
|
||||
response["hits"]["hits"][index] = None
|
||||
else:
|
||||
response["hits"]["hits"][index][data_index][
|
||||
"exemption"
|
||||
] = True
|
||||
|
||||
# Actually get rid of all the things we set to None
|
||||
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
|
||||
|
||||
|
||||
def parse_results(results):
|
||||
results_parsed = []
|
||||
stringify = ["host", "channel"]
|
||||
if "hits" in results.keys():
|
||||
if "hits" in results["hits"]:
|
||||
for item in results["hits"]["hits"]:
|
||||
if "_source" in item.keys():
|
||||
data_index = "_source"
|
||||
elif "fields" in item.keys():
|
||||
data_index = "fields"
|
||||
else:
|
||||
return False
|
||||
element = item[data_index]
|
||||
for field in stringify:
|
||||
if field in element:
|
||||
element[field] = str(element[field])
|
||||
# Why are fields in lists...
|
||||
if data_index == "fields":
|
||||
element = {k: v[0] for k, v in element.items() if len(v)}
|
||||
element["id"] = item["_id"]
|
||||
|
||||
# Remove empty values
|
||||
for field in list(element.keys()):
|
||||
if element[field] == "":
|
||||
del element[field]
|
||||
|
||||
# Split the timestamp into date and time
|
||||
if "ts" not in element:
|
||||
if "time" in element: # will fix data later
|
||||
ts = element["time"]
|
||||
del element["time"]
|
||||
element["ts"] = ts
|
||||
if "ts" in element:
|
||||
if isinstance(element["ts"], str):
|
||||
ts = element["ts"]
|
||||
else:
|
||||
ts = datetime.utcfromtimestamp(element["ts"]).strftime('%Y-%m-%dT%H:%M:%S')
|
||||
ts_spl = ts.split("T")
|
||||
date = ts_spl[0]
|
||||
time = ts_spl[1]
|
||||
element["date"] = date
|
||||
if "." in time:
|
||||
time_spl = time.split(".")
|
||||
if len(time_spl) == 2:
|
||||
element["time"] = time.split(".")[0]
|
||||
else:
|
||||
element["time"] = time
|
||||
else:
|
||||
element["time"] = time
|
||||
results_parsed.append(element)
|
||||
return results_parsed
|
||||
Loading…
Reference in New Issue