Update to run with Podman
This commit is contained in:
@@ -168,6 +168,71 @@ class StorageBackend(ABC):
|
||||
# Actually get rid of all the things we set to None
|
||||
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
|
||||
|
||||
def add_bool(self, search_query, add_bool):
|
||||
"""
|
||||
Add the specified boolean matches to search query.
|
||||
"""
|
||||
if not add_bool:
|
||||
return
|
||||
for item in add_bool:
|
||||
search_query["query"]["bool"]["must"].append({"match_phrase": item})
|
||||
|
||||
def add_top(self, search_query, add_top, negative=False):
|
||||
"""
|
||||
Merge add_top with the base of the search_query.
|
||||
"""
|
||||
if not add_top:
|
||||
return
|
||||
if negative:
|
||||
for item in add_top:
|
||||
if "must_not" in search_query["query"]["bool"]:
|
||||
search_query["query"]["bool"]["must_not"].append(item)
|
||||
else:
|
||||
search_query["query"]["bool"]["must_not"] = [item]
|
||||
else:
|
||||
for item in add_top:
|
||||
if "query" not in search_query:
|
||||
search_query["query"] = {"bool": {"must": []}}
|
||||
search_query["query"]["bool"]["must"].append(item)
|
||||
|
||||
def schedule_check_aggregations(self, rule_object, result_map):
|
||||
"""
|
||||
Check the results of a scheduled query for aggregations.
|
||||
"""
|
||||
if rule_object.aggs is None:
|
||||
return result_map
|
||||
for index, (meta, result) in result_map.items():
|
||||
# Default to true, if no aggs are found, we still want to match
|
||||
match = True
|
||||
for agg_name, (operator, number) in rule_object.aggs.items():
|
||||
if agg_name in meta["aggs"]:
|
||||
agg_value = meta["aggs"][agg_name]["value"]
|
||||
|
||||
# TODO: simplify this, match is default to True
|
||||
if operator == ">":
|
||||
if agg_value > number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
elif operator == "<":
|
||||
if agg_value < number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
elif operator == "=":
|
||||
if agg_value == number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
else:
|
||||
match = False
|
||||
else:
|
||||
# No aggregation found, but it is required
|
||||
match = False
|
||||
result_map[index][0]["aggs"][agg_name]["match"] = match
|
||||
|
||||
return result_map
|
||||
|
||||
def query(self, user, search_query, **kwargs):
|
||||
# For time tracking
|
||||
start = time.process_time()
|
||||
@@ -188,6 +253,7 @@ class StorageBackend(ABC):
|
||||
"took": time_took_rounded,
|
||||
"cache": True,
|
||||
}
|
||||
print("S2", search_query)
|
||||
response = self.run_query(user, search_query, **kwargs)
|
||||
|
||||
# For Elasticsearch
|
||||
@@ -198,7 +264,20 @@ class StorageBackend(ABC):
|
||||
if "took" in response:
|
||||
if response["took"] is None:
|
||||
return None
|
||||
if len(response["hits"]["hits"]) == 0:
|
||||
if "error" in response:
|
||||
message = f"Error: {response['error']}"
|
||||
message_class = "danger"
|
||||
time_took = (time.process_time() - start) * 1000
|
||||
# Round to 3 significant figures
|
||||
time_took_rounded = round(
|
||||
time_took, 3 - int(floor(log10(abs(time_took)))) - 1
|
||||
)
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
"took": time_took_rounded,
|
||||
}
|
||||
elif len(response["hits"]["hits"]) == 0:
|
||||
message = "No results."
|
||||
message_class = "danger"
|
||||
time_took = (time.process_time() - start) * 1000
|
||||
@@ -213,7 +292,7 @@ class StorageBackend(ABC):
|
||||
}
|
||||
|
||||
# For Druid
|
||||
if "error" in response:
|
||||
elif "error" in response:
|
||||
if "errorMessage" in response:
|
||||
context = {
|
||||
"message": response["errorMessage"],
|
||||
@@ -240,6 +319,106 @@ class StorageBackend(ABC):
|
||||
time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1)
|
||||
return {"object_list": response_parsed, "took": time_took_rounded}
|
||||
|
||||
def construct_context_query(
|
||||
self, index, net, channel, src, num, size, type=None, nicks=None
|
||||
):
|
||||
# Get the initial query
|
||||
query = self.construct_query(None, size, blank=True)
|
||||
|
||||
extra_must = []
|
||||
extra_should = []
|
||||
extra_should2 = []
|
||||
if num:
|
||||
extra_must.append({"match_phrase": {"num": num}})
|
||||
if net:
|
||||
extra_must.append({"match_phrase": {"net": net}})
|
||||
if channel:
|
||||
extra_must.append({"match": {"channel": channel}})
|
||||
if nicks:
|
||||
for nick in nicks:
|
||||
extra_should2.append({"match": {"nick": nick}})
|
||||
|
||||
types = ["msg", "notice", "action", "kick", "topic", "mode"]
|
||||
fields = [
|
||||
"nick",
|
||||
"ident",
|
||||
"host",
|
||||
"channel",
|
||||
"ts",
|
||||
"msg",
|
||||
"type",
|
||||
"net",
|
||||
"src",
|
||||
"tokens",
|
||||
]
|
||||
query["fields"] = fields
|
||||
|
||||
if index == "internal":
|
||||
fields.append("mtype")
|
||||
if channel == "*status" or type == "znc":
|
||||
if {"match": {"channel": channel}} in extra_must:
|
||||
extra_must.remove({"match": {"channel": channel}})
|
||||
extra_should2 = []
|
||||
# Type is one of msg or notice
|
||||
# extra_should.append({"match": {"mtype": "msg"}})
|
||||
# extra_should.append({"match": {"mtype": "notice"}})
|
||||
extra_should.append({"match": {"type": "znc"}})
|
||||
extra_should.append({"match": {"type": "self"}})
|
||||
|
||||
extra_should2.append({"match": {"type": "znc"}})
|
||||
extra_should2.append({"match": {"nick": channel}})
|
||||
elif type == "auth":
|
||||
if {"match": {"channel": channel}} in extra_must:
|
||||
extra_must.remove({"match": {"channel": channel}})
|
||||
extra_should2 = []
|
||||
extra_should2.append({"match": {"nick": channel}})
|
||||
# extra_should2.append({"match": {"mtype": "msg"}})
|
||||
# extra_should2.append({"match": {"mtype": "notice"}})
|
||||
|
||||
extra_should.append({"match": {"type": "query"}})
|
||||
extra_should2.append({"match": {"type": "self"}})
|
||||
extra_should.append({"match": {"nick": channel}})
|
||||
else:
|
||||
for ctype in types:
|
||||
extra_should.append({"match": {"mtype": ctype}})
|
||||
else:
|
||||
for ctype in types:
|
||||
extra_should.append({"match": {"type": ctype}})
|
||||
# query = {
|
||||
# "index": index,
|
||||
# "limit": size,
|
||||
# "query": {
|
||||
# "bool": {
|
||||
# "must": [
|
||||
# # {"equals": {"src": src}},
|
||||
# # {
|
||||
# # "bool": {
|
||||
# # "should": [*extra_should],
|
||||
# # }
|
||||
# # },
|
||||
# # {
|
||||
# # "bool": {
|
||||
# # "should": [*extra_should2],
|
||||
# # }
|
||||
# # },
|
||||
# *extra_must,
|
||||
# ]
|
||||
# }
|
||||
# },
|
||||
# "fields": fields,
|
||||
# # "_source": False,
|
||||
# }
|
||||
if extra_must:
|
||||
for x in extra_must:
|
||||
query["query"]["bool"]["must"].append(x)
|
||||
if extra_should:
|
||||
query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
|
||||
if extra_should2:
|
||||
query["query"]["bool"]["must"].append(
|
||||
{"bool": {"should": [*extra_should2]}}
|
||||
)
|
||||
return query
|
||||
|
||||
@abstractmethod
|
||||
def query_results(self, **kwargs):
|
||||
pass
|
||||
|
||||
@@ -374,44 +374,6 @@ class ElasticsearchBackend(StorageBackend):
|
||||
|
||||
return search_query
|
||||
|
||||
def schedule_check_aggregations(self, rule_object, result_map):
|
||||
"""
|
||||
Check the results of a scheduled query for aggregations.
|
||||
"""
|
||||
if rule_object.aggs is None:
|
||||
return result_map
|
||||
for index, (meta, result) in result_map.items():
|
||||
# Default to true, if no aggs are found, we still want to match
|
||||
match = True
|
||||
for agg_name, (operator, number) in rule_object.aggs.items():
|
||||
if agg_name in meta["aggs"]:
|
||||
agg_value = meta["aggs"][agg_name]["value"]
|
||||
|
||||
# TODO: simplify this, match is default to True
|
||||
if operator == ">":
|
||||
if agg_value > number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
elif operator == "<":
|
||||
if agg_value < number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
elif operator == "=":
|
||||
if agg_value == number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
else:
|
||||
match = False
|
||||
else:
|
||||
# No aggregation found, but it is required
|
||||
match = False
|
||||
result_map[index][0]["aggs"][agg_name]["match"] = match
|
||||
|
||||
return result_map
|
||||
|
||||
def schedule_query_results_test_sync(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query test with reduced functionality.
|
||||
|
||||
@@ -1,12 +1,24 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pprint import pprint
|
||||
import httpx
|
||||
import orjson
|
||||
|
||||
import requests
|
||||
from django.conf import settings
|
||||
|
||||
from core.db import StorageBackend, add_defaults, dedup_list
|
||||
from core.db.processing import annotate_results, parse_results
|
||||
from core.db.processing import parse_results
|
||||
from core.lib.parsing import (
|
||||
QueryError,
|
||||
parse_date_time,
|
||||
parse_index,
|
||||
parse_rule,
|
||||
parse_sentiment,
|
||||
parse_size,
|
||||
parse_sort,
|
||||
parse_source,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,29 +33,113 @@ class ManticoreBackend(StorageBackend):
|
||||
"""
|
||||
pass # we use requests
|
||||
|
||||
def construct_query(self, query, size, index, blank=False):
|
||||
async def async_initialise(self, **kwargs):
|
||||
"""
|
||||
Initialise the Manticore client in async mode
|
||||
"""
|
||||
pass # we use requests
|
||||
|
||||
def delete_rule_entries(self, rule_id):
|
||||
"""
|
||||
Delete all entries for a given rule.
|
||||
:param rule_id: The rule ID to delete.
|
||||
"""
|
||||
# TODO
|
||||
|
||||
def construct_query(self, query, size=None, blank=False, **kwargs):
|
||||
"""
|
||||
Accept some query parameters and construct an OpenSearch query.
|
||||
"""
|
||||
if not size:
|
||||
size = 5
|
||||
query_base = {
|
||||
"index": index,
|
||||
"index": kwargs.get("index"),
|
||||
"limit": size,
|
||||
"query": {"bool": {"must": []}},
|
||||
}
|
||||
print("BASE", query_base)
|
||||
query_string = {
|
||||
"query_string": query,
|
||||
}
|
||||
if not blank:
|
||||
query_base["query"]["bool"]["must"].append(query_string)
|
||||
return query_base
|
||||
|
||||
def parse(self, response, **kwargs):
|
||||
parsed = parse_results(response, **kwargs)
|
||||
return parsed
|
||||
|
||||
def run_query(self, client, user, search_query):
|
||||
response = requests.post(
|
||||
f"{settings.MANTICORE_URL}/json/search", json=search_query
|
||||
)
|
||||
return response
|
||||
def run_query(self, user, search_query, **kwargs):
|
||||
"""
|
||||
Low level helper to run Manticore query.
|
||||
"""
|
||||
index = kwargs.get("index")
|
||||
raw = kwargs.get("raw")
|
||||
if search_query and not raw:
|
||||
search_query["index"] = index
|
||||
pprint(search_query)
|
||||
|
||||
|
||||
path = kwargs.get("path", "json/search")
|
||||
if raw:
|
||||
response = requests.post(
|
||||
f"{settings.MANTICORE_URL}/{path}", search_query
|
||||
)
|
||||
else:
|
||||
response = requests.post(
|
||||
f"{settings.MANTICORE_URL}/{path}", json=search_query
|
||||
)
|
||||
|
||||
return orjson.loads(response.text)
|
||||
|
||||
async def async_run_query(self, user, search_query, **kwargs):
|
||||
"""
|
||||
Low level helper to run Manticore query asynchronously.
|
||||
"""
|
||||
index = kwargs.get("index")
|
||||
search_query["index"] = index
|
||||
pprint(search_query)
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
f"{settings.MANTICORE_URL}/json/search", json=search_query
|
||||
)
|
||||
return orjson.loads(response.text)
|
||||
|
||||
async def async_store_matches(self, matches):
|
||||
"""
|
||||
Store a list of matches in Manticore.
|
||||
:param index: The index to store the matches in.
|
||||
:param matches: A list of matches to store.
|
||||
"""
|
||||
# TODO
|
||||
|
||||
def store_matches(self, matches):
|
||||
"""
|
||||
Store a list of matches in Manticore.
|
||||
:param index: The index to store the matches in.
|
||||
:param matches: A list of matches to store.
|
||||
"""
|
||||
# TODO
|
||||
|
||||
def prepare_schedule_query(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query with reduced functionality.
|
||||
"""
|
||||
# TODO
|
||||
|
||||
def schedule_query_results_test_sync(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query test with reduced functionality.
|
||||
Sync version for running from Django forms.
|
||||
Does not return results.
|
||||
"""
|
||||
# TODO
|
||||
|
||||
async def schedule_query_results(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query with reduced functionality and async.
|
||||
"""
|
||||
# TODO
|
||||
|
||||
def query_results(
|
||||
self,
|
||||
@@ -67,117 +163,77 @@ class ManticoreBackend(StorageBackend):
|
||||
query_created = False
|
||||
source = None
|
||||
add_defaults(query_params)
|
||||
# Check size
|
||||
|
||||
# Now, run the helpers for SIQTSRSS/ADR
|
||||
# S - Size
|
||||
# I - Index
|
||||
# Q - Query
|
||||
# T - Tags
|
||||
# S - Source
|
||||
# R - Ranges
|
||||
# S - Sort
|
||||
# S - Sentiment
|
||||
# A - Annotate
|
||||
# D - Dedup
|
||||
# R - Reverse
|
||||
|
||||
# S - Size
|
||||
if request.user.is_anonymous:
|
||||
sizes = settings.MANTICORE_MAIN_SIZES_ANON
|
||||
sizes = settings.MAIN_SIZES_ANON
|
||||
else:
|
||||
sizes = settings.MANTICORE_MAIN_SIZES
|
||||
sizes = settings.MAIN_SIZES
|
||||
if not size:
|
||||
if "size" in query_params:
|
||||
size = query_params["size"]
|
||||
if size not in sizes:
|
||||
message = "Size is not permitted"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
size = int(size)
|
||||
else:
|
||||
size = 20
|
||||
size = parse_size(query_params, sizes)
|
||||
if isinstance(size, dict):
|
||||
return size
|
||||
|
||||
rule_object = parse_rule(request.user, query_params)
|
||||
if isinstance(rule_object, dict):
|
||||
return rule_object
|
||||
|
||||
# Check index
|
||||
if "index" in query_params:
|
||||
index = query_params["index"]
|
||||
if index == "main":
|
||||
index = settings.MANTICORE_INDEX_MAIN
|
||||
else:
|
||||
if not request.user.has_perm(f"core.index_{index}"):
|
||||
message = "Not permitted to search by this index"
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
if index == "meta":
|
||||
index = settings.MANTICORE_INDEX_META
|
||||
elif index == "internal":
|
||||
index = settings.MANTICORE_INDEX_INT
|
||||
else:
|
||||
message = "Index is not valid."
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
if rule_object is not None:
|
||||
index = settings.INDEX_RULE_STORAGE
|
||||
add_bool.append({"rule_id": str(rule_object.id)})
|
||||
else:
|
||||
index = settings.MANTICORE_INDEX_MAIN
|
||||
# I - Index
|
||||
index = parse_index(request.user, query_params)
|
||||
if isinstance(index, dict):
|
||||
return index
|
||||
|
||||
# Create the search query
|
||||
if "query" in query_params:
|
||||
query = query_params["query"]
|
||||
search_query = self.construct_query(query, size, index)
|
||||
query_created = True
|
||||
else:
|
||||
if custom_query:
|
||||
search_query = custom_query
|
||||
# Q/T - Query/Tags
|
||||
search_query = self.parse_query(
|
||||
query_params, tags, size, custom_query, add_bool
|
||||
)
|
||||
# Query should be a dict, so check if it contains message here
|
||||
if "message" in search_query:
|
||||
return search_query
|
||||
|
||||
if tags:
|
||||
# Get a blank search query
|
||||
if not query_created:
|
||||
search_query = self.construct_query(None, size, index, blank=True)
|
||||
query_created = True
|
||||
for tagname, tagvalue in tags.items():
|
||||
add_bool.append({tagname: tagvalue})
|
||||
# S - Sources
|
||||
sources = parse_source(request.user, query_params)
|
||||
if isinstance(sources, dict):
|
||||
return sources
|
||||
total_count = len(sources)
|
||||
# Total is -1 due to the "all" source
|
||||
total_sources = (
|
||||
len(settings.MAIN_SOURCES) - 1 + len(settings.SOURCES_RESTRICTED)
|
||||
)
|
||||
|
||||
required_any = ["query_full", "query", "tags"]
|
||||
if not any([field in query_params.keys() for field in required_any]):
|
||||
if not custom_query:
|
||||
message = "Empty query!"
|
||||
message_class = "warning"
|
||||
return {"message": message, "class": message_class}
|
||||
# If the sources the user has access to are equal to all
|
||||
# possible sources, then we don't need to add the source
|
||||
# filter to the query.
|
||||
if total_count != total_sources:
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append(
|
||||
{"match_phrase": {"src": source_iter}}
|
||||
)
|
||||
if query_params["source"] != "all":
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
# Check for a source
|
||||
if "source" in query_params:
|
||||
source = query_params["source"]
|
||||
|
||||
if source in settings.SOURCES_RESTRICTED:
|
||||
if not request.user.has_perm("core.restricted_sources"):
|
||||
message = "Access denied"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
elif source not in settings.MAIN_SOURCES:
|
||||
message = "Invalid source"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
if source == "all":
|
||||
source = None # the next block will populate it
|
||||
|
||||
if source:
|
||||
sources = [source]
|
||||
else:
|
||||
sources = list(settings.MAIN_SOURCES)
|
||||
if request.user.has_perm("core.restricted_sources"):
|
||||
for source_iter in settings.SOURCES_RESTRICTED:
|
||||
sources.append(source_iter)
|
||||
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
total_count = 0
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}})
|
||||
total_count += 1
|
||||
total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
|
||||
if not total_count == total_sources:
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
# Date/time range
|
||||
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
|
||||
query_params.keys()
|
||||
):
|
||||
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
|
||||
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
|
||||
from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
|
||||
to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
|
||||
from_ts = int(from_ts.timestamp())
|
||||
to_ts = int(to_ts.timestamp())
|
||||
# R - Ranges
|
||||
# date_query = False
|
||||
from_ts, to_ts = parse_date_time(query_params)
|
||||
if from_ts:
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
@@ -188,115 +244,87 @@ class ManticoreBackend(StorageBackend):
|
||||
}
|
||||
add_top.append(range_query)
|
||||
|
||||
# Sorting
|
||||
if "sorting" in query_params:
|
||||
sorting = query_params["sorting"]
|
||||
if sorting not in ("asc", "desc", "none"):
|
||||
message = "Invalid sort"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if sorting in ("asc", "desc"):
|
||||
sort = [
|
||||
{
|
||||
"ts": {
|
||||
"order": sorting,
|
||||
}
|
||||
}
|
||||
]
|
||||
# S - Sort
|
||||
sort = parse_sort(query_params)
|
||||
if isinstance(sort, dict):
|
||||
return sort
|
||||
|
||||
# Sentiment handling
|
||||
if "check_sentiment" in query_params:
|
||||
if "sentiment_method" not in query_params:
|
||||
message = "No sentiment method"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if "sentiment" in query_params:
|
||||
sentiment = query_params["sentiment"]
|
||||
try:
|
||||
sentiment = float(sentiment)
|
||||
except ValueError:
|
||||
message = "Sentiment is not a float"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
sentiment_method = query_params["sentiment_method"]
|
||||
range_query_compare = {"range": {"sentiment": {}}}
|
||||
if rule_object is not None:
|
||||
field = "match_ts"
|
||||
else:
|
||||
field = "ts"
|
||||
if sort:
|
||||
# For Druid compatibility
|
||||
sort_map = {"ascending": "asc", "descending": "desc"}
|
||||
sorting = [
|
||||
{
|
||||
field: {
|
||||
"order": sort_map[sort],
|
||||
}
|
||||
}
|
||||
]
|
||||
search_query["sort"] = sorting
|
||||
|
||||
# S - Sentiment
|
||||
sentiment_r = parse_sentiment(query_params)
|
||||
if isinstance(sentiment_r, dict):
|
||||
return sentiment_r
|
||||
if sentiment_r:
|
||||
if rule_object is not None:
|
||||
sentiment_index = "meta.aggs.avg_sentiment.value"
|
||||
else:
|
||||
sentiment_index = "sentiment"
|
||||
sentiment_method, sentiment = sentiment_r
|
||||
range_query_compare = {"range": {sentiment_index: {}}}
|
||||
range_query_precise = {
|
||||
"match": {
|
||||
"sentiment": None,
|
||||
sentiment_index: None,
|
||||
}
|
||||
}
|
||||
if sentiment_method == "below":
|
||||
range_query_compare["range"]["sentiment"]["lt"] = sentiment
|
||||
range_query_compare["range"][sentiment_index]["lt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "above":
|
||||
range_query_compare["range"]["sentiment"]["gt"] = sentiment
|
||||
range_query_compare["range"][sentiment_index]["gt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "exact":
|
||||
range_query_precise["match"]["sentiment"] = sentiment
|
||||
range_query_precise["match"][sentiment_index] = sentiment
|
||||
add_top.append(range_query_precise)
|
||||
elif sentiment_method == "nonzero":
|
||||
range_query_precise["match"]["sentiment"] = 0
|
||||
range_query_precise["match"][sentiment_index] = 0
|
||||
add_top_negative.append(range_query_precise)
|
||||
|
||||
if add_bool:
|
||||
# if "bool" not in search_query["query"]:
|
||||
# search_query["query"]["bool"] = {}
|
||||
# if "must" not in search_query["query"]["bool"]:
|
||||
# search_query["query"]["bool"] = {"must": []}
|
||||
# Add in the additional information we already populated
|
||||
self.add_bool(search_query, add_bool)
|
||||
self.add_top(search_query, add_top)
|
||||
self.add_top(search_query, add_top_negative, negative=True)
|
||||
|
||||
for item in add_bool:
|
||||
search_query["query"]["bool"]["must"].append({"match": item})
|
||||
|
||||
if add_top:
|
||||
for item in add_top:
|
||||
search_query["query"]["bool"]["must"].append(item)
|
||||
if add_top_negative:
|
||||
for item in add_top_negative:
|
||||
if "must_not" in search_query["query"]["bool"]:
|
||||
search_query["query"]["bool"]["must_not"].append(item)
|
||||
else:
|
||||
search_query["query"]["bool"]["must_not"] = [item]
|
||||
if sort:
|
||||
search_query["sort"] = sort
|
||||
|
||||
pprint(search_query)
|
||||
results = self.run_query(
|
||||
self.client,
|
||||
request.user, # passed through run_main_query to filter_blacklisted
|
||||
response = self.query(
|
||||
request.user,
|
||||
search_query,
|
||||
index=index,
|
||||
)
|
||||
if not results:
|
||||
if not response:
|
||||
message = "Error running query"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
# results = results.to_dict()
|
||||
if "error" in results:
|
||||
message = results["error"]
|
||||
if "error" in response:
|
||||
message = response["error"]
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
results_parsed = parse_results(results)
|
||||
if annotate:
|
||||
annotate_results(results_parsed)
|
||||
if "dedup" in query_params:
|
||||
if query_params["dedup"] == "on":
|
||||
dedup = True
|
||||
else:
|
||||
dedup = False
|
||||
else:
|
||||
dedup = False
|
||||
if "message" in response:
|
||||
return response
|
||||
|
||||
if reverse:
|
||||
results_parsed = results_parsed[::-1]
|
||||
# A/D/R - Annotate/Dedup/Reverse
|
||||
response["object_list"] = self.process_results(
|
||||
response["object_list"],
|
||||
annotate=annotate,
|
||||
dedup=dedup,
|
||||
dedup_fields=dedup_fields,
|
||||
reverse=reverse,
|
||||
)
|
||||
|
||||
if dedup:
|
||||
if not dedup_fields:
|
||||
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
|
||||
results_parsed = dedup_list(results_parsed, dedup_fields)
|
||||
context = {
|
||||
"object_list": results_parsed,
|
||||
"card": results["hits"]["total"],
|
||||
"took": results["took"],
|
||||
}
|
||||
if "cache" in results:
|
||||
context["cache"] = results["cache"]
|
||||
context = response
|
||||
return context
|
||||
|
||||
302
core/db/manticore_orig.py
Normal file
302
core/db/manticore_orig.py
Normal file
@@ -0,0 +1,302 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pprint import pprint
|
||||
|
||||
import requests
|
||||
from django.conf import settings
|
||||
|
||||
from core.db import StorageBackend, add_defaults, dedup_list
|
||||
from core.db.processing import annotate_results, parse_results
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManticoreBackend(StorageBackend):
|
||||
def __init__(self):
|
||||
super().__init__("manticore")
|
||||
|
||||
def initialise(self, **kwargs):
|
||||
"""
|
||||
Initialise the Manticore client
|
||||
"""
|
||||
pass # we use requests
|
||||
|
||||
def construct_query(self, query, size, index, blank=False):
|
||||
"""
|
||||
Accept some query parameters and construct an OpenSearch query.
|
||||
"""
|
||||
if not size:
|
||||
size = 5
|
||||
query_base = {
|
||||
"index": index,
|
||||
"limit": size,
|
||||
"query": {"bool": {"must": []}},
|
||||
}
|
||||
query_string = {
|
||||
"query_string": query,
|
||||
}
|
||||
if not blank:
|
||||
query_base["query"]["bool"]["must"].append(query_string)
|
||||
return query_base
|
||||
|
||||
def run_query(self, client, user, search_query):
|
||||
response = requests.post(
|
||||
f"{settings.MANTICORE_URL}/json/search", json=search_query
|
||||
)
|
||||
return response
|
||||
|
||||
def query_results(
|
||||
self,
|
||||
request,
|
||||
query_params,
|
||||
size=None,
|
||||
annotate=True,
|
||||
custom_query=False,
|
||||
reverse=False,
|
||||
dedup=False,
|
||||
dedup_fields=None,
|
||||
tags=None,
|
||||
):
|
||||
query = None
|
||||
message = None
|
||||
message_class = None
|
||||
add_bool = []
|
||||
add_top = []
|
||||
add_top_negative = []
|
||||
sort = None
|
||||
query_created = False
|
||||
source = None
|
||||
add_defaults(query_params)
|
||||
# Check size
|
||||
if request.user.is_anonymous:
|
||||
sizes = settings.MANTICORE_MAIN_SIZES_ANON
|
||||
else:
|
||||
sizes = settings.MANTICORE_MAIN_SIZES
|
||||
if not size:
|
||||
if "size" in query_params:
|
||||
size = query_params["size"]
|
||||
if size not in sizes:
|
||||
message = "Size is not permitted"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
size = int(size)
|
||||
else:
|
||||
size = 20
|
||||
|
||||
# Check index
|
||||
if "index" in query_params:
|
||||
index = query_params["index"]
|
||||
if index == "main":
|
||||
index = settings.MANTICORE_INDEX_MAIN
|
||||
else:
|
||||
if not request.user.has_perm(f"core.index_{index}"):
|
||||
message = "Not permitted to search by this index"
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
if index == "meta":
|
||||
index = settings.MANTICORE_INDEX_META
|
||||
elif index == "internal":
|
||||
index = settings.MANTICORE_INDEX_INT
|
||||
else:
|
||||
message = "Index is not valid."
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
else:
|
||||
index = settings.MANTICORE_INDEX_MAIN
|
||||
|
||||
# Create the search query
|
||||
if "query" in query_params:
|
||||
query = query_params["query"]
|
||||
search_query = self.construct_query(query, size, index)
|
||||
query_created = True
|
||||
else:
|
||||
if custom_query:
|
||||
search_query = custom_query
|
||||
|
||||
if tags:
|
||||
# Get a blank search query
|
||||
if not query_created:
|
||||
search_query = self.construct_query(None, size, index, blank=True)
|
||||
query_created = True
|
||||
for tagname, tagvalue in tags.items():
|
||||
add_bool.append({tagname: tagvalue})
|
||||
|
||||
required_any = ["query_full", "query", "tags"]
|
||||
if not any([field in query_params.keys() for field in required_any]):
|
||||
if not custom_query:
|
||||
message = "Empty query!"
|
||||
message_class = "warning"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
# Check for a source
|
||||
if "source" in query_params:
|
||||
source = query_params["source"]
|
||||
|
||||
if source in settings.SOURCES_RESTRICTED:
|
||||
if not request.user.has_perm("core.restricted_sources"):
|
||||
message = "Access denied"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
elif source not in settings.MAIN_SOURCES:
|
||||
message = "Invalid source"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
if source == "all":
|
||||
source = None # the next block will populate it
|
||||
|
||||
if source:
|
||||
sources = [source]
|
||||
else:
|
||||
sources = list(settings.MAIN_SOURCES)
|
||||
if request.user.has_perm("core.restricted_sources"):
|
||||
for source_iter in settings.SOURCES_RESTRICTED:
|
||||
sources.append(source_iter)
|
||||
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
total_count = 0
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}})
|
||||
total_count += 1
|
||||
total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
|
||||
if not total_count == total_sources:
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
# Date/time range
|
||||
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
|
||||
query_params.keys()
|
||||
):
|
||||
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
|
||||
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
|
||||
from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
|
||||
to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
|
||||
from_ts = int(from_ts.timestamp())
|
||||
to_ts = int(to_ts.timestamp())
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
"gt": from_ts,
|
||||
"lt": to_ts,
|
||||
}
|
||||
}
|
||||
}
|
||||
add_top.append(range_query)
|
||||
|
||||
# Sorting
|
||||
if "sorting" in query_params:
|
||||
sorting = query_params["sorting"]
|
||||
if sorting not in ("asc", "desc", "none"):
|
||||
message = "Invalid sort"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if sorting in ("asc", "desc"):
|
||||
sort = [
|
||||
{
|
||||
"ts": {
|
||||
"order": sorting,
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Sentiment handling
|
||||
if "check_sentiment" in query_params:
|
||||
if "sentiment_method" not in query_params:
|
||||
message = "No sentiment method"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if "sentiment" in query_params:
|
||||
sentiment = query_params["sentiment"]
|
||||
try:
|
||||
sentiment = float(sentiment)
|
||||
except ValueError:
|
||||
message = "Sentiment is not a float"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
sentiment_method = query_params["sentiment_method"]
|
||||
range_query_compare = {"range": {"sentiment": {}}}
|
||||
range_query_precise = {
|
||||
"match": {
|
||||
"sentiment": None,
|
||||
}
|
||||
}
|
||||
if sentiment_method == "below":
|
||||
range_query_compare["range"]["sentiment"]["lt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "above":
|
||||
range_query_compare["range"]["sentiment"]["gt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "exact":
|
||||
range_query_precise["match"]["sentiment"] = sentiment
|
||||
add_top.append(range_query_precise)
|
||||
elif sentiment_method == "nonzero":
|
||||
range_query_precise["match"]["sentiment"] = 0
|
||||
add_top_negative.append(range_query_precise)
|
||||
|
||||
if add_bool:
|
||||
# if "bool" not in search_query["query"]:
|
||||
# search_query["query"]["bool"] = {}
|
||||
# if "must" not in search_query["query"]["bool"]:
|
||||
# search_query["query"]["bool"] = {"must": []}
|
||||
|
||||
for item in add_bool:
|
||||
search_query["query"]["bool"]["must"].append({"match": item})
|
||||
|
||||
if add_top:
|
||||
for item in add_top:
|
||||
search_query["query"]["bool"]["must"].append(item)
|
||||
if add_top_negative:
|
||||
for item in add_top_negative:
|
||||
if "must_not" in search_query["query"]["bool"]:
|
||||
search_query["query"]["bool"]["must_not"].append(item)
|
||||
else:
|
||||
search_query["query"]["bool"]["must_not"] = [item]
|
||||
if sort:
|
||||
search_query["sort"] = sort
|
||||
|
||||
pprint(search_query)
|
||||
results = self.run_query(
|
||||
self.client,
|
||||
request.user, # passed through run_main_query to filter_blacklisted
|
||||
search_query,
|
||||
)
|
||||
if not results:
|
||||
message = "Error running query"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
# results = results.to_dict()
|
||||
if "error" in results:
|
||||
message = results["error"]
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
results_parsed = parse_results(results)
|
||||
if annotate:
|
||||
annotate_results(results_parsed)
|
||||
if "dedup" in query_params:
|
||||
if query_params["dedup"] == "on":
|
||||
dedup = True
|
||||
else:
|
||||
dedup = False
|
||||
else:
|
||||
dedup = False
|
||||
|
||||
if reverse:
|
||||
results_parsed = results_parsed[::-1]
|
||||
|
||||
if dedup:
|
||||
if not dedup_fields:
|
||||
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
|
||||
results_parsed = dedup_list(results_parsed, dedup_fields)
|
||||
context = {
|
||||
"object_list": results_parsed,
|
||||
"card": results["hits"]["total"],
|
||||
"took": results["took"],
|
||||
}
|
||||
if "cache" in results:
|
||||
context["cache"] = results["cache"]
|
||||
return context
|
||||
@@ -1,5 +1,5 @@
|
||||
from datetime import datetime
|
||||
|
||||
import ast
|
||||
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
|
||||
|
||||
|
||||
@@ -92,6 +92,11 @@ def parse_results(results, meta=None):
|
||||
for field in list(element.keys()):
|
||||
if element[field] == "":
|
||||
del element[field]
|
||||
# Unfold the tokens
|
||||
if "tokens" in element:
|
||||
if element["tokens"].startswith('["'):
|
||||
tokens_parsed = ast.literal_eval(element["tokens"])
|
||||
element["tokens"] = tokens_parsed
|
||||
|
||||
# Split the timestamp into date and time
|
||||
if "ts" not in element:
|
||||
|
||||
Reference in New Issue
Block a user