Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use generic meta variable for returning more data about the search

This commit is contained in:
Mark Veidemanis 2023-01-16 07:20:37 +00:00
parent bea84ee2b6
commit 2eb090f088
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
3 changed files with 20 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
core/db/elastic.py
View File

@ -309,20 +309,21 @@ class ElasticsearchBackend(StorageBackend):
if len(response["hits"]["hits"]) == 0: if len(response["hits"]["hits"]) == 0:
# No results, skip # No results, skip
continue continue
aggs, response = self.parse(response, aggs=True) meta, response = self.parse(response, meta=True)
print("Parsed response", response)
if "message" in response: if "message" in response:
self.log.error(f"Error running scheduled search: {response['message']}") self.log.error(f"Error running scheduled search: {response['message']}")
continue continue
result_map[index] = (aggs, response) result_map[index] = (meta, response)
# Average aggregation check # Average aggregation check
# Could probably do this in elasticsearch # Could probably do this in elasticsearch
for index, (aggs, result) in result_map.items(): for index, (meta, result) in result_map.items():
# Default to true, if no aggs are found, we still want to match # Default to true, if no aggs are found, we still want to match
match = True match = True
for agg_name, (operator, number) in rule_object.aggs.items(): for agg_name, (operator, number) in rule_object.aggs.items():
if agg_name in aggs: if agg_name in meta:
agg_value = aggs[agg_name]["value"] agg_value = meta["aggs"][agg_name]["value"]
# TODO: simplify this, match is default to True # TODO: simplify this, match is default to True
if operator == ">": if operator == ">":
@ -345,7 +346,7 @@ class ElasticsearchBackend(StorageBackend):
else: else:
# No aggregation found, but it is required # No aggregation found, but it is required
match = False match = False
result_map[index][0][agg_name]["match"] = match result_map[index][0]["aggs"][agg_name]["match"] = match
return result_map return result_map

10
core/db/processing.py
View File

@ -58,7 +58,7 @@ def annotate_results(results):
item["num_chans"] = num_chans[item["nick"]] item["num_chans"] = num_chans[item["nick"]]
def parse_results(results, aggs=None): def parse_results(results, meta=None):
results_parsed = [] results_parsed = []
stringify = ["host", "channel"] stringify = ["host", "channel"]
if "hits" in results.keys(): if "hits" in results.keys():
@ -110,13 +110,13 @@ def parse_results(results, aggs=None):
else: else:
element["time"] = time element["time"] = time
results_parsed.append(element) results_parsed.append(element)
if aggs: if meta:
aggregations = {} meta = {"aggs": {}}
if "aggregations" in results: if "aggregations" in results:
for field in ["avg_sentiment"]: # Add other number fields here for field in ["avg_sentiment"]: # Add other number fields here
if field in results["aggregations"]: if field in results["aggregations"]:
aggregations[field] = results["aggregations"][field] meta["aggs"][field] = results["aggregations"][field]
return (aggregations, results_parsed) return (meta, results_parsed)
return results_parsed return results_parsed

14
core/lib/rules.py
View File

@ -292,7 +292,7 @@ class NotificationRuleData(object):
if not response: if not response:
# No results in the result_map # No results in the result_map
self.rule_no_match() self.rule_no_match()
for index, (aggs, results) in response.items(): for index, (meta, results) in response.items():
if not results: if not results:
# Falsy results, no matches # Falsy results, no matches
self.rule_not_matched(index) self.rule_not_matched(index)
@ -300,15 +300,17 @@ class NotificationRuleData(object):
# Add the match values of all aggregations to a list # Add the match values of all aggregations to a list
aggs_for_index = [] aggs_for_index = []
for agg_name in self.aggs.keys(): for agg_name in self.aggs.keys():
if agg_name in aggs: if agg_name in meta["aggs"]:
if "match" in aggs[agg_name]: if "match" in meta["aggs"][agg_name]:
aggs_for_index.append(aggs[agg_name]["match"]) aggs_for_index.append(meta["aggs"][agg_name]["match"])
# All required aggs are present # All required aggs are present
if len(aggs_for_index) == len(self.aggs.keys()): if len(aggs_for_index) == len(self.aggs.keys()):
if all(aggs_for_index): if all(aggs_for_index):
# All aggs have matched # All aggs have matched
self.rule_matched(index, results[: self.object.amount], aggs) self.rule_matched(
index, results[: self.object.amount], meta["aggs"]
)
continue continue
# Default branch, since the happy path has a continue keyword # Default branch, since the happy path has a continue keyword
self.rule_no_match(index) self.rule_no_match(index)
@ -536,7 +538,7 @@ class NotificationRuleData(object):
try: try:
self.parsed = load(self.data, Loader=Loader) self.parsed = load(self.data, Loader=Loader)
except (ScannerError, ParserError) as e: except (ScannerError, ParserError) as e:
raise RuleParseError("data", f"Invalid YAML: {e}") raise RuleParseError(f"Invalid YAML: {e}", "data")
def __str__(self): def __str__(self):
""" """