XF/neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement notification rules and settings

Browse Source
This commit is contained in:
Mark Veidemanis
2023-01-12 07:20:43 +00:00
parent a70bc16d22
commit f93d37d1c0
18 changed files with 545 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
core/db/__init__.py
View File

@@ -12,7 +12,46 @@ from siphashc import siphash
from core import r
from core.db.processing import annotate_results
from core.util import logs
from core.views import helpers
def remove_defaults(query_params):
for field, value in list(query_params.items()):
if field in settings.DRILLDOWN_DEFAULT_PARAMS:
if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
del query_params[field]
def add_defaults(query_params):
for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
if field not in query_params:
query_params[field] = value
def dedup_list(data, check_keys):
"""
Remove duplicate dictionaries from list.
"""
seen = set()
out = []
dup_count = 0
for x in data:
dedupeKey = tuple(x[k] for k in check_keys if k in x)
if dedupeKey in seen:
dup_count += 1
continue
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
dup_count = 0
out.append(x)
seen.add(dedupeKey)
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
return out
class QueryError(Exception):
pass
class StorageBackend(ABC):
@@ -60,14 +99,16 @@ class StorageBackend(ABC):
return size
def parse_index(self, user, query_params):
def parse_index(self, user, query_params, raise_error=False):
if "index" in query_params:
index = query_params["index"]
if index == "main":
index = settings.INDEX_MAIN
else:
if not user.has_perm(f"core.index_{index}"):
message = "Not permitted to search by this index"
message = f"Not permitted to search by this index: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
@@ -79,7 +120,9 @@ class StorageBackend(ABC):
index = settings.INDEX_INT
elif index == "restricted":
if not user.has_perm("core.restricted_sources"):
message = "Not permitted to search by this index"
message = f"Not permitted to search by this index: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
@@ -87,7 +130,9 @@ class StorageBackend(ABC):
}
index = settings.INDEX_RESTRICTED
else:
message = "Index is not valid."
message = f"Index is not valid: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
@@ -132,17 +177,22 @@ class StorageBackend(ABC):
message_class = "warning"
return {"message": message, "class": message_class}
def parse_source(self, user, query_params):
def parse_source(self, user, query_params, raise_error=False):
source = None
if "source" in query_params:
source = query_params["source"]
if source in settings.SOURCES_RESTRICTED:
if not user.has_perm("core.restricted_sources"):
message = "Access denied"
message = f"Access denied: {source}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {"message": message, "class": message_class}
elif source not in settings.MAIN_SOURCES:
message = "Invalid source"
message = f"Invalid source: {source}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {"message": message, "class": message_class}
@@ -333,7 +383,7 @@ class StorageBackend(ABC):
dedup_fields = kwargs.get("dedup_fields")
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
response = helpers.dedup_list(response, dedup_fields)
response = dedup_list(response, dedup_fields)
return response
@abstractmethod

5
core/db/druid.py
View File

@@ -4,9 +4,8 @@ import orjson
import requests
from django.conf import settings
from core.db import StorageBackend
from core.db import StorageBackend, add_defaults
from core.db.processing import parse_druid
from core.views import helpers
logger = logging.getLogger(__name__)
@@ -135,7 +134,7 @@ class DruidBackend(StorageBackend):
add_bool = []
add_in = {}
helpers.add_defaults(query_params)
add_defaults(query_params)
# Now, run the helpers for SIQTSRSS/ADR
# S - Size

5
core/db/elastic.py
View File

@@ -5,12 +5,11 @@ from django.conf import settings
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError, RequestError
from core.db import StorageBackend
from core.db import StorageBackend, add_defaults
# from json import dumps
# pp = lambda x: print(dumps(x, indent=2))
from core.db.processing import parse_results
from core.views import helpers
class ElasticsearchBackend(StorageBackend):
@@ -204,7 +203,7 @@ class ElasticsearchBackend(StorageBackend):
add_top = []
add_top_negative = []
helpers.add_defaults(query_params)
add_defaults(query_params)
# Now, run the helpers for SIQTSRSS/ADR
# S - Size

7
core/db/manticore.py
View File

@@ -5,9 +5,8 @@ from pprint import pprint
import requests
from django.conf import settings
from core.db import StorageBackend
from core.db import StorageBackend, add_defaults, dedup_list
from core.db.processing import annotate_results, parse_results
from core.views import helpers
logger = logging.getLogger(__name__)
@@ -67,7 +66,7 @@ class ManticoreBackend(StorageBackend):
sort = None
query_created = False
source = None
helpers.add_defaults(query_params)
add_defaults(query_params)
# Check size
if request.user.is_anonymous:
sizes = settings.MANTICORE_MAIN_SIZES_ANON
@@ -292,7 +291,7 @@ class ManticoreBackend(StorageBackend):
if dedup:
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
results_parsed = helpers.dedup_list(results_parsed, dedup_fields)
results_parsed = dedup_list(results_parsed, dedup_fields)
context = {
"object_list": results_parsed,
"card": results["hits"]["total"],