From f93d37d1c0d18826ee84b5a3f87383ec54bf95d1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 12 Jan 2023 07:20:43 +0000 Subject: [PATCH] Implement notification rules and settings --- app/urls.py | 28 +++++++- core/db/__init__.py | 68 +++++++++++++++--- core/db/druid.py | 5 +- core/db/elastic.py | 5 +- core/db/manticore.py | 7 +- core/forms.py | 54 ++++++++++++++- core/lib/rules.py | 60 ++++++++++++++++ core/migrations/0012_notificationrule.py | 25 +++++++ core/migrations/0013_notificationsettings.py | 24 +++++++ core/models.py | 19 +++++ core/templates/base.html | 20 +++++- core/templates/partials/rule-list.html | 69 +++++++++++++++++++ .../templates/window-content/object-form.html | 34 +++++++++ core/templates/window-content/object.html | 45 ++++++++++++ core/templates/window-content/objects.html | 45 ++++++++++++ core/views/helpers.py | 49 ------------- core/views/notifications.py | 57 +++++++++++++++ core/views/ui/drilldown.py | 8 +-- 18 files changed, 545 insertions(+), 77 deletions(-) create mode 100644 core/lib/rules.py create mode 100644 core/migrations/0012_notificationrule.py create mode 100644 core/migrations/0013_notificationsettings.py create mode 100644 core/templates/partials/rule-list.html create mode 100644 core/templates/window-content/object-form.html create mode 100644 core/templates/window-content/object.html create mode 100644 core/templates/window-content/objects.html create mode 100644 core/views/notifications.py diff --git a/app/urls.py b/app/urls.py index a930b1b..4a6b3be 100644 --- a/app/urls.py +++ b/app/urls.py @@ -19,8 +19,9 @@ from django.contrib import admin from django.urls import include, path from django.views.generic import TemplateView +# Notification settings and rules # Threshold API stuff -from core.views import About, Billing, Cancel, Order, Portal, Signup +from core.views import About, Billing, Cancel, Order, Portal, Signup, notifications from core.views.callbacks import Callback from core.views.manage.threshold.irc import ( ThresholdIRCNetworkList, # Actions and just get list output @@ -261,4 +262,29 @@ urlpatterns = [ name="threshold_irc_msg", ), ## + path( + "notifications//update/", + notifications.NotificationsUpdate.as_view(), + name="notifications_update", + ), + path( + "rules//", + notifications.RuleList.as_view(), + name="rules", + ), + path( + "rule//create/", + notifications.RuleCreate.as_view(), + name="rule_create", + ), + path( + "rule//update//", + notifications.RuleUpdate.as_view(), + name="rule_update", + ), + path( + "rule//delete//", + notifications.RuleDelete.as_view(), + name="rule_delete", + ), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/core/db/__init__.py b/core/db/__init__.py index 4a45e9d..2ad08cb 100644 --- a/core/db/__init__.py +++ b/core/db/__init__.py @@ -12,7 +12,46 @@ from siphashc import siphash from core import r from core.db.processing import annotate_results from core.util import logs -from core.views import helpers + + +def remove_defaults(query_params): + for field, value in list(query_params.items()): + if field in settings.DRILLDOWN_DEFAULT_PARAMS: + if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]: + del query_params[field] + + +def add_defaults(query_params): + for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items(): + if field not in query_params: + query_params[field] = value + + +def dedup_list(data, check_keys): + """ + Remove duplicate dictionaries from list. + """ + seen = set() + out = [] + + dup_count = 0 + for x in data: + dedupeKey = tuple(x[k] for k in check_keys if k in x) + if dedupeKey in seen: + dup_count += 1 + continue + if dup_count > 0: + out.append({"type": "control", "hidden": dup_count}) + dup_count = 0 + out.append(x) + seen.add(dedupeKey) + if dup_count > 0: + out.append({"type": "control", "hidden": dup_count}) + return out + + +class QueryError(Exception): + pass class StorageBackend(ABC): @@ -60,14 +99,16 @@ class StorageBackend(ABC): return size - def parse_index(self, user, query_params): + def parse_index(self, user, query_params, raise_error=False): if "index" in query_params: index = query_params["index"] if index == "main": index = settings.INDEX_MAIN else: if not user.has_perm(f"core.index_{index}"): - message = "Not permitted to search by this index" + message = f"Not permitted to search by this index: {index}" + if raise_error: + raise QueryError(message) message_class = "danger" return { "message": message, @@ -79,7 +120,9 @@ class StorageBackend(ABC): index = settings.INDEX_INT elif index == "restricted": if not user.has_perm("core.restricted_sources"): - message = "Not permitted to search by this index" + message = f"Not permitted to search by this index: {index}" + if raise_error: + raise QueryError(message) message_class = "danger" return { "message": message, @@ -87,7 +130,9 @@ class StorageBackend(ABC): } index = settings.INDEX_RESTRICTED else: - message = "Index is not valid." + message = f"Index is not valid: {index}" + if raise_error: + raise QueryError(message) message_class = "danger" return { "message": message, @@ -132,17 +177,22 @@ class StorageBackend(ABC): message_class = "warning" return {"message": message, "class": message_class} - def parse_source(self, user, query_params): + def parse_source(self, user, query_params, raise_error=False): + source = None if "source" in query_params: source = query_params["source"] if source in settings.SOURCES_RESTRICTED: if not user.has_perm("core.restricted_sources"): - message = "Access denied" + message = f"Access denied: {source}" + if raise_error: + raise QueryError(message) message_class = "danger" return {"message": message, "class": message_class} elif source not in settings.MAIN_SOURCES: - message = "Invalid source" + message = f"Invalid source: {source}" + if raise_error: + raise QueryError(message) message_class = "danger" return {"message": message, "class": message_class} @@ -333,7 +383,7 @@ class StorageBackend(ABC): dedup_fields = kwargs.get("dedup_fields") if not dedup_fields: dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"] - response = helpers.dedup_list(response, dedup_fields) + response = dedup_list(response, dedup_fields) return response @abstractmethod diff --git a/core/db/druid.py b/core/db/druid.py index 158f16f..2b7c797 100644 --- a/core/db/druid.py +++ b/core/db/druid.py @@ -4,9 +4,8 @@ import orjson import requests from django.conf import settings -from core.db import StorageBackend +from core.db import StorageBackend, add_defaults from core.db.processing import parse_druid -from core.views import helpers logger = logging.getLogger(__name__) @@ -135,7 +134,7 @@ class DruidBackend(StorageBackend): add_bool = [] add_in = {} - helpers.add_defaults(query_params) + add_defaults(query_params) # Now, run the helpers for SIQTSRSS/ADR # S - Size diff --git a/core/db/elastic.py b/core/db/elastic.py index c27a771..45165d0 100644 --- a/core/db/elastic.py +++ b/core/db/elastic.py @@ -5,12 +5,11 @@ from django.conf import settings from elasticsearch import Elasticsearch from elasticsearch.exceptions import NotFoundError, RequestError -from core.db import StorageBackend +from core.db import StorageBackend, add_defaults # from json import dumps # pp = lambda x: print(dumps(x, indent=2)) from core.db.processing import parse_results -from core.views import helpers class ElasticsearchBackend(StorageBackend): @@ -204,7 +203,7 @@ class ElasticsearchBackend(StorageBackend): add_top = [] add_top_negative = [] - helpers.add_defaults(query_params) + add_defaults(query_params) # Now, run the helpers for SIQTSRSS/ADR # S - Size diff --git a/core/db/manticore.py b/core/db/manticore.py index 0eee282..a3b9737 100644 --- a/core/db/manticore.py +++ b/core/db/manticore.py @@ -5,9 +5,8 @@ from pprint import pprint import requests from django.conf import settings -from core.db import StorageBackend +from core.db import StorageBackend, add_defaults, dedup_list from core.db.processing import annotate_results, parse_results -from core.views import helpers logger = logging.getLogger(__name__) @@ -67,7 +66,7 @@ class ManticoreBackend(StorageBackend): sort = None query_created = False source = None - helpers.add_defaults(query_params) + add_defaults(query_params) # Check size if request.user.is_anonymous: sizes = settings.MANTICORE_MAIN_SIZES_ANON @@ -292,7 +291,7 @@ class ManticoreBackend(StorageBackend): if dedup: if not dedup_fields: dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"] - results_parsed = helpers.dedup_list(results_parsed, dedup_fields) + results_parsed = dedup_list(results_parsed, dedup_fields) context = { "object_list": results_parsed, "card": results["hits"]["total"], diff --git a/core/forms.py b/core/forms.py index 99829e6..6cfb89b 100644 --- a/core/forms.py +++ b/core/forms.py @@ -1,8 +1,12 @@ from django import forms from django.contrib.auth.forms import UserCreationForm from django.core.exceptions import FieldDoesNotExist +from django.forms import ModelForm -from .models import User +from core.db import QueryError +from core.lib.rules import NotificationRuleData + +from .models import NotificationRule, NotificationSettings, User # from django.forms import ModelForm @@ -64,3 +68,51 @@ class CustomUserCreationForm(UserCreationForm): class Meta: model = User fields = "__all__" + + +class NotificationSettingsForm(RestrictedFormMixin, ModelForm): + class Meta: + model = NotificationSettings + fields = ( + "ntfy_topic", + "ntfy_url", + ) + help_texts = { + "ntfy_topic": "The topic to send notifications to.", + "ntfy_url": "Custom NTFY server. Leave blank to use the default server.", + } + + +class NotificationRuleForm(RestrictedFormMixin, ModelForm): + class Meta: + model = NotificationRule + fields = ( + "name", + "enabled", + "data", + ) + help_texts = { + "name": "The name of the rule.", + "enabled": "Whether the rule is enabled.", + "data": "The notification rule definition.", + } + + def clean(self): + cleaned_data = super(NotificationRuleForm, self).clean() + data = cleaned_data.get("data") + try: + parsed_data = NotificationRuleData(self.request.user, data) + except ValueError as e: + self.add_error("data", f"Parsing error: {e}") + return + except QueryError as e: + self.add_error("data", f"Query error: {e}") + return + + # Write back the validated data + # We need this to populate the index and source variable if + # they are not set + to_store = str(parsed_data) + cleaned_data["data"] = to_store + + return cleaned_data diff --git a/core/lib/rules.py b/core/lib/rules.py new file mode 100644 index 0000000..f3ba41f --- /dev/null +++ b/core/lib/rules.py @@ -0,0 +1,60 @@ +from core.db.storage import db +from yaml import load, dump +from yaml.scanner import ScannerError +from yaml.parser import ParserError +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +class NotificationRuleData(object): + def __init__(self, user, data): + self.user = user + self.data = data + self.parsed = None + + self.parse_data() + self.validate_permissions() + + def validate_permissions(self): + """ + Validate permissions for the source and index variables. + """ + if "index" in self.parsed: + index = self.parsed["index"] + if type(index) == list: + for i in index: + db.parse_index(self.user, {"index": i}, raise_error=True) + else: + db.parse_index(self.user, {"index": index}, raise_error=True) + else: + # Get the default value for the user if not present + index = db.parse_index(self.user, {}, raise_error=True) + self.parsed["index"] = index + + if "source" in self.parsed: + source = self.parsed["source"] + if type(source) == list: + for i in source: + db.parse_source(self.user, {"source": i}, raise_error=True) + else: + db.parse_source(self.user, {"source": source}, raise_error=True) + else: + # Get the default value for the user if not present + source = db.parse_source(self.user, {}, raise_error=True) + self.parsed["source"] = source + + def parse_data(self): + """ + Parse the data in the text field to YAML. + """ + try: + self.parsed = load(self.data, Loader=Loader) + except (ScannerError, ParserError) as e: + raise ValueError(f"Invalid YAML: {e}") + + def __str__(self): + return dump(self.parsed, Dumper=Dumper) + + def get_data(self): + return self.parsed \ No newline at end of file diff --git a/core/migrations/0012_notificationrule.py b/core/migrations/0012_notificationrule.py new file mode 100644 index 0000000..6aea12a --- /dev/null +++ b/core/migrations/0012_notificationrule.py @@ -0,0 +1,25 @@ +# Generated by Django 4.1.3 on 2023-01-12 15:12 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0011_alter_perms_options'), + ] + + operations = [ + migrations.CreateModel( + name='NotificationRule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('enabled', models.BooleanField(default=True)), + ('data', models.TextField()), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/core/migrations/0013_notificationsettings.py b/core/migrations/0013_notificationsettings.py new file mode 100644 index 0000000..4350968 --- /dev/null +++ b/core/migrations/0013_notificationsettings.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.3 on 2023-01-12 15:25 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_notificationrule'), + ] + + operations = [ + migrations.CreateModel( + name='NotificationSettings', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ntfy_topic', models.CharField(blank=True, max_length=255, null=True)), + ('ntfy_url', models.CharField(blank=True, max_length=255, null=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/core/models.py b/core/models.py index d5fd569..9281104 100644 --- a/core/models.py +++ b/core/models.py @@ -122,3 +122,22 @@ class Perms(models.Model): ("index_restricted", "Can use the restricted index"), ("restricted_sources", "Can access restricted sources"), ) + + +class NotificationRule(models.Model): + user = models.ForeignKey(User, on_delete=models.CASCADE) + name = models.CharField(max_length=255) + enabled = models.BooleanField(default=True) + data = models.TextField() + + def __str__(self): + return f"{self.user} - {self.rule}" + + +class NotificationSettings(models.Model): + user = models.OneToOneField(User, on_delete=models.CASCADE) + ntfy_topic = models.CharField(max_length=255, null=True, blank=True) + ntfy_url = models.CharField(max_length=255, null=True, blank=True) + + def __str__(self): + return f"Notification settings for {self.user}" diff --git a/core/templates/base.html b/core/templates/base.html index 57ba656..eef348d 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -249,10 +249,24 @@ Search + + Rules + {% if user.is_authenticated %} - - Billing - + {% endif %} {% if user.is_superuser %}