Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement notification rules and settings

This commit is contained in:
Mark Veidemanis 2023-01-12 07:20:43 +00:00
parent a70bc16d22
commit f93d37d1c0
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
18 changed files with 545 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/urls.py
View File

@ -19,8 +19,9 @@ from django.contrib import admin
from django.urls import include, path
from django.views.generic import TemplateView
# Notification settings and rules
# Threshold API stuff
from core.views import About, Billing, Cancel, Order, Portal, Signup
from core.views import About, Billing, Cancel, Order, Portal, Signup, notifications
from core.views.callbacks import Callback
from core.views.manage.threshold.irc import (
ThresholdIRCNetworkList, # Actions and just get list output
@ -261,4 +262,29 @@ urlpatterns = [
name="threshold_irc_msg",
),
##
path(
"notifications/<str:type>/update/",
notifications.NotificationsUpdate.as_view(),
name="notifications_update",
),
path(
"rules/<str:type>/",
notifications.RuleList.as_view(),
name="rules",
),
path(
"rule/<str:type>/create/",
notifications.RuleCreate.as_view(),
name="rule_create",
),
path(
"rule/<str:type>/update/<str:pk>/",
notifications.RuleUpdate.as_view(),
name="rule_update",
),
path(
"rule/<str:type>/delete/<str:pk>/",
notifications.RuleDelete.as_view(),
name="rule_delete",
),
] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

68
core/db/__init__.py
View File

@ -12,7 +12,46 @@ from siphashc import siphash
from core import r
from core.db.processing import annotate_results
from core.util import logs
from core.views import helpers
def remove_defaults(query_params):
for field, value in list(query_params.items()):
if field in settings.DRILLDOWN_DEFAULT_PARAMS:
if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
del query_params[field]
def add_defaults(query_params):
for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
if field not in query_params:
query_params[field] = value
def dedup_list(data, check_keys):
"""
Remove duplicate dictionaries from list.
"""
seen = set()
out = []
dup_count = 0
for x in data:
dedupeKey = tuple(x[k] for k in check_keys if k in x)
if dedupeKey in seen:
dup_count += 1
continue
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
dup_count = 0
out.append(x)
seen.add(dedupeKey)
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
return out
class QueryError(Exception):
pass
class StorageBackend(ABC):
@ -60,14 +99,16 @@ class StorageBackend(ABC):
return size
def parse_index(self, user, query_params):
def parse_index(self, user, query_params, raise_error=False):
if "index" in query_params:
index = query_params["index"]
if index == "main":
index = settings.INDEX_MAIN
else:
if not user.has_perm(f"core.index_{index}"):
message = "Not permitted to search by this index"
message = f"Not permitted to search by this index: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
@ -79,7 +120,9 @@ class StorageBackend(ABC):
index = settings.INDEX_INT
elif index == "restricted":
if not user.has_perm("core.restricted_sources"):
message = "Not permitted to search by this index"
message = f"Not permitted to search by this index: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
@ -87,7 +130,9 @@ class StorageBackend(ABC):
}
index = settings.INDEX_RESTRICTED
else:
message = "Index is not valid."
message = f"Index is not valid: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
@ -132,17 +177,22 @@ class StorageBackend(ABC):
message_class = "warning"
return {"message": message, "class": message_class}
def parse_source(self, user, query_params):
def parse_source(self, user, query_params, raise_error=False):
source = None
if "source" in query_params:
source = query_params["source"]
if source in settings.SOURCES_RESTRICTED:
if not user.has_perm("core.restricted_sources"):
message = "Access denied"
message = f"Access denied: {source}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {"message": message, "class": message_class}
elif source not in settings.MAIN_SOURCES:
message = "Invalid source"
message = f"Invalid source: {source}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {"message": message, "class": message_class}
@ -333,7 +383,7 @@ class StorageBackend(ABC):
dedup_fields = kwargs.get("dedup_fields")
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
response = helpers.dedup_list(response, dedup_fields)
response = dedup_list(response, dedup_fields)
return response
@abstractmethod

5
core/db/druid.py
View File

@ -4,9 +4,8 @@ import orjson
import requests
from django.conf import settings
from core.db import StorageBackend
from core.db import StorageBackend, add_defaults
from core.db.processing import parse_druid
from core.views import helpers
logger = logging.getLogger(__name__)
@ -135,7 +134,7 @@ class DruidBackend(StorageBackend):
add_bool = []
add_in = {}
helpers.add_defaults(query_params)
add_defaults(query_params)
# Now, run the helpers for SIQTSRSS/ADR
# S - Size

5
core/db/elastic.py
View File

@ -5,12 +5,11 @@ from django.conf import settings
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError, RequestError
from core.db import StorageBackend
from core.db import StorageBackend, add_defaults
# from json import dumps
# pp = lambda x: print(dumps(x, indent=2))
from core.db.processing import parse_results
from core.views import helpers
class ElasticsearchBackend(StorageBackend):
@ -204,7 +203,7 @@ class ElasticsearchBackend(StorageBackend):
add_top = []
add_top_negative = []
helpers.add_defaults(query_params)
add_defaults(query_params)
# Now, run the helpers for SIQTSRSS/ADR
# S - Size

7
core/db/manticore.py
View File

@ -5,9 +5,8 @@ from pprint import pprint
import requests
from django.conf import settings
from core.db import StorageBackend
from core.db import StorageBackend, add_defaults, dedup_list
from core.db.processing import annotate_results, parse_results
from core.views import helpers
logger = logging.getLogger(__name__)
@ -67,7 +66,7 @@ class ManticoreBackend(StorageBackend):
sort = None
query_created = False
source = None
helpers.add_defaults(query_params)
add_defaults(query_params)
# Check size
if request.user.is_anonymous:
sizes = settings.MANTICORE_MAIN_SIZES_ANON
@ -292,7 +291,7 @@ class ManticoreBackend(StorageBackend):
if dedup:
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
results_parsed = helpers.dedup_list(results_parsed, dedup_fields)
results_parsed = dedup_list(results_parsed, dedup_fields)
context = {
"object_list": results_parsed,
"card": results["hits"]["total"],

54
core/forms.py
View File

@ -1,8 +1,12 @@
from django import forms
from django.contrib.auth.forms import UserCreationForm
from django.core.exceptions import FieldDoesNotExist
from django.forms import ModelForm
from .models import User
from core.db import QueryError
from core.lib.rules import NotificationRuleData
from .models import NotificationRule, NotificationSettings, User
# from django.forms import ModelForm
@ -64,3 +68,51 @@ class CustomUserCreationForm(UserCreationForm):
class Meta:
model = User
fields = "__all__"
class NotificationSettingsForm(RestrictedFormMixin, ModelForm):
class Meta:
model = NotificationSettings
fields = (
"ntfy_topic",
"ntfy_url",
)
help_texts = {
"ntfy_topic": "The topic to send notifications to.",
"ntfy_url": "Custom NTFY server. Leave blank to use the default server.",
}
class NotificationRuleForm(RestrictedFormMixin, ModelForm):
class Meta:
model = NotificationRule
fields = (
"name",
"enabled",
"data",
)
help_texts = {
"name": "The name of the rule.",
"enabled": "Whether the rule is enabled.",
"data": "The notification rule definition.",
}
def clean(self):
cleaned_data = super(NotificationRuleForm, self).clean()
data = cleaned_data.get("data")
try:
parsed_data = NotificationRuleData(self.request.user, data)
except ValueError as e:
self.add_error("data", f"Parsing error: {e}")
return
except QueryError as e:
self.add_error("data", f"Query error: {e}")
return
# Write back the validated data
# We need this to populate the index and source variable if
# they are not set
to_store = str(parsed_data)
cleaned_data["data"] = to_store
return cleaned_data

60
core/lib/rules.py Normal file
View File

@ -0,0 +1,60 @@
from core.db.storage import db
from yaml import load, dump
from yaml.scanner import ScannerError
from yaml.parser import ParserError
try:
from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
from yaml import Loader, Dumper
class NotificationRuleData(object):
def __init__(self, user, data):
self.user = user
self.data = data
self.parsed = None
self.parse_data()
self.validate_permissions()
def validate_permissions(self):
"""
Validate permissions for the source and index variables.
"""
if "index" in self.parsed:
index = self.parsed["index"]
if type(index) == list:
for i in index:
db.parse_index(self.user, {"index": i}, raise_error=True)
else:
db.parse_index(self.user, {"index": index}, raise_error=True)
else:
# Get the default value for the user if not present
index = db.parse_index(self.user, {}, raise_error=True)
self.parsed["index"] = index
if "source" in self.parsed:
source = self.parsed["source"]
if type(source) == list:
for i in source:
db.parse_source(self.user, {"source": i}, raise_error=True)
else:
db.parse_source(self.user, {"source": source}, raise_error=True)
else:
# Get the default value for the user if not present
source = db.parse_source(self.user, {}, raise_error=True)
self.parsed["source"] = source
def parse_data(self):
"""
Parse the data in the text field to YAML.
"""
try:
self.parsed = load(self.data, Loader=Loader)
except (ScannerError, ParserError) as e:
raise ValueError(f"Invalid YAML: {e}")
def __str__(self):
return dump(self.parsed, Dumper=Dumper)
def get_data(self):
return self.parsed

25
core/migrations/0012_notificationrule.py Normal file
View File

@ -0,0 +1,25 @@
# Generated by Django 4.1.3 on 2023-01-12 15:12
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0011_alter_perms_options'),
]
operations = [
migrations.CreateModel(
name='NotificationRule',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=255)),
('enabled', models.BooleanField(default=True)),
('data', models.TextField()),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
),
]

24
core/migrations/0013_notificationsettings.py Normal file
View File

@ -0,0 +1,24 @@
# Generated by Django 4.1.3 on 2023-01-12 15:25
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0012_notificationrule'),
]
operations = [
migrations.CreateModel(
name='NotificationSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('ntfy_topic', models.CharField(blank=True, max_length=255, null=True)),
('ntfy_url', models.CharField(blank=True, max_length=255, null=True)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
),
]

19
core/models.py
View File

@ -122,3 +122,22 @@ class Perms(models.Model):
("index_restricted", "Can use the restricted index"),
("restricted_sources", "Can access restricted sources"),
)
class NotificationRule(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE)
name = models.CharField(max_length=255)
enabled = models.BooleanField(default=True)
data = models.TextField()
def __str__(self):
return f"{self.user} - {self.rule}"
class NotificationSettings(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
ntfy_topic = models.CharField(max_length=255, null=True, blank=True)
ntfy_url = models.CharField(max_length=255, null=True, blank=True)
def __str__(self):
return f"Notification settings for {self.user}"

20
core/templates/base.html
View File

@ -249,10 +249,24 @@
<a class="navbar-item" href="{% url 'home' %}">
Search
</a>
<a class="navbar-item" href="{% url 'rules' type='page' %}">
Rules
</a>
{% if user.is_authenticated %}
<a class="navbar-item" href="{% url 'billing' %}">
Billing
</a>
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link">
Account
</a>
<div class="navbar-dropdown">
<a class="navbar-item" href="{% url 'billing' %}">
Billing
</a>
<a class="navbar-item" href="{% url 'notifications_update' type='page' %}">
Notifications
</a>
</div>
</div>
{% endif %}
{% if user.is_superuser %}
<div class="navbar-item has-dropdown is-hoverable">

69
core/templates/partials/rule-list.html Normal file
View File

@ -0,0 +1,69 @@
{% include 'partials/notify.html' %}
<table
class="table is-fullwidth is-hoverable"
hx-target="#{{ context_object_name }}-table"
id="{{ context_object_name }}-table"
hx-swap="outerHTML"
hx-trigger="{{ context_object_name_singular }}Event from:body"
hx-get="{{ list_url }}">
<thead>
<th>id</th>
<th>user</th>
<th>name</th>
<th>enabled</th>
<th>data length</th>
<th>actions</th>
</thead>
{% for item in object_list %}
<tr>
<td>{{ item.id }}</td>
<td>{{ item.user }}</td>
<td>{{ item.name }}</td>
<td>
{% if item.enabled %}
<span class="icon">
<i class="fa-solid fa-check"></i>
</span>
{% else %}
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
{% endif %}
</td>
<td>{{ item.data|length }}</td>
<td>
<div class="buttons">
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'rule_update' type=type pk=item.id %}"
hx-trigger="click"
hx-target="#{{ type }}s-here"
hx-swap="innerHTML"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-pencil"></i>
</span>
</span>
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-delete="{% url 'rule_delete' type=type pk=item.id %}"
hx-trigger="click"
hx-target="#modals-here"
hx-swap="innerHTML"
hx-confirm="Are you sure you wish to delete {{ item.name }}?"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
</span>
</button>
</div>
</td>
</tr>
{% endfor %}
</table>

34
core/templates/window-content/object-form.html Normal file
View File

@ -0,0 +1,34 @@
{% include 'partials/notify.html' %}
{% if page_title is not None %}
<h1 class="title is-4">{{ page_title }}</h1>
{% endif %}
{% if page_subtitle is not None %}
<h1 class="subtitle">{{ page_subtitle }}</h1>
{% endif %}
{% load crispy_forms_tags %}
{% load crispy_forms_bulma_field %}
<form
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{{ submit_url }}"
hx-target="#modals-here"
hx-swap="innerHTML">
{% csrf_token %}
{{ form|crispy }}
{% if hide_cancel is not True %}
<button
type="button"
class="button is-light modal-close-button">
Cancel
</button>
{% endif %}
<button type="submit" class="button modal-close-button">Submit</button>
</form>

45
core/templates/window-content/object.html Normal file
View File

@ -0,0 +1,45 @@
{% include 'partials/notify.html' %}
{% if page_title is not None %}
<h1 class="title is-4">{{ page_title }}</h1>
{% endif %}
{% if page_subtitle is not None %}
<h1 class="subtitle">{{ page_subtitle }}</h1>
{% endif %}
<div class="buttons">
{% if submit_url is not None %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{{ submit_url }}"
hx-trigger="click"
hx-target="#modals-here"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-plus"></i>
</span>
<span>{{ title_singular }}</span>
</span>
</button>
{% endif %}
{% if delete_all_url is not None %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-delete="{{ delete_all_url }}"
hx-trigger="click"
hx-target="#modals-here"
hx-swap="innerHTML"
hx-confirm="Are you sure you wish to delete all {{ context_object_name }}?"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
<span>Delete all {{ context_object_name }} </span>
</span>
</button>
{% endif %}
</div>
{% include detail_template %}

45
core/templates/window-content/objects.html Normal file
View File

@ -0,0 +1,45 @@
{% include 'partials/notify.html' %}
{% if page_title is not None %}
<h1 class="title is-4">{{ page_title }}</h1>
{% endif %}
{% if page_subtitle is not None %}
<h1 class="subtitle">{{ page_subtitle }}</h1>
{% endif %}
<div class="buttons">
{% if submit_url is not None %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{{ submit_url }}"
hx-trigger="click"
hx-target="#modals-here"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-plus"></i>
</span>
<span>{{ title_singular }}</span>
</span>
</button>
{% endif %}
{% if delete_all_url is not None %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-delete="{{ delete_all_url }}"
hx-trigger="click"
hx-target="#modals-here"
hx-swap="innerHTML"
hx-confirm="Are you sure you wish to delete all {{ context_object_name }}?"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
<span>Delete all {{ context_object_name }} </span>
</span>
</button>
{% endif %}
</div>
{% include list_template %}

49
core/views/helpers.py
View File

@ -11,7 +11,6 @@
import uuid
# from core import r
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.paginator import Paginator
from django.db.models import QuerySet
@ -490,54 +489,6 @@ class ObjectDelete(RestrictedViewMixin, ObjectNameMixin, DeleteView):
return response
class SearchDenied:
def __init__(self, key, value):
self.key = key
self.value = value
class LookupDenied:
def __init__(self, key, value):
self.key = key
self.value = value
def remove_defaults(query_params):
for field, value in list(query_params.items()):
if field in settings.DRILLDOWN_DEFAULT_PARAMS:
if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
del query_params[field]
def add_defaults(query_params):
for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
if field not in query_params:
query_params[field] = value
def dedup_list(data, check_keys):
"""
Remove duplicate dictionaries from list.
"""
seen = set()
out = []
dup_count = 0
for x in data:
dedupeKey = tuple(x[k] for k in check_keys if k in x)
if dedupeKey in seen:
dup_count += 1
continue
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
dup_count = 0
out.append(x)
seen.add(dedupeKey)
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
return out
# from random import randint
# from timeit import timeit
# entries = 10000

57
core/views/notifications.py Normal file
View File

@ -0,0 +1,57 @@
from django.contrib.auth.mixins import LoginRequiredMixin
from core.forms import NotificationRuleForm, NotificationSettingsForm
from core.models import NotificationRule, NotificationSettings
from core.views.helpers import ObjectCreate, ObjectList, ObjectUpdate, ObjectDelete
# Notifications - we create a new notification settings object if there isn't one
# Hence, there is only an update view, not a create view.
class NotificationsUpdate(LoginRequiredMixin, ObjectUpdate):
model = NotificationSettings
form_class = NotificationSettingsForm
page_title = "Update your notification settings"
page_subtitle = (
"At least the topic must be set if you want to receive notifications."
)
submit_url_name = "notifications_update"
submit_url_args = ["type"]
pk_required = False
hide_cancel = True
def get_object(self, **kwargs):
notification_settings, _ = NotificationSettings.objects.get_or_create(
user=self.request.user
)
return notification_settings
class RuleList(LoginRequiredMixin, ObjectList):
list_template = "partials/rule-list.html"
model = NotificationRule
page_title = "List of notification rules."
list_url_name = "rules"
list_url_args = ["type"]
submit_url_name = "rule_create"
class RuleCreate(LoginRequiredMixin, ObjectCreate):
model = NotificationRule
form_class = NotificationRuleForm
submit_url_name = "rule_create"
class RuleUpdate(LoginRequiredMixin, ObjectUpdate):
model = NotificationRule
form_class = NotificationRuleForm
submit_url_name = "rule_update"
class RuleDelete(LoginRequiredMixin, ObjectDelete):
model = NotificationRule

8
core/views/ui/drilldown.py
View File

@ -10,6 +10,7 @@ from django_tables2 import SingleTableView
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.db import add_defaults, remove_defaults
from core.db.storage import db
from core.lib.threshold import (
annotate_num_chans,
@ -17,7 +18,6 @@ from core.lib.threshold import (
get_chans,
get_users,
)
from core.views import helpers
from core.views.ui.tables import DrilldownTable
# from copy import deepcopy
@ -125,7 +125,7 @@ class DrilldownTableView(SingleTableView):
# No query, this is a fresh page load
# Don't try to search, since there's clearly nothing to do
params_with_defaults = {}
helpers.add_defaults(params_with_defaults)
add_defaults(params_with_defaults)
context = {
"sizes": sizes,
"params": params_with_defaults,
@ -194,12 +194,12 @@ class DrilldownTableView(SingleTableView):
# Add any default parameters to the context
params_with_defaults = dict(query_params)
helpers.add_defaults(params_with_defaults)
add_defaults(params_with_defaults)
context["params"] = params_with_defaults
# Remove anything that we or the user set to a default for
# pretty URLs
helpers.remove_defaults(query_params)
remove_defaults(query_params)
url_params = urllib.parse.urlencode(query_params)
context["client_uri"] = url_params