From 2a1e6b3292e239705da9ba74c2207575de251a60 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 14 Jan 2023 16:36:22 +0000 Subject: [PATCH] Allow scheduling notification rules --- core/forms.py | 2 +- core/lib/rules.py | 41 ++++++++++++++++++- core/management/commands/scheduling.py | 39 ++++++++++++++++++ .../0017_alter_notificationrule_interval.py | 18 ++++++++ core/views/notifications.py | 14 ++++--- docker-compose.yml | 30 ++++++++++++++ requirements.txt | 1 + 7 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 core/management/commands/scheduling.py create mode 100644 core/migrations/0017_alter_notificationrule_interval.py diff --git a/core/forms.py b/core/forms.py index 97e3a7c..8a82032 100644 --- a/core/forms.py +++ b/core/forms.py @@ -100,7 +100,7 @@ class NotificationRuleForm(RestrictedFormMixin, ModelForm): "topic": "The topic to send notifications to. Leave blank for default.", "enabled": "Whether the rule is enabled.", "data": "The notification rule definition.", - "interval": "How often to run the search. On demand only evaluates messages as they are received.", + "interval": "How often to run the search. On demand evaluates messages as they are received, without running a scheduled search. The remaining options schedule a search of the database with the window below.", "window": "Time window to search: 1d, 1h, 1m, 1s, etc.", } diff --git a/core/lib/rules.py b/core/lib/rules.py index e90593d..0f750ef 100644 --- a/core/lib/rules.py +++ b/core/lib/rules.py @@ -16,6 +16,10 @@ from core.util import logs log = logs.get_logger("rules") +SECONDS_PER_UNIT = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800} + +MAX_WINDOW = 2592000 + class RuleParseError(Exception): def __init__(self, message, field): @@ -111,10 +115,19 @@ class NotificationRuleData(object): self.data = self.cleaned_data.get("data") self.parsed = None + self.validate_user_permissions() + self.parse_data() self.validate_permissions() self.validate_time_fields() + def validate_user_permissions(self): + """ + Ensure the user can use notification rules. + """ + if not self.user.has_perm("core.use_rules"): + raise RuleParseError("User does not have permission to use rules", "data") + def validate_time_fields(self): """ Validate the interval and window fields. @@ -122,11 +135,35 @@ class NotificationRuleData(object): """ interval = self.cleaned_data.get("interval") window = self.cleaned_data.get("window") - if interval == "ondemand" and window is not None: + if interval == 0 and window is not None: raise RuleParseError( - "Window cannot be specified with ondemand interval", "window" + "Window cannot be specified with on-demand interval", "window" ) + if interval is not None and window is None: + raise RuleParseError( + "Window must be specified with non-on-demand interval", "window" + ) + + if window is not None: + window_number = window[:-1] + if not window_number.isdigit(): + raise RuleParseError("Window prefix must be a number", "window") + window_number = int(window_number) + window_unit = window[-1] + if window_unit not in SECONDS_PER_UNIT: + raise RuleParseError( + f"Window unit must be one of {', '.join(SECONDS_PER_UNIT.keys())}, not '{window_unit}'", + "window", + ) + window_seconds = window_number * SECONDS_PER_UNIT[window_unit] + print("Window seconds", window_seconds) + if window_seconds > MAX_WINDOW: + raise RuleParseError( + f"Window cannot be larger than {MAX_WINDOW} seconds (30 days)", + "window", + ) + def validate_permissions(self): """ Validate permissions for the source and index variables. diff --git a/core/management/commands/scheduling.py b/core/management/commands/scheduling.py new file mode 100644 index 0000000..635651d --- /dev/null +++ b/core/management/commands/scheduling.py @@ -0,0 +1,39 @@ +from django.core.management.base import BaseCommand + +from core.util import logs +from core.models import NotificationRule +from core.db.storage import db +import aioschedule as schedule +import asyncio +from time import sleep + +log = logs.get_logger("scheduling") + +# INTERVAL_CHOICES = ( +# (0, "On demand"), +# (60, "Every minute"), +# (900, "Every 15 minutes"), +# (1800, "Every 30 minutes"), +# (3600, "Every hour"), +# (14400, "Every 4 hours"), +# (86400, "Every day"), +# ) + +INTERVALS = [60, 900, 1800, 3600, 14400, 86400] + +def run_schedule(interval_seconds): + print("Running schedule", interval_seconds) + matching_rules = NotificationRule.objects.filter( + enabled=True, interval=interval_seconds + ) + +class Command(BaseCommand): + def handle(self, *args, **options): + for interval in INTERVALS: + schedule.every(interval).seconds.do(run_schedule, interval_seconds=interval) + + loop = asyncio.get_event_loop() + + while True: + loop.run_until_complete(schedule.run_pending()) + sleep(10) \ No newline at end of file diff --git a/core/migrations/0017_alter_notificationrule_interval.py b/core/migrations/0017_alter_notificationrule_interval.py new file mode 100644 index 0000000..a5e15cf --- /dev/null +++ b/core/migrations/0017_alter_notificationrule_interval.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2023-01-14 14:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0016_notificationrule_interval_notificationrule_window'), + ] + + operations = [ + migrations.AlterField( + model_name='notificationrule', + name='interval', + field=models.IntegerField(choices=[(0, 'On demand'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=0), + ), + ] diff --git a/core/views/notifications.py b/core/views/notifications.py index 0275b36..2f95c62 100644 --- a/core/views/notifications.py +++ b/core/views/notifications.py @@ -1,4 +1,4 @@ -from django.contrib.auth.mixins import LoginRequiredMixin +from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin from core.forms import NotificationRuleForm, NotificationSettingsForm from core.models import NotificationRule, NotificationSettings @@ -7,7 +7,8 @@ from core.views.helpers import ObjectCreate, ObjectDelete, ObjectList, ObjectUpd # Notifications - we create a new notification settings object if there isn't one # Hence, there is only an update view, not a create view. -class NotificationsUpdate(LoginRequiredMixin, ObjectUpdate): +class NotificationsUpdate(LoginRequiredMixin, PermissionRequiredMixin, ObjectUpdate): + permission_required = "use_rules" model = NotificationSettings form_class = NotificationSettingsForm @@ -41,19 +42,22 @@ class RuleList(LoginRequiredMixin, ObjectList): submit_url_name = "rule_create" -class RuleCreate(LoginRequiredMixin, ObjectCreate): +class RuleCreate(LoginRequiredMixin, PermissionRequiredMixin, ObjectCreate): + permission_required = "use_rules" model = NotificationRule form_class = NotificationRuleForm submit_url_name = "rule_create" -class RuleUpdate(LoginRequiredMixin, ObjectUpdate): +class RuleUpdate(LoginRequiredMixin, PermissionRequiredMixin, ObjectUpdate): + permission_required = "use_rules" model = NotificationRule form_class = NotificationRuleForm submit_url_name = "rule_update" -class RuleDelete(LoginRequiredMixin, ObjectDelete): +class RuleDelete(LoginRequiredMixin, PermissionRequiredMixin, ObjectDelete): + permission_required = "use_rules" model = NotificationRule diff --git a/docker-compose.yml b/docker-compose.yml index 33464cb..b4a77e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,36 @@ services: networks: - default + scheduling: + image: pathogen/neptune:latest + container_name: scheduling_neptune + build: + context: . + args: + OPERATION: ${OPERATION} + command: sh -c '. /venv/bin/activate && python manage.py scheduling' + volumes: + - ${PORTAINER_GIT_DIR}:/code + - ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini + - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py + - ${APP_DATABASE_FILE}:/code/db.sqlite3 + - neptune_static:${STATIC_ROOT} + env_file: + - stack.env + volumes_from: + - tmp + depends_on: + redis: + condition: service_healthy + migration: + condition: service_started + collectstatic: + condition: service_started + networks: + - default + - pathogen + - elastic + migration: image: pathogen/neptune:latest container_name: migration_neptune diff --git a/requirements.txt b/requirements.txt index 0d27460..69a3dad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ django-debug-toolbar django-debug-toolbar-template-profiler orjson msgpack +aioschedule