Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: Pathogen:master
Branches Tags
Pathogen:master
Pathogen:modern-tables
Pathogen:prod
..
compare: Pathogen:modern-tables
Branches Tags
Pathogen:master
Pathogen:modern-tables
Pathogen:prod

No commits in common. "master" and "modern-tables" have entirely different histories.
master ... modern-tab

130 changed files with 2201 additions and 8427 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -154,5 +154,4 @@ cython_debug/
.idea/
.bash_history
.python_history
.vscode/

22
.pre-commit-config.yaml
View File

@ -1,30 +1,22 @@
repos:
- repo: https://github.com/psf/black
rev: 23.1.0
rev: 22.6.0
hooks:
- id: black
exclude: ^core/migrations
- repo: https://github.com/PyCQA/isort
rev: 5.11.5
rev: 5.10.1
hooks:
- id: isort
args: ["--profile", "black"]
- repo: https://github.com/PyCQA/flake8
rev: 6.0.0
rev: 4.0.1
hooks:
- id: flake8
args: [--max-line-length=88]
exclude: ^core/migrations
- repo: https://github.com/rtts/djhtml
rev: v2.0.0
- repo: https://github.com/thibaudcolas/curlylint
rev: v0.13.1
hooks:
- id: djhtml
args: [-t 2]
- id: djcss
exclude : ^core/static/css # slow
- id: djjs
exclude: ^core/static/js # slow
- repo: https://github.com/sirwart/ripsecrets.git
rev: v0.1.5
hooks:
- id: ripsecrets
- id: curlylint
files: \.(html|sls)$

28
Dockerfile
View File

@ -1,28 +0,0 @@
# syntax=docker/dockerfile:1
FROM python:3
ARG OPERATION
RUN useradd -d /code pathogen
RUN mkdir -p /code
RUN chown -R pathogen:pathogen /code
RUN mkdir -p /conf/static
RUN chown -R pathogen:pathogen /conf
RUN mkdir /venv
RUN chown pathogen:pathogen /venv
USER pathogen
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
WORKDIR /code
COPY requirements.txt /code/
RUN python -m venv /venv
RUN . /venv/bin/activate && pip install -r requirements.txt
# CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini
CMD if [ "$OPERATION" = "uwsgi" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi
# CMD . /venv/bin/activate && uvicorn --reload --reload-include *.html --workers 2 --uds /var/run/socks/app.sock app.asgi:application
# CMD . /venv/bin/activate && gunicorn -b 0.0.0.0:8000 --reload app.asgi:application -k uvicorn.workers.UvicornWorker

20
Makefile
View File

@ -1,20 +0,0 @@
run:
docker-compose --env-file=stack.env up -d
build:
docker-compose --env-file=stack.env build
stop:
docker-compose --env-file=stack.env down
log:
docker-compose --env-file=stack.env logs -f
migrate:
docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py migrate"
makemigrations:
docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py makemigrations"
auth:
docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py createsuperuser"

97
app/local_settings.example.py
View File

@ -1,68 +1,29 @@
# Elasticsearch settings
ELASTICSEARCH_URL = "10.1.0.1"
ELASTICSEARCH_PORT = 9200
ELASTICSEARCH_TLS = True
ELASTICSEARCH_USERNAME = "admin"
ELASTICSEARCH_PASSWORD = "secret"
# Secret key
SECRET_KEY = ""
# Manticore settings
MANTICORE_URL = "http://example-db-1:9308"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DB_BACKEND = "ELASTICSEARCH"
# OpenSearch settings
OPENSEARCH_URL = "127.0.0.1"
OPENSEARCH_PORT = 9200
OPENSEARCH_TLS = True
OPENSEARCH_USERNAME = "opensearch_user1"
OPENSEARCH_PASSWORD = "hunter2"
# Common DB settings
INDEX_MAIN = "main"
INDEX_RESTRICTED = "restricted"
INDEX_META = "meta"
INDEX_INT = "internal"
INDEX_RULE_STORAGE = "rule_storage"
OPENSEARCH_INDEX_MAIN = "main"
OPENSEARCH_INDEX_META = "meta"
MAIN_SIZES = ["1", "5", "15", "30", "50", "100", "250", "500", "1000"]
MAIN_SIZES_ANON = ["1", "5", "15", "30", "50", "100"]
MAIN_SOURCES = ["dis", "4ch", "all"]
SOURCES_RESTRICTED = ["irc"]
CACHE = False
CACHE_TIMEOUT = 2
OPENSEARCH_MAIN_SEARCH_FIELDS = ["msg", "nick", "host", "ident"]
OPENSEARCH_MAIN_SIZES = ["5", "10", "15", "20", "50", "100", "200"]
OPENSEARCH_MAIN_TIMESCALES = ["minute", "hour", "day", "week", "month", "6months"]
DRILLDOWN_RESULTS_PER_PAGE = 15
DRILLDOWN_DEFAULT_PARAMS = {
"size": "15",
"index": "main",
"sorting": "desc",
"source": "all",
OPENSEARCH_BLACKLISTED = {
"msg": ["example.com"],
"nick": ["me"],
}
# Encryption
# ENCRYPTION = False
# ENCRYPTION_KEY = b""
# Hashing
# HASHING = True
# HASHING_KEY = "xxx"
# Obfuscation
# OBFUSCATION = True
# # Fields obfuscate based on separators
# OBFUSCATE_FIELDS_SEP = ["date", "time"]
# # Fields to obfuscate based on length
# OBFUSCATE_FIELDS = ["ts"]
# OBFUSCATE_KEEP_RATIO = 0.9
# # DON'T obfuscate the last X fields of values separates by dashes
# OBFUSCATE_DASH_NUM = 2
# # DON'T obfuscate the last X fields of values separates by colons
# OBFUSCATE_COLON_NUM = 1
# SEARCH_FIELDS_DENY = ["ts", "date", "time"]
# DELAY_RESULTS = True
# # Delay results by this many days
# DELAY_DURATION = 10
ELASTICSEARCH_BLACKLISTED = {}
# URLs\
# URLs
DOMAIN = "example.com"
URL = f"https://{DOMAIN}"
@ -74,23 +35,23 @@ CSRF_TRUSTED_ORIGINS = [URL]
# Stripe
STRIPE_TEST = True
STRIPE_API_KEY_TEST = ""
STRIPE_PUBLIC_API_KEY_TEST = ""
STRIPE_API_KEY_TEST = "sk_test_xxx"
STRIPE_PUBLIC_API_KEY_TEST = "pk_test_xxx"
STRIPE_API_KEY_PROD = ""
STRIPE_PUBLIC_API_KEY_PROD = ""
STRIPE_API_KEY_PROD = "sk_prod_xxx"
STRIPE_PUBLIC_API_KEY_PROD = "pk_prod_xxx"
STRIPE_ENDPOINT_SECRET = ""
STATIC_ROOT = ""
SECRET_KEY = "a"
STRIPE_ADMIN_COUPON = ""
STRIPE_ADMIN_COUPON = "promo"
# Threshold
THRESHOLD_ENDPOINT = "http://threshold:13869"
THRESHOLD_API_KEY = "api_1"
THRESHOLD_API_TOKEN = ""
THRESHOLD_API_COUNTER = ""
THRESHOLD_ENDPOINT = "http://127.0.0.1:13869"
THRESHOLD_API_KEY = "name"
THRESHOLD_API_TOKEN = "token"
THRESHOLD_API_COUNTER = "counter"
# NickTrace
NICKTRACE_MAX_ITERATIONS = 4
@ -103,4 +64,4 @@ META_MAX_CHUNK_SIZE = 500
META_QUERY_SIZE = 10000
DEBUG = True
PROFILER = False
PROFILER = True

57
app/settings.py
View File

@ -35,42 +35,20 @@ INSTALLED_APPS = [
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"debug_toolbar",
"template_profiler_panel",
"django_htmx",
"crispy_forms",
"crispy_bulma",
"django_tables2",
"django_tables2_bulma_template",
"prettyjson",
"mixins",
"cachalot",
]
# Performance optimisations
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "unix:///var/run/socks/redis.sock",
"OPTIONS": {
"db": "10",
# "parser_class": "django_redis.cache.RedisCache",
"pool_class": "redis.BlockingConnectionPool",
},
}
}
CRISPY_TEMPLATE_PACK = "bulma"
CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
DJANGO_TABLES2_TEMPLATE = "django-tables2/bulma.html"
MIDDLEWARE = [
"debug_toolbar.middleware.DebugToolbarMiddleware",
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
# 'django.middleware.cache.UpdateCacheMiddleware',
"django.middleware.common.CommonMiddleware",
# 'django.middleware.cache.FetchFromCacheMiddleware',
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
@ -105,7 +83,7 @@ WSGI_APPLICATION = "app.wsgi.application"
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": "/conf/db.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
}
}
@ -161,30 +139,6 @@ REST_FRAMEWORK = {
]
}
INTERNAL_IPS = [
"127.0.0.1",
"10.1.10.11",
]
DEBUG_TOOLBAR_PANELS = [
"template_profiler_panel.panels.template.TemplateProfilerPanel",
"debug_toolbar.panels.history.HistoryPanel",
"debug_toolbar.panels.versions.VersionsPanel",
"debug_toolbar.panels.timer.TimerPanel",
"debug_toolbar.panels.settings.SettingsPanel",
"debug_toolbar.panels.headers.HeadersPanel",
"debug_toolbar.panels.request.RequestPanel",
"debug_toolbar.panels.sql.SQLPanel",
"debug_toolbar.panels.staticfiles.StaticFilesPanel",
"debug_toolbar.panels.templates.TemplatesPanel",
"debug_toolbar.panels.cache.CachePanel",
"debug_toolbar.panels.signals.SignalsPanel",
"debug_toolbar.panels.logging.LoggingPanel",
"debug_toolbar.panels.redirects.RedirectsPanel",
"debug_toolbar.panels.profiling.ProfilingPanel",
"cachalot.panels.CachalotPanel",
]
from app.local_settings import * # noqa
if PROFILER: # noqa - trust me its there
@ -198,12 +152,3 @@ if PROFILER: # noqa - trust me its there
# "region": f'{os.getenv("REGION")}',
# }
)
def show_toolbar(request):
return DEBUG # noqa: from local imports
DEBUG_TOOLBAR_CONFIG = {
"SHOW_TOOLBAR_CALLBACK": show_toolbar,
}

132
app/urls.py
View File

@ -19,36 +19,28 @@ from django.contrib import admin
from django.urls import include, path
from django.views.generic import TemplateView
# Notification settings and rules
# Threshold API stuff
from core.views import About, Billing, Cancel, Order, Portal, Signup, notifications
from core.api.views.threshold import ThresholdChans, ThresholdOnline, ThresholdUsers
from core.views import About, Billing, Cancel, Order, Portal, Signup
from core.views.callbacks import Callback
from core.views.manage.threshold.irc import (
ThresholdIRCNetworkList, # Actions and just get list output
)
from core.views.manage.threshold.irc import (
ThresholdIRCActions,
ThresholdIRCActionsAddNetwork,
ThresholdIRCActionsRegistration,
ThresholdIRCActionsRegistrationAuth,
ThresholdIRCAliases,
ThresholdIRCAliasesEdit,
ThresholdIRCNetworkActions,
ThresholdIRCNetworkActionsAuto,
ThresholdIRCNetworkActionsList,
ThresholdIRCNetworkActionsRelay,
ThresholdIRCNetworkChannels,
ThresholdIRCNetworkChannelsAPI,
ThresholdIRCNetworkDel,
ThresholdIRCNetworkInfo,
ThresholdIRCNetworkInfoEdit,
ThresholdIRCNetworkRelayAuth,
ThresholdIRCNetworkRelayDel,
ThresholdIRCNetworkRelayProvision,
ThresholdIRCNetworkRelays,
ThresholdIRCNetworkRelayStatus,
ThresholdIRCNetworks,
ThresholdIRCOverviewAlerts,
ThresholdIRCSendMessage,
ThresholdIRCStats,
)
@ -59,11 +51,7 @@ from core.views.manage.threshold.threshold import (
)
# Main tool pages
from core.views.ui.drilldown import ( # DrilldownTableView,; Drilldown,
DrilldownContextModal,
DrilldownTableView,
ThresholdInfoModal,
)
from core.views.ui.drilldown import Drilldown, ThresholdInfoModal # DrilldownTableView,
from core.views.ui.insights import (
Insights,
InsightsChannels,
@ -74,10 +62,7 @@ from core.views.ui.insights import (
)
urlpatterns = [
path("__debug__/", include("debug_toolbar.urls")),
path("", DrilldownTableView.as_view(), name="home"),
path("search/", DrilldownTableView.as_view(), name="search"),
path("search/partial/", DrilldownTableView.as_view(), name="search_partial"),
path("", Drilldown.as_view(), name="home"),
path("about/", About.as_view(), name="about"),
path("callback", Callback.as_view(), name="callback"),
path("billing/", Billing.as_view(), name="billing"),
@ -98,36 +83,13 @@ urlpatterns = [
##
# path("drilldown/", Drilldown.as_view(), name="drilldown"),
path("modal/", ThresholdInfoModal.as_view(), name="modal_drilldown"),
path("modal/<str:type>/", ThresholdInfoModal.as_view(), name="modal_drilldown"),
path("context/", DrilldownContextModal.as_view(), name="modal_context"),
path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"),
##
path("ui/insights/index/<str:index>/", Insights.as_view(), name="insights"),
path(
"ui/insights/index/<str:index>/search/",
InsightsSearch.as_view(),
name="search_insights",
),
path(
"ui/insights/index/<str:index>/channels/",
InsightsChannels.as_view(),
name="chans_insights",
),
path(
"ui/insights/index/<str:index>/nicks/",
InsightsNicks.as_view(),
name="nicks_insights",
),
path(
"ui/insights/index/<str:index>/meta/",
InsightsMeta.as_view(),
name="meta_insights",
),
path(
"ui/insights/index/<str:index>/modal/",
InsightsInfoModal.as_view(),
name="modal_insights",
),
path("ui/insights/", Insights.as_view(), name="insights"),
path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"),
path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"),
path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"),
path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"),
path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"),
##
path(
"manage/threshold/irc/overview/",
@ -159,31 +121,6 @@ urlpatterns = [
ThresholdIRCActionsAddNetwork.as_view(),
name="threshold_irc_actions_add-network",
),
path(
"manage/threshold/irc/actions/registration/auth/",
ThresholdIRCActionsRegistrationAuth.as_view(),
name="threshold_irc_actions_registration_auth",
),
path(
"manage/threshold/irc/actions/registration/<str:net>/",
ThresholdIRCActionsRegistration.as_view(),
name="threshold_irc_actions_registration_net",
),
path(
"manage/threshold/irc/network/<str:net>/<int:num>/provision/",
ThresholdIRCNetworkRelayProvision.as_view(),
name="threshold_irc_network_relay_provision",
),
path(
"manage/threshold/irc/network/<str:net>/<int:num>/auth/",
ThresholdIRCNetworkRelayAuth.as_view(),
name="threshold_irc_network_relay_auth",
),
path(
"manage/threshold/irc/actions/registration/",
ThresholdIRCActionsRegistration.as_view(),
name="threshold_irc_actions_registration",
),
path(
"manage/threshold/irc/network/<str:net>/",
ThresholdIRCNetwork.as_view(),
@ -229,11 +166,6 @@ urlpatterns = [
ThresholdIRCNetworkChannels.as_view(),
name="threshold_irc_network_channels",
),
path(
"manage/threshold/irc/network/<str:net>/channel/json/",
ThresholdIRCNetworkChannelsAPI.as_view(),
name="threshold_irc_network_channel_json",
),
path(
"manage/threshold/irc/network/<str:net>/channel/<channel>/",
ThresholdIRCNetworkChannels.as_view(),
@ -272,43 +204,11 @@ urlpatterns = [
),
path(
"manage/threshold/irc/list/<str:net>/",
ThresholdIRCNetworkList.as_view(),
name="threshold_irc_network_list",
),
path(
"manage/threshold/irc/msg/<str:net>/<str:num>/",
ThresholdIRCSendMessage.as_view(),
name="threshold_irc_msg",
ThresholdIRCNetworkActionsList.as_view(),
name="threshold_irc_network_actions_list",
),
##
path(
"notifications/<str:type>/update/",
notifications.NotificationsUpdate.as_view(),
name="notifications_update",
),
path(
"rules/<str:type>/",
notifications.RuleList.as_view(),
name="rules",
),
path(
"rule/<str:type>/create/",
notifications.RuleCreate.as_view(),
name="rule_create",
),
path(
"rule/<str:type>/update/<str:pk>/",
notifications.RuleUpdate.as_view(),
name="rule_update",
),
path(
"rule/<str:type>/delete/<str:pk>/",
notifications.RuleDelete.as_view(),
name="rule_delete",
),
path(
"rule/<str:type>/clear/<str:pk>/",
notifications.RuleClear.as_view(),
name="rule_clear",
),
path("api/chans/", ThresholdChans.as_view(), name="chans"),
path("api/users/", ThresholdUsers.as_view(), name="users"),
path("api/online/", ThresholdOnline.as_view(), name="online"),
] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

8
core/__init__.py
View File

@ -1,13 +1,5 @@
import os
import stripe
from django.conf import settings
from redis import StrictRedis
os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
r = StrictRedis(unix_socket_path="/var/run/socks/redis.sock", db=0)
if settings.STRIPE_TEST:
stripe.api_key = settings.STRIPE_API_KEY_TEST

0
core/lib/__init__.py → core/api/views/__init__.py
View File

67
core/api/views/threshold.py Normal file
View File

@ -0,0 +1,67 @@
import logging
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponse, JsonResponse
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.lib.threshold import annotate_online, get_chans, get_users
logger = logging.getLogger(__name__)
class ThresholdChans(LoginRequiredMixin, APIView):
parser_classes = [FormParser]
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
if "net" not in request.data:
return JsonResponse({"success": False})
if "query" not in request.data:
return JsonResponse({"success": False})
net = request.data["net"]
query = request.data["query"]
channels = get_chans(net, [query])
if not channels:
return HttpResponse("")
channels_human = ", ".join(channels)
return HttpResponse(channels_human)
class ThresholdUsers(LoginRequiredMixin, APIView):
parser_classes = [FormParser]
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
if "net" not in request.data:
return JsonResponse({"success": False})
if "query" not in request.data:
return JsonResponse({"success": False})
net = request.data["net"]
query = request.data["query"]
users = get_users(net, [query])
if not users:
return HttpResponse("")
users_human = ", ".join(users)
return HttpResponse(users_human)
class ThresholdOnline(LoginRequiredMixin, APIView):
parser_classes = [FormParser]
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
if "net" not in request.data:
return JsonResponse({"success": False})
if "query" not in request.data:
return JsonResponse({"success": False})
net = request.data["net"]
query = request.data["query"]
online_info = annotate_online(net, query)
return JsonResponse(online_info)

261
core/db/__init__.py
View File

@ -1,261 +0,0 @@
import random
import string
import time
from abc import ABC, abstractmethod
from math import floor, log10
import orjson
from django.conf import settings
from siphashc import siphash
from core import r
from core.db.processing import annotate_results
from core.util import logs
def remove_defaults(query_params):
for field, value in list(query_params.items()):
if field in settings.DRILLDOWN_DEFAULT_PARAMS:
if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
del query_params[field]
def add_defaults(query_params):
for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
if field not in query_params:
query_params[field] = value
def dedup_list(data, check_keys):
"""
Remove duplicate dictionaries from list.
"""
seen = set()
out = []
dup_count = 0
for x in data:
dedupeKey = tuple(x[k] for k in check_keys if k in x)
if dedupeKey in seen:
dup_count += 1
continue
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
dup_count = 0
out.append(x)
seen.add(dedupeKey)
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
return out
class StorageBackend(ABC):
def __init__(self, name):
self.log = logs.get_logger(name)
self.log.info(f"Initialising storage backend {name}")
self.initialise_caching()
# self.initialise()
@abstractmethod
def initialise(self, **kwargs):
pass
def initialise_caching(self):
hash_key = r.get("cache_hash_key")
if not hash_key:
letters = string.ascii_lowercase
hash_key = "".join(random.choice(letters) for i in range(16))
self.log.debug(f"Created new hash key: {hash_key}")
r.set("cache_hash_key", hash_key)
else:
hash_key = hash_key.decode("ascii")
self.log.debug(f"Decoded hash key: {hash_key}")
self.hash_key = hash_key
@abstractmethod
def construct_query(self, **kwargs):
pass
def parse_query(self, query_params, tags, size, custom_query, add_bool, **kwargs):
query_created = False
if "query" in query_params:
query = query_params["query"]
search_query = self.construct_query(query, size, **kwargs)
query_created = True
else:
if custom_query:
search_query = custom_query
else:
search_query = self.construct_query(None, size, blank=True, **kwargs)
if tags:
# Get a blank search query
if not query_created:
search_query = self.construct_query(None, size, blank=True, **kwargs)
query_created = True
for item in tags:
for tagname, tagvalue in item.items():
add_bool.append({tagname: tagvalue})
bypass_check = kwargs.get("bypass_check", False)
if not bypass_check:
valid = self.check_valid_query(query_params, custom_query, **kwargs)
if isinstance(valid, dict):
return valid
return search_query
def check_valid_query(self, query_params, custom_query):
required_any = ["query", "tags"]
if not any([field in query_params.keys() for field in required_any]):
if not custom_query:
message = "Empty query!"
message_class = "warning"
return {"message": message, "class": message_class}
@abstractmethod
def run_query(self, **kwargs):
pass
def filter_blacklisted(self, user, response):
"""
Low level filter to take the raw search response and remove
objects from it we want to keep secret.
Does not return, the object is mutated in place.
"""
response["redacted"] = 0
response["exemption"] = None
if user.is_superuser:
response["exemption"] = True
# is_anonymous = isinstance(user, AnonymousUser)
# For every hit from ES
for index, item in enumerate(list(response["hits"]["hits"])):
# For every blacklisted type
for blacklisted_type in settings.ELASTICSEARCH_BLACKLISTED.keys():
# Check this field we are matching exists
if "_source" in item.keys():
data_index = "_source"
elif "fields" in item.keys():
data_index = "fields"
else:
return False
if blacklisted_type in item[data_index].keys():
content = item[data_index][blacklisted_type]
# For every item in the blacklisted array for the type
for blacklisted_item in settings.BLACKLISTED[blacklisted_type]:
if blacklisted_item == str(content):
# Remove the item
if item in response["hits"]["hits"]:
# Let the UI know something was redacted
if (
"exemption"
not in response["hits"]["hits"][index][data_index]
):
response["redacted"] += 1
# Anonymous
if user.is_anonymous:
# Just set it to none so the index is not off
response["hits"]["hits"][index] = None
else:
if not user.has_perm("core.bypass_blacklist"):
response["hits"]["hits"][index] = None
else:
response["hits"]["hits"][index][data_index][
"exemption"
] = True
# Actually get rid of all the things we set to None
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
def query(self, user, search_query, **kwargs):
# For time tracking
start = time.process_time()
if settings.CACHE:
# Sort the keys so the hash is the same
query_normalised = orjson.dumps(search_query, option=orjson.OPT_SORT_KEYS)
hash = siphash(self.hash_key, query_normalised)
cache_hit = r.get(f"query_cache.{user.id}.{hash}")
if cache_hit:
response = orjson.loads(cache_hit)
time_took = (time.process_time() - start) * 1000
# Round to 3 significant figures
time_took_rounded = round(
time_took, 3 - int(floor(log10(abs(time_took)))) - 1
)
return {
"object_list": response,
"took": time_took_rounded,
"cache": True,
}
response = self.run_query(user, search_query, **kwargs)
# For Elasticsearch
if isinstance(response, Exception):
message = f"Error: {response.info['error']['root_cause'][0]['type']}"
message_class = "danger"
return {"message": message, "class": message_class}
if "took" in response:
if response["took"] is None:
return None
if len(response["hits"]["hits"]) == 0:
message = "No results."
message_class = "danger"
time_took = (time.process_time() - start) * 1000
# Round to 3 significant figures
time_took_rounded = round(
time_took, 3 - int(floor(log10(abs(time_took)))) - 1
)
return {
"message": message,
"class": message_class,
"took": time_took_rounded,
}
# For Druid
if "error" in response:
if "errorMessage" in response:
context = {
"message": response["errorMessage"],
"class": "danger",
}
return context
else:
return response
# Removed for now, no point given we have restricted indexes
# self.filter_blacklisted(user, response)
# Parse the response
response_parsed = self.parse(response)
# Write cache
if settings.CACHE:
to_write_cache = orjson.dumps(response_parsed)
r.set(f"query_cache.{user.id}.{hash}", to_write_cache)
r.expire(f"query_cache.{user.id}.{hash}", settings.CACHE_TIMEOUT)
time_took = (time.process_time() - start) * 1000
# Round to 3 significant figures
time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1)
return {"object_list": response_parsed, "took": time_took_rounded}
@abstractmethod
def query_results(self, **kwargs):
pass
def process_results(self, response, **kwargs):
if kwargs.get("annotate"):
annotate_results(response)
if kwargs.get("reverse"):
response.reverse()
if kwargs.get("dedup"):
dedup_fields = kwargs.get("dedup_fields")
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
response = dedup_list(response, dedup_fields)
return response
@abstractmethod
def parse(self, response):
pass

272
core/db/druid.py
View File

@ -1,272 +0,0 @@
import logging
import orjson
import requests
from django.conf import settings
from core.db import StorageBackend, add_defaults
from core.db.processing import parse_druid
from core.lib.parsing import (
parse_date_time,
parse_index,
parse_sentiment,
parse_size,
parse_sort,
parse_source,
)
logger = logging.getLogger(__name__)
class DruidBackend(StorageBackend):
def __init__(self):
super().__init__("druid")
def initialise(self, **kwargs):
# self.client = PyDruid("http://broker:8082", "druid/v2")
pass # we use requests
def construct_context_query(
self, index, net, channel, src, num, size, type=None, nicks=None
):
search_query = self.construct_query(None, size, index, blank=True)
extra_must = []
extra_should = []
extra_should2 = []
if num:
extra_must.append({"num": num})
if net:
extra_must.append({"net": net})
if channel:
extra_must.append({"channel": channel})
if nicks:
for nick in nicks:
extra_should2.append({"nick": nick})
types = ["msg", "notice", "action", "kick", "topic", "mode"]
if index == "internal":
if channel == "*status" or type == "znc":
if {"channel": channel} in extra_must:
extra_must.remove({"channel": channel})
extra_should2 = []
# Type is one of msg or notice
# extra_should.append({"match": {"mtype": "msg"}})
# extra_should.append({"match": {"mtype": "notice"}})
extra_should.append({"type": "znc"})
extra_should.append({"type": "self"})
extra_should2.append({"type": "znc"})
extra_should2.append({"nick": channel})
elif type == "auth":
if {"match": {"channel": channel}} in extra_must:
extra_must.remove({"channel": channel})
extra_should2 = []
extra_should2.append({"nick": channel})
# extra_should2.append({"match": {"mtype": "msg"}})
# extra_should2.append({"match": {"mtype": "notice"}})
extra_should.append({"type": "query"})
extra_should2.append({"type": "self"})
extra_should.append({"nick": channel})
else:
for ctype in types:
extra_should.append({"mtype": ctype})
else:
for ctype in types:
extra_should.append({"type": ctype})
if extra_must:
self.add_type("and", search_query, extra_must)
if extra_should:
self.add_type("or", search_query, extra_should)
if extra_should2:
self.add_type("or", search_query, extra_should2)
return search_query
def construct_query(self, query, size, blank=False, **kwargs):
index = kwargs.get("index")
search_query = {
"limit": size,
"queryType": "scan",
"dataSource": index,
"intervals": ["1999-01-01/2999-01-01"],
}
base_filter = {
"type": "and",
"fields": [],
}
to_add = {
"type": "search",
"dimension": "msg",
"query": {
"type": "insensitive_contains",
"value": query,
},
}
if blank:
return search_query
else:
search_query["filter"] = base_filter
search_query["filter"]["fields"].append(to_add)
return search_query
def parse(self, response):
parsed = parse_druid(response)
return parsed
def run_query(self, user, search_query):
ss = orjson.dumps(search_query, option=orjson.OPT_INDENT_2)
ss = ss.decode()
response = requests.post("http://druid:8082/druid/v2", json=search_query)
response = orjson.loads(response.text)
return response
def filter_blacklisted(self, user, response):
pass
def query_results(
self,
request,
query_params,
size=None,
annotate=True,
custom_query=False,
reverse=False,
dedup=False,
dedup_fields=None,
tags=None,
):
add_bool = []
add_in = {}
add_defaults(query_params)
# Now, run the helpers for SIQTSRSS/ADR
# S - Size
# I - Index
# Q - Query
# T - Tags
# S - Source
# R - Ranges
# S - Sort
# S - Sentiment
# A - Annotate
# D - Dedup
# R - Reverse
# S - Size
if request.user.is_anonymous:
sizes = settings.MAIN_SIZES_ANON
else:
sizes = settings.MAIN_SIZES
if not size:
size = parse_size(query_params, sizes)
if isinstance(size, dict):
return size
# I - Index
index = parse_index(request.user, query_params)
if isinstance(index, dict):
return index
# Q/T - Query/Tags
search_query = self.parse_query(
query_params, tags, size, custom_query, add_bool, index=index
)
# Query should be a dict, so check if it contains message here
if "message" in search_query:
return search_query
# S - Sources
sources = parse_source(request.user, query_params)
if isinstance(sources, dict):
return sources
total_count = len(sources)
total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
if total_count != total_sources:
add_in["src"] = sources
# R - Ranges
from_ts, to_ts = parse_date_time(query_params)
if from_ts:
addendum = f"{from_ts}/{to_ts}"
search_query["intervals"] = [addendum]
# S - Sort
sort = parse_sort(query_params)
if isinstance(sort, dict):
return sort
if sort:
search_query["order"] = sort
# S - Sentiment
sentiment_r = parse_sentiment(query_params)
if isinstance(sentiment_r, dict):
return sentiment_r
if sentiment_r:
sentiment_method, sentiment = sentiment_r
sentiment_query = {"type": "bound", "dimension": "sentiment"}
if sentiment_method == "below":
sentiment_query["upper"] = sentiment
elif sentiment_method == "above":
sentiment_query["lower"] = sentiment
elif sentiment_method == "exact":
sentiment_query["lower"] = sentiment
sentiment_query["upper"] = sentiment
elif sentiment_method == "nonzero":
sentiment_query["lower"] = -0.0001
sentiment_query["upper"] = 0.0001
sentiment_query["lowerStrict"] = True
sentiment_query["upperStrict"] = True
# add_bool.append(sentiment_query)
self.add_filter(search_query)
search_query["filter"]["fields"].append(sentiment_query)
# Add in the additional information we already populated
if add_bool:
self.add_type("and", search_query, add_bool)
if add_in:
self.add_in(search_query, add_in)
response = self.query(request.user, search_query)
# A/D/R - Annotate/Dedup/Reverse
response = self.process_results(
response,
annotate=annotate,
dedup=dedup,
dedup_fields=dedup_fields,
reverse=reverse,
)
context = response
return context
def add_filter(self, search_query):
if "filter" not in search_query:
search_query["filter"] = {
"type": "and",
"fields": [],
}
def add_in(self, search_query, add_in):
self.add_filter(search_query)
for key, value in add_in.items():
to_add = {"type": "in", "dimension": key, "values": value}
search_query["filter"]["fields"].append(to_add)
def add_type(self, gate, search_query, add_bool):
top_level_bool = {"type": gate, "fields": []}
self.add_filter(search_query)
for item in add_bool:
for key, value in item.items():
to_add = {"type": "selector", "dimension": key, "value": value}
top_level_bool["fields"].append(to_add)
search_query["filter"]["fields"].append(top_level_bool)
def check_valid_query(self, query_params, custom_query):
# We can do blank queries with this data source
pass

692
core/db/elastic.py
View File

@ -1,692 +0,0 @@
# from copy import deepcopy
# from datetime import datetime, timedelta
from django.conf import settings
from elasticsearch import AsyncElasticsearch, Elasticsearch
from elasticsearch.exceptions import NotFoundError, RequestError
from core.db import StorageBackend, add_defaults
# from json import dumps
# pp = lambda x: print(dumps(x, indent=2))
from core.db.processing import parse_results
from core.lib.parsing import (
QueryError,
parse_date_time,
parse_index,
parse_rule,
parse_sentiment,
parse_size,
parse_sort,
parse_source,
)
# These are sometimes numeric, sometimes strings.
# If they are seen to be numeric first, ES will erroneously
# index them as "long" and then subsequently fail to index messages
# with strings in the field.
keyword_fields = ["nick_id", "user_id", "net_id"]
mapping = {
"mappings": {
"properties": {
"ts": {"type": "date", "format": "epoch_second"},
"match_ts": {"type": "date", "format": "iso8601"},
"file_tim": {"type": "date", "format": "epoch_millis"},
"rule_id": {"type": "keyword"},
}
}
}
for field in keyword_fields:
mapping["mappings"]["properties"][field] = {"type": "text"}
class ElasticsearchBackend(StorageBackend):
def __init__(self):
super().__init__("elasticsearch")
self.client = None
self.async_client = None
def initialise(self, **kwargs):
"""
Inititialise the Elasticsearch API endpoint.
"""
auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
client = Elasticsearch(
settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
)
self.client = client
async def async_initialise(self, **kwargs):
"""
Inititialise the Elasticsearch API endpoint in async mode.
"""
global mapping
auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
client = AsyncElasticsearch(
settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
)
self.async_client = client
# Create the rule storage indices
if await client.indices.exists(index=settings.INDEX_RULE_STORAGE):
await client.indices.put_mapping(
index=settings.INDEX_RULE_STORAGE,
properties=mapping["mappings"]["properties"],
)
else:
await client.indices.create(
index=settings.INDEX_RULE_STORAGE, mappings=mapping["mappings"]
)
def delete_rule_entries(self, rule_id):
"""
Delete all entries for a given rule.
:param rule_id: The rule ID to delete.
"""
if self.client is None:
self.initialise()
search_query = self.construct_query(None, None, blank=True)
search_query["query"]["bool"]["must"].append(
{"match_phrase": {"rule_id": rule_id}}
)
return self.client.delete_by_query(
index=settings.INDEX_RULE_STORAGE, body=search_query
)
def construct_context_query(
self, index, net, channel, src, num, size, type=None, nicks=None
):
# Get the initial query
query = self.construct_query(None, size, blank=True)
extra_must = []
extra_should = []
extra_should2 = []
if num:
extra_must.append({"match_phrase": {"num": num}})
if net:
extra_must.append({"match_phrase": {"net": net}})
if channel:
extra_must.append({"match": {"channel": channel}})
if nicks:
for nick in nicks:
extra_should2.append({"match": {"nick": nick}})
types = ["msg", "notice", "action", "kick", "topic", "mode"]
fields = [
"nick",
"ident",
"host",
"channel",
"ts",
"msg",
"type",
"net",
"src",
"tokens",
]
query["fields"] = fields
if index == "internal":
fields.append("mtype")
if channel == "*status" or type == "znc":
if {"match": {"channel": channel}} in extra_must:
extra_must.remove({"match": {"channel": channel}})
extra_should2 = []
# Type is one of msg or notice
# extra_should.append({"match": {"mtype": "msg"}})
# extra_should.append({"match": {"mtype": "notice"}})
extra_should.append({"match": {"type": "znc"}})
extra_should.append({"match": {"type": "self"}})
extra_should2.append({"match": {"type": "znc"}})
extra_should2.append({"match": {"nick": channel}})
elif type == "auth":
if {"match": {"channel": channel}} in extra_must:
extra_must.remove({"match": {"channel": channel}})
extra_should2 = []
extra_should2.append({"match": {"nick": channel}})
# extra_should2.append({"match": {"mtype": "msg"}})
# extra_should2.append({"match": {"mtype": "notice"}})
extra_should.append({"match": {"type": "query"}})
extra_should2.append({"match": {"type": "self"}})
extra_should.append({"match": {"nick": channel}})
else:
for ctype in types:
extra_should.append({"match": {"mtype": ctype}})
else:
for ctype in types:
extra_should.append({"match": {"type": ctype}})
# query = {
# "index": index,
# "limit": size,
# "query": {
# "bool": {
# "must": [
# # {"equals": {"src": src}},
# # {
# # "bool": {
# # "should": [*extra_should],
# # }
# # },
# # {
# # "bool": {
# # "should": [*extra_should2],
# # }
# # },
# *extra_must,
# ]
# }
# },
# "fields": fields,
# # "_source": False,
# }
if extra_must:
for x in extra_must:
query["query"]["bool"]["must"].append(x)
if extra_should:
query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
if extra_should2:
query["query"]["bool"]["must"].append(
{"bool": {"should": [*extra_should2]}}
)
return query
def construct_query(self, query, size=None, blank=False, **kwargs):
"""
Accept some query parameters and construct an Elasticsearch query.
"""
query_base = {
# "size": size,
"query": {"bool": {"must": []}},
}
if size:
query_base["size"] = size
query_string = {
"query_string": {
"query": query,
# "fields": fields,
# "default_field": "msg",
# "type": "best_fields",
"fuzziness": "AUTO",
"fuzzy_transpositions": True,
"fuzzy_max_expansions": 50,
"fuzzy_prefix_length": 0,
# "minimum_should_match": 1,
"default_operator": "and",
"analyzer": "standard",
"lenient": True,
"boost": 1,
"allow_leading_wildcard": True,
# "enable_position_increments": False,
"phrase_slop": 3,
# "max_determinized_states": 10000,
"quote_field_suffix": "",
"quote_analyzer": "standard",
"analyze_wildcard": False,
"auto_generate_synonyms_phrase_query": True,
}
}
if not blank:
query_base["query"]["bool"]["must"].append(query_string)
return query_base
def parse(self, response, **kwargs):
parsed = parse_results(response, **kwargs)
return parsed
def run_query(self, user, search_query, **kwargs):
"""
Low level helper to run an ES query.
Accept a user to pass it to the filter, so we can
avoid filtering for superusers.
Accept fields and size, for the fields we want to match and the
number of results to return.
"""
if self.client is None:
self.initialise()
index = kwargs.get("index")
try:
response = self.client.search(body=search_query, index=index)
except RequestError as err:
print("Elasticsearch error", err)
return err
except NotFoundError as err:
print("Elasticsearch error", err)
return err
return response
async def async_run_query(self, user, search_query, **kwargs):
"""
Low level helper to run an ES query.
Accept a user to pass it to the filter, so we can
avoid filtering for superusers.
Accept fields and size, for the fields we want to match and the
number of results to return.
"""
if self.async_client is None:
await self.async_initialise()
index = kwargs.get("index")
try:
response = await self.async_client.search(body=search_query, index=index)
except RequestError as err:
print("Elasticsearch error", err)
return err
except NotFoundError as err:
print("Elasticsearch error", err)
return err
return response
async def async_store_matches(self, matches):
"""
Store a list of matches in Elasticsearch.
:param index: The index to store the matches in.
:param matches: A list of matches to store.
"""
if self.async_client is None:
await self.async_initialise()
for match in matches:
result = await self.async_client.index(
index=settings.INDEX_RULE_STORAGE, body=match
)
if not result["result"] == "created":
self.log.error(f"Indexing failed: {result}")
self.log.debug(f"Indexed {len(matches)} messages in ES")
def store_matches(self, matches):
"""
Store a list of matches in Elasticsearch.
:param index: The index to store the matches in.
:param matches: A list of matches to store.
"""
if self.client is None:
self.initialise()
for match in matches:
result = self.client.index(index=settings.INDEX_RULE_STORAGE, body=match)
if not result["result"] == "created":
self.log.error(f"Indexing failed: {result}")
self.log.debug(f"Indexed {len(matches)} messages in ES")
def prepare_schedule_query(self, rule_object):
"""
Helper to run a scheduled query with reduced functionality.
"""
data = rule_object.parsed
if "tags" in data:
tags = data["tags"]
else:
tags = []
if "query" in data:
query = data["query"][0]
data["query"] = query
add_bool = []
add_top = []
if "source" in data:
total_count = len(data["source"])
total_sources = len(settings.MAIN_SOURCES) + len(
settings.SOURCES_RESTRICTED
)
if total_count != total_sources:
add_top_tmp = {"bool": {"should": []}}
for source_iter in data["source"]:
add_top_tmp["bool"]["should"].append(
{"match_phrase": {"src": source_iter}}
)
add_top.append(add_top_tmp)
if "tokens" in data:
add_top_tmp = {"bool": {"should": []}}
for token in data["tokens"]:
add_top_tmp["bool"]["should"].append(
{"match_phrase": {"tokens": token}}
)
add_top.append(add_top_tmp)
for field, values in data.items():
if field not in ["source", "index", "tags", "query", "sentiment", "tokens"]:
for value in values:
add_top.append({"match": {field: value}})
# Bypass the check for query and tags membership since we can search by msg, etc
search_query = self.parse_query(
data, tags, None, False, add_bool, bypass_check=True
)
if rule_object.window is not None:
range_query = {
"range": {
"ts": {
"gte": f"now-{rule_object.window}",
"lte": "now",
}
}
}
add_top.append(range_query)
self.add_bool(search_query, add_bool)
self.add_top(search_query, add_top)
# if "sentiment" in data:
search_query["aggs"] = {
"avg_sentiment": {
"avg": {"field": "sentiment"},
}
}
return search_query
def schedule_check_aggregations(self, rule_object, result_map):
"""
Check the results of a scheduled query for aggregations.
"""
if rule_object.aggs is None:
return result_map
for index, (meta, result) in result_map.items():
# Default to true, if no aggs are found, we still want to match
match = True
for agg_name, (operator, number) in rule_object.aggs.items():
if agg_name in meta["aggs"]:
agg_value = meta["aggs"][agg_name]["value"]
# TODO: simplify this, match is default to True
if operator == ">":
if agg_value > number:
match = True
else:
match = False
elif operator == "<":
if agg_value < number:
match = True
else:
match = False
elif operator == "=":
if agg_value == number:
match = True
else:
match = False
else:
match = False
else:
# No aggregation found, but it is required
match = False
result_map[index][0]["aggs"][agg_name]["match"] = match
return result_map
def schedule_query_results_test_sync(self, rule_object):
"""
Helper to run a scheduled query test with reduced functionality.
Sync version for running from Django forms.
Does not return results.
"""
data = rule_object.parsed
search_query = self.prepare_schedule_query(rule_object)
for index in data["index"]:
if "message" in search_query:
self.log.error(f"Error parsing test query: {search_query['message']}")
continue
response = self.run_query(
rule_object.user,
search_query,
index=index,
)
self.log.debug(f"Running scheduled test query on {index}: {search_query}")
# self.log.debug(f"Response from scheduled query: {response}")
if isinstance(response, Exception):
error = response.info["error"]["root_cause"][0]["reason"]
self.log.error(f"Error running test scheduled search: {error}")
raise QueryError(error)
async def schedule_query_results(self, rule_object):
"""
Helper to run a scheduled query with reduced functionality and async.
"""
result_map = {}
data = rule_object.parsed
search_query = self.prepare_schedule_query(rule_object)
for index in data["index"]:
if "message" in search_query:
self.log.error(f"Error parsing query: {search_query['message']}")
continue
response = await self.async_run_query(
rule_object.user,
search_query,
index=index,
)
self.log.debug(f"Running scheduled query on {index}: {search_query}")
# self.log.debug(f"Response from scheduled query: {response}")
if isinstance(response, Exception):
error = response.info["error"]["root_cause"][0]["reason"]
self.log.error(f"Error running scheduled search: {error}")
raise QueryError(error)
if len(response["hits"]["hits"]) == 0:
# No results, skip
result_map[index] = ({}, [])
continue
meta, response = self.parse(response, meta=True)
# print("Parsed response", response)
if "message" in response:
self.log.error(f"Error running scheduled search: {response['message']}")
continue
result_map[index] = (meta, response)
# Average aggregation check
# Could probably do this in elasticsearch
result_map = self.schedule_check_aggregations(rule_object, result_map)
return result_map
def query_results(
self,
request,
query_params,
size=None,
annotate=True,
custom_query=False,
reverse=False,
dedup=False,
dedup_fields=None,
tags=None,
):
add_bool = []
add_top = []
add_top_negative = []
add_defaults(query_params)
# Now, run the helpers for SIQTSRSS/ADR
# S - Size
# I - Index
# Q - Query
# T - Tags
# S - Source
# R - Ranges
# S - Sort
# S - Sentiment
# A - Annotate
# D - Dedup
# R - Reverse
# S - Size
if request.user.is_anonymous:
sizes = settings.MAIN_SIZES_ANON
else:
sizes = settings.MAIN_SIZES
if not size:
size = parse_size(query_params, sizes)
if isinstance(size, dict):
return size
rule_object = parse_rule(request.user, query_params)
if isinstance(rule_object, dict):
return rule_object
if rule_object is not None:
index = settings.INDEX_RULE_STORAGE
add_bool.append({"rule_id": str(rule_object.id)})
else:
# I - Index
index = parse_index(request.user, query_params)
if isinstance(index, dict):
return index
# Q/T - Query/Tags
search_query = self.parse_query(
query_params, tags, size, custom_query, add_bool
)
# Query should be a dict, so check if it contains message here
if "message" in search_query:
return search_query
# S - Sources
sources = parse_source(request.user, query_params)
if isinstance(sources, dict):
return sources
total_count = len(sources)
# Total is -1 due to the "all" source
total_sources = (
len(settings.MAIN_SOURCES) - 1 + len(settings.SOURCES_RESTRICTED)
)
# If the sources the user has access to are equal to all
# possible sources, then we don't need to add the source
# filter to the query.
if total_count != total_sources:
add_top_tmp = {"bool": {"should": []}}
for source_iter in sources:
add_top_tmp["bool"]["should"].append(
{"match_phrase": {"src": source_iter}}
)
if query_params["source"] != "all":
add_top.append(add_top_tmp)
# R - Ranges
# date_query = False
from_ts, to_ts = parse_date_time(query_params)
if from_ts:
range_query = {
"range": {
"ts": {
"gt": from_ts,
"lt": to_ts,
}
}
}
add_top.append(range_query)
# S - Sort
sort = parse_sort(query_params)
if isinstance(sort, dict):
return sort
if rule_object is not None:
field = "match_ts"
else:
field = "ts"
if sort:
# For Druid compatibility
sort_map = {"ascending": "asc", "descending": "desc"}
sorting = [
{
field: {
"order": sort_map[sort],
}
}
]
search_query["sort"] = sorting
# S - Sentiment
sentiment_r = parse_sentiment(query_params)
if isinstance(sentiment_r, dict):
return sentiment_r
if sentiment_r:
if rule_object is not None:
sentiment_index = "meta.aggs.avg_sentiment.value"
else:
sentiment_index = "sentiment"
sentiment_method, sentiment = sentiment_r
range_query_compare = {"range": {sentiment_index: {}}}
range_query_precise = {
"match": {
sentiment_index: None,
}
}
if sentiment_method == "below":
range_query_compare["range"][sentiment_index]["lt"] = sentiment
add_top.append(range_query_compare)
elif sentiment_method == "above":
range_query_compare["range"][sentiment_index]["gt"] = sentiment
add_top.append(range_query_compare)
elif sentiment_method == "exact":
range_query_precise["match"][sentiment_index] = sentiment
add_top.append(range_query_precise)
elif sentiment_method == "nonzero":
range_query_precise["match"][sentiment_index] = 0
add_top_negative.append(range_query_precise)
# Add in the additional information we already populated
self.add_bool(search_query, add_bool)
self.add_top(search_query, add_top)
self.add_top(search_query, add_top_negative, negative=True)
response = self.query(
request.user,
search_query,
index=index,
)
if "message" in response:
return response
# A/D/R - Annotate/Dedup/Reverse
response["object_list"] = self.process_results(
response["object_list"],
annotate=annotate,
dedup=dedup,
dedup_fields=dedup_fields,
reverse=reverse,
)
context = response
return context
def query_single_result(self, request, query_params):
context = self.query_results(request, query_params, size=100)
if not context:
return {"message": "Failed to run query", "message_class": "danger"}
if "message" in context:
return context
dedup_set = {item["nick"] for item in context["object_list"]}
if dedup_set:
context["item"] = context["object_list"][0]
return context
def add_bool(self, search_query, add_bool):
"""
Add the specified boolean matches to search query.
"""
if not add_bool:
return
for item in add_bool:
search_query["query"]["bool"]["must"].append({"match_phrase": item})
def add_top(self, search_query, add_top, negative=False):
"""
Merge add_top with the base of the search_query.
"""
if not add_top:
return
if negative:
for item in add_top:
if "must_not" in search_query["query"]["bool"]:
search_query["query"]["bool"]["must_not"].append(item)
else:
search_query["query"]["bool"]["must_not"] = [item]
else:
for item in add_top:
if "query" not in search_query:
search_query["query"] = {"bool": {"must": []}}
search_query["query"]["bool"]["must"].append(item)

302
core/db/manticore.py
View File

@ -1,302 +0,0 @@
import logging
from datetime import datetime
from pprint import pprint
import requests
from django.conf import settings
from core.db import StorageBackend, add_defaults, dedup_list
from core.db.processing import annotate_results, parse_results
logger = logging.getLogger(__name__)
class ManticoreBackend(StorageBackend):
def __init__(self):
super().__init__("manticore")
def initialise(self, **kwargs):
"""
Initialise the Manticore client
"""
pass # we use requests
def construct_query(self, query, size, index, blank=False):
"""
Accept some query parameters and construct an OpenSearch query.
"""
if not size:
size = 5
query_base = {
"index": index,
"limit": size,
"query": {"bool": {"must": []}},
}
query_string = {
"query_string": query,
}
if not blank:
query_base["query"]["bool"]["must"].append(query_string)
return query_base
def run_query(self, client, user, search_query):
response = requests.post(
f"{settings.MANTICORE_URL}/json/search", json=search_query
)
return response
def query_results(
self,
request,
query_params,
size=None,
annotate=True,
custom_query=False,
reverse=False,
dedup=False,
dedup_fields=None,
tags=None,
):
query = None
message = None
message_class = None
add_bool = []
add_top = []
add_top_negative = []
sort = None
query_created = False
source = None
add_defaults(query_params)
# Check size
if request.user.is_anonymous:
sizes = settings.MANTICORE_MAIN_SIZES_ANON
else:
sizes = settings.MANTICORE_MAIN_SIZES
if not size:
if "size" in query_params:
size = query_params["size"]
if size not in sizes:
message = "Size is not permitted"
message_class = "danger"
return {"message": message, "class": message_class}
size = int(size)
else:
size = 20
# Check index
if "index" in query_params:
index = query_params["index"]
if index == "main":
index = settings.MANTICORE_INDEX_MAIN
else:
if not request.user.has_perm(f"core.index_{index}"):
message = "Not permitted to search by this index"
message_class = "danger"
return {
"message": message,
"class": message_class,
}
if index == "meta":
index = settings.MANTICORE_INDEX_META
elif index == "internal":
index = settings.MANTICORE_INDEX_INT
else:
message = "Index is not valid."
message_class = "danger"
return {
"message": message,
"class": message_class,
}
else:
index = settings.MANTICORE_INDEX_MAIN
# Create the search query
if "query" in query_params:
query = query_params["query"]
search_query = self.construct_query(query, size, index)
query_created = True
else:
if custom_query:
search_query = custom_query
if tags:
# Get a blank search query
if not query_created:
search_query = self.construct_query(None, size, index, blank=True)
query_created = True
for tagname, tagvalue in tags.items():
add_bool.append({tagname: tagvalue})
required_any = ["query_full", "query", "tags"]
if not any([field in query_params.keys() for field in required_any]):
if not custom_query:
message = "Empty query!"
message_class = "warning"
return {"message": message, "class": message_class}
# Check for a source
if "source" in query_params:
source = query_params["source"]
if source in settings.SOURCES_RESTRICTED:
if not request.user.has_perm("core.restricted_sources"):
message = "Access denied"
message_class = "danger"
return {"message": message, "class": message_class}
elif source not in settings.MAIN_SOURCES:
message = "Invalid source"
message_class = "danger"
return {"message": message, "class": message_class}
if source == "all":
source = None # the next block will populate it
if source:
sources = [source]
else:
sources = list(settings.MAIN_SOURCES)
if request.user.has_perm("core.restricted_sources"):
for source_iter in settings.SOURCES_RESTRICTED:
sources.append(source_iter)
add_top_tmp = {"bool": {"should": []}}
total_count = 0
for source_iter in sources:
add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}})
total_count += 1
total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
if not total_count == total_sources:
add_top.append(add_top_tmp)
# Date/time range
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
query_params.keys()
):
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
from_ts = int(from_ts.timestamp())
to_ts = int(to_ts.timestamp())
range_query = {
"range": {
"ts": {
"gt": from_ts,
"lt": to_ts,
}
}
}
add_top.append(range_query)
# Sorting
if "sorting" in query_params:
sorting = query_params["sorting"]
if sorting not in ("asc", "desc", "none"):
message = "Invalid sort"
message_class = "danger"
return {"message": message, "class": message_class}
if sorting in ("asc", "desc"):
sort = [
{
"ts": {
"order": sorting,
}
}
]
# Sentiment handling
if "check_sentiment" in query_params:
if "sentiment_method" not in query_params:
message = "No sentiment method"
message_class = "danger"
return {"message": message, "class": message_class}
if "sentiment" in query_params:
sentiment = query_params["sentiment"]
try:
sentiment = float(sentiment)
except ValueError:
message = "Sentiment is not a float"
message_class = "danger"
return {"message": message, "class": message_class}
sentiment_method = query_params["sentiment_method"]
range_query_compare = {"range": {"sentiment": {}}}
range_query_precise = {
"match": {
"sentiment": None,
}
}
if sentiment_method == "below":
range_query_compare["range"]["sentiment"]["lt"] = sentiment
add_top.append(range_query_compare)
elif sentiment_method == "above":
range_query_compare["range"]["sentiment"]["gt"] = sentiment
add_top.append(range_query_compare)
elif sentiment_method == "exact":
range_query_precise["match"]["sentiment"] = sentiment
add_top.append(range_query_precise)
elif sentiment_method == "nonzero":
range_query_precise["match"]["sentiment"] = 0
add_top_negative.append(range_query_precise)
if add_bool:
# if "bool" not in search_query["query"]:
# search_query["query"]["bool"] = {}
# if "must" not in search_query["query"]["bool"]:
# search_query["query"]["bool"] = {"must": []}
for item in add_bool:
search_query["query"]["bool"]["must"].append({"match": item})
if add_top:
for item in add_top:
search_query["query"]["bool"]["must"].append(item)
if add_top_negative:
for item in add_top_negative:
if "must_not" in search_query["query"]["bool"]:
search_query["query"]["bool"]["must_not"].append(item)
else:
search_query["query"]["bool"]["must_not"] = [item]
if sort:
search_query["sort"] = sort
pprint(search_query)
results = self.run_query(
self.client,
request.user, # passed through run_main_query to filter_blacklisted
search_query,
)
if not results:
message = "Error running query"
message_class = "danger"
return {"message": message, "class": message_class}
# results = results.to_dict()
if "error" in results:
message = results["error"]
message_class = "danger"
return {"message": message, "class": message_class}
results_parsed = parse_results(results)
if annotate:
annotate_results(results_parsed)
if "dedup" in query_params:
if query_params["dedup"] == "on":
dedup = True
else:
dedup = False
else:
dedup = False
if reverse:
results_parsed = results_parsed[::-1]
if dedup:
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
results_parsed = dedup_list(results_parsed, dedup_fields)
context = {
"object_list": results_parsed,
"card": results["hits"]["total"],
"took": results["took"],
}
if "cache" in results:
context["cache"] = results["cache"]
return context

143
core/db/processing.py
View File

@ -1,143 +0,0 @@
from datetime import datetime
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
def annotate_results(results):
"""
Accept a list of dict objects, search for the number of channels and users.
Add them to the object.
Mutate it in place. Does not return anything.
"""
# Figure out items with net (not discord)
nets = set()
for x in results:
if "net" in x:
nets.add(x["net"])
for net in nets:
# Annotate the online attribute from Threshold
nicks = list(
set(
[
x["nick"]
for x in results
if {"nick", "src", "net"}.issubset(x)
and x["src"] == "irc"
and x["net"] == net
]
)
)
channels = list(
set(
[
x["channel"]
for x in results
if {"channel", "src", "net"}.issubset(x)
and x["src"] == "irc"
and x["net"] == net
]
)
)
online_info = None
num_users = None
num_chans = None
if nicks:
online_info = annotate_online(net, nicks)
# Annotate the number of users in the channel
if channels:
num_users = annotate_num_users(net, channels)
# Annotate the number channels the user is on
if nicks:
num_chans = annotate_num_chans(net, nicks)
for item in results:
if "net" in item:
if item["net"] == net:
if "nick" in item:
if online_info:
if item["nick"] in online_info:
item["online"] = online_info[item["nick"]]
if "channel" in item:
if num_users:
if item["channel"] in num_users:
item["num_users"] = num_users[item["channel"]]
if "nick" in item:
if num_chans:
if item["nick"] in num_chans:
item["num_chans"] = num_chans[item["nick"]]
def parse_results(results, meta=None):
results_parsed = []
stringify = ["host", "channel"]
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
if "_source" in item.keys():
data_index = "_source"
elif "fields" in item.keys():
data_index = "fields"
else:
return False
element = item[data_index]
for field in stringify:
if field in element:
element[field] = str(element[field])
# Why are fields in lists...
if data_index == "fields":
element = {k: v[0] for k, v in element.items() if len(v)}
element["id"] = item["_id"]
# Remove empty values
for field in list(element.keys()):
if element[field] == "":
del element[field]
# Split the timestamp into date and time
if "ts" not in element:
if "time" in element: # will fix data later
ts = element["time"]
del element["time"]
element["ts"] = ts
if "ts" in element:
if isinstance(element["ts"], str):
ts = element["ts"]
else:
ts = datetime.utcfromtimestamp(element["ts"]).strftime(
"%Y-%m-%dT%H:%M:%S"
)
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
if "." in time:
time_spl = time.split(".")
if len(time_spl) == 2:
element["time"] = time.split(".")[0]
else:
element["time"] = time
else:
element["time"] = time
results_parsed.append(element)
if meta:
meta = {"aggs": {}}
if "aggregations" in results:
for field in ["avg_sentiment"]: # Add other number fields here
if field in results["aggregations"]:
meta["aggs"][field] = results["aggregations"][field]
total_hits = results["hits"]["total"]["value"]
meta["total_hits"] = total_hits
return (meta, results_parsed)
return results_parsed
def parse_druid(response):
results_parsed = []
for item in response:
if "events" in item:
for event in item["events"]:
results_parsed.append(event)
else:
raise Exception(f"events not in item {item}")
return results_parsed

21
core/db/storage.py
View File

@ -1,21 +0,0 @@
from django.conf import settings
def get_db():
if settings.DB_BACKEND == "DRUID":
from core.db.druid import DruidBackend
return DruidBackend()
elif settings.DB_BACKEND == "ELASTICSEARCH":
from core.db.elastic import ElasticsearchBackend
return ElasticsearchBackend()
elif settings.DB_BACKEND == "MANTICORE":
from core.db.manticore import ManticoreBackend
return ManticoreBackend()
else:
raise Exception("Invalid DB backend")
db = get_db()

108
core/forms.py
View File

@ -1,16 +1,9 @@
from django import forms
from django.contrib.auth.forms import UserCreationForm
from django.core.exceptions import FieldDoesNotExist
from django.forms import ModelForm
from mixins.restrictions import RestrictedFormMixin
from core.db.storage import db
from core.lib.parsing import QueryError
from core.lib.rules import NotificationRuleData, RuleParseError
from .models import User
from .models import NotificationRule, NotificationSettings, User
# flake8: noqa: E501
# Create your forms here.
class NewUserForm(UserCreationForm):
@ -39,100 +32,3 @@ class CustomUserCreationForm(UserCreationForm):
class Meta:
model = User
fields = "__all__"
class NotificationSettingsForm(RestrictedFormMixin, ModelForm):
def __init__(self, *args, **kwargs):
super(NotificationSettingsForm, self).__init__(*args, **kwargs)
self.fields["url"].label = "URL"
class Meta:
model = NotificationSettings
fields = (
"topic",
"url",
"service",
)
help_texts = {
"topic": "The topic to send notifications to.",
"url": "Custom NTFY server/webhook destination. Leave blank to use the default server for NTFY. For webhooks this field is required.",
"service": "The service to use for notifications.",
}
def clean(self):
cleaned_data = super(NotificationSettingsForm, self).clean()
if "service" in cleaned_data:
if cleaned_data["service"] == "webhook":
if not cleaned_data.get("url"):
self.add_error(
"url",
"You must set a URL for webhooks.",
)
class NotificationRuleForm(RestrictedFormMixin, ModelForm):
def __init__(self, *args, **kwargs):
super(NotificationRuleForm, self).__init__(*args, **kwargs)
self.fields["url"].label = "URL"
class Meta:
model = NotificationRule
fields = (
"name",
"data",
"interval",
"window",
"amount",
"priority",
"topic",
"url",
"service",
"policy",
"ingest",
"enabled",
)
help_texts = {
"name": "The name of the rule.",
"priority": "The notification priority of the rule.",
"url": "Custom NTFY server/webhook destination. Leave blank to use the default server for NTFY. For webhooks this field is required.",
"service": "The service to use for notifications",
"topic": "The topic to send notifications to. Leave blank for default.",
"enabled": "Whether the rule is enabled.",
"data": "The notification rule definition.",
"interval": "How often to run the search. On demand evaluates messages as they are received, without running a scheduled search. The remaining options schedule a search of the database with the window below.",
"window": "Time window to search: 1d, 1h, 1m, 1s, etc.",
"amount": "Amount of matches to be returned for scheduled queries. Cannot be used with on-demand queries.",
"policy": "When to trigger this policy.",
"ingest": "Whether to ingest matches.",
}
def clean(self):
cleaned_data = super(NotificationRuleForm, self).clean()
# TODO: should this be in rules.py?
if "service" in cleaned_data:
if cleaned_data["service"] == "webhook":
if not cleaned_data.get("url"):
self.add_error(
"url",
"You must set a URL for webhooks.",
)
try:
# Passing db to avoid circular import
parsed_data = NotificationRuleData(self.request.user, cleaned_data, db=db)
if cleaned_data["enabled"]:
parsed_data.test_schedule()
except RuleParseError as e:
self.add_error(e.field, f"Parsing error: {e}")
return
except QueryError as e:
self.add_error("data", f"Query error: {e}")
return
# Write back the validated data
# We need this to populate the index and source variable if
# they are not set
to_store = str(parsed_data)
cleaned_data["data"] = to_store
return cleaned_data

87
core/lib/context.py
View File

@ -1,87 +0,0 @@
def construct_query(index, net, channel, src, num, size, type=None, nicks=None):
# Get the initial query
extra_must = []
extra_should = []
extra_should2 = []
if num:
extra_must.append({"match_phrase": {"num": num}})
if net:
extra_must.append({"match_phrase": {"net": net}})
if channel:
extra_must.append({"match": {"channel": channel}})
if nicks:
for nick in nicks:
extra_should2.append({"match": {"nick": nick}})
types = ["msg", "notice", "action", "kick", "topic", "mode"]
fields = [
"nick",
"ident",
"host",
"channel",
"ts",
"msg",
"type",
"net",
"src",
"tokens",
]
if index == "internal":
fields.append("mtype")
if channel == "*status" or type == "znc":
if {"match": {"channel": channel}} in extra_must:
extra_must.remove({"match": {"channel": channel}})
extra_should2 = []
# Type is one of msg or notice
# extra_should.append({"match": {"mtype": "msg"}})
# extra_should.append({"match": {"mtype": "notice"}})
extra_should.append({"match": {"type": "znc"}})
extra_should.append({"match": {"type": "self"}})
extra_should2.append({"match": {"type": "znc"}})
extra_should2.append({"match": {"nick": channel}})
elif type == "auth":
if {"match": {"channel": channel}} in extra_must:
extra_must.remove({"match": {"channel": channel}})
extra_should2 = []
extra_should2.append({"match": {"nick": channel}})
# extra_should2.append({"match": {"mtype": "msg"}})
# extra_should2.append({"match": {"mtype": "notice"}})
extra_should.append({"match": {"type": "query"}})
extra_should2.append({"match": {"type": "self"}})
extra_should.append({"match": {"nick": channel}})
else:
for ctype in types:
extra_should.append({"match": {"mtype": ctype}})
else:
for ctype in types:
extra_should.append({"match": {"type": ctype}})
query = {
"index": index,
"limit": size,
"query": {
"bool": {
"must": [
# {"equals": {"src": src}},
# {
# "bool": {
# "should": [*extra_should],
# }
# },
# {
# "bool": {
# "should": [*extra_should2],
# }
# },
*extra_must,
]
}
},
"fields": fields,
# "_source": False,
}
if extra_should:
query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
if extra_should2:
query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should2]}})
return query

106
core/lib/manage/threshold.py
View File

@ -1,3 +1,8 @@
import urllib.parse
from django.conf import settings
from core.lib.opensearch import client, run_main_query
from core.lib.threshold import threshold_request
@ -61,8 +66,9 @@ def get_irc_channels(net):
def part_channel(net, channel):
url = f"irc/network/{net}/channel"
payload = {"channel": channel}
channel = urllib.parse.quote(channel, safe="")
url = f"irc/network/{net}/channel/{channel}"
payload = {}
parted = threshold_request(url, payload, method="DELETE")
if not parted:
return {}
@ -70,8 +76,9 @@ def part_channel(net, channel):
def join_channel(net, channel):
url = f"irc/network/{net}/channel"
payload = {"channel": channel}
channel = urllib.parse.quote(channel, safe="")
url = f"irc/network/{net}/channel/{channel}"
payload = {}
joined = threshold_request(url, payload, method="PUT")
if not joined:
return {}
@ -159,69 +166,30 @@ def construct_alert_query():
return query
def send_irc_message(net, num, channel, msg, nick=None):
url = f"irc/msg/{net}/{num}"
payload = {"msg": msg, "channel": channel}
if nick:
payload["nick"] = nick
messaged = threshold_request(url, payload, method="PUT")
return messaged
def get_irc_alerts(user):
query = construct_alert_query()
results = run_main_query(
client,
user, # passed through run_main_query to filter_blacklisted
query,
custom_query=True,
index=settings.OPENSEARCH_INDEX_INT,
)
if not results:
return []
results_parsed = []
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
element["id"] = item["_id"]
def get_irc_nick(net, num):
url = f"irc/nick/{net}/{num}"
payload = {}
nick = threshold_request(url, payload, method="GET")
return nick
def get_irc_list_info(net):
url = f"irc/list/{net}"
payload = {}
listinfo = threshold_request(url, payload, method="GET")
return listinfo
def irc_get_unreg(net=None):
if net:
url = f"irc/reg/{net}"
else:
url = "irc/reg"
payload = {}
unreg = threshold_request(url, payload, method="GET")
return unreg
def irc_confirm_accounts(tokens):
url = "irc/reg"
payload = tokens
updated = threshold_request(url, payload, method="PUT")
return updated
def irc_provision_relay(net, num):
url = f"irc/network/{net}/{num}/provision"
payload = {}
provisioned = threshold_request(url, payload, method="POST")
return provisioned
def irc_enable_auth(net, num):
url = f"irc/network/{net}/{num}/auth"
payload = {}
enabled = threshold_request(url, payload, method="POST")
return enabled
def irc_check_auth(data):
url = "irc/auth"
payload = data
updated = threshold_request(url, payload, method="POST")
return updated
def get_irc_sinst(net):
url = f"irc/sinst/{net}"
payload = {}
authentity = threshold_request(url, payload, method="GET")
return authentity
# Split the timestamp into date and time
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
results_parsed.append(element)
return results_parsed

23
core/lib/meta.py
View File

@ -3,7 +3,7 @@ from math import ceil
from django.conf import settings
from numpy import array_split
from core.db.storage import db
from core.lib.opensearch import client, run_main_query
def construct_query(net, nicks):
@ -43,13 +43,26 @@ def get_meta(request, net, nicks, iter=True):
break
meta_tmp = []
query = construct_query(net, nicks_chunked)
results = db.query(
results = run_main_query(
client,
request.user,
query,
index=settings.INDEX_META,
custom_query=True,
index=settings.OPENSEARCH_INDEX_META,
)
if "object_list" in results.keys():
for element in results["object_list"]:
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
element["id"] = item["_id"]
# Split the timestamp into date and time
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
meta_tmp.append(element)
for x in meta_tmp:
if x not in meta:

4
core/lib/nicktrace.py
View File

@ -3,7 +3,7 @@ from math import ceil
from django.conf import settings
from numpy import array_split
from core.db.storage import db
from core.lib.opensearch import client, run_main_query
def construct_query(net, nicks):
@ -45,7 +45,7 @@ def get_nicks(request, net, nicks, iter=True):
if len(nicks_chunked) == 0:
break
query = construct_query(net, nicks_chunked)
results = db.query(request.user, query)
results = run_main_query(client, request.user, query, custom_query=True)
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:

107
core/lib/notify.py
View File

@ -1,107 +0,0 @@
import requests
from core.util import logs
NTFY_URL = "https://ntfy.sh"
log = logs.get_logger(__name__)
# Actual function to send a message to a topic
def ntfy_sendmsg(**kwargs):
"""
Send a message to a topic using NTFY.
kwargs:
msg: Message to send, must be specified
notification_settings: Notification settings, must be specified
url: URL to NTFY server, can be None to use default
topic: Topic to send message to, must be specified
priority: Priority of message, optional
title: Title of message, optional
tags: Tags to add to message, optional
"""
msg = kwargs.get("msg", None)
notification_settings = kwargs.get("notification_settings")
title = kwargs.get("title", None)
priority = notification_settings.get("priority", None)
tags = kwargs.get("tags", None)
url = notification_settings.get("url") or NTFY_URL
topic = notification_settings.get("topic", None)
headers = {"Title": "Fisk"}
if title:
headers["Title"] = title
if priority:
headers["Priority"] = priority
if tags:
headers["Tags"] = tags
try:
requests.post(
f"{url}/{topic}",
data=msg,
headers=headers,
)
except requests.exceptions.ConnectionError as e:
log.error(f"Error sending notification: {e}")
def webhook_sendmsg(**kwargs):
"""
Send a message to a webhook.
kwargs:
msg: Message to send, must be specified
notification_settings: Notification settings, must be specified
url: URL to webhook, must be specified"""
msg = kwargs.get("msg", None)
notification_settings = kwargs.get("notification_settings")
url = notification_settings.get("url")
headers = {"Content-type": "application/json"}
try:
requests.post(
f"{url}",
headers=headers,
data=msg,
)
except requests.exceptions.ConnectionError as e:
log.error(f"Error sending webhook: {e}")
# Sendmsg helper to send a message to a user's notification settings
def sendmsg(**kwargs):
"""
Send a message to a user's notification settings.
Fetches the user's default notification settings if not specified.
kwargs:
user: User to send message to, must be specified
notification_settings: Notification settings, optional
service: Notification service to use
kwargs for both services:
msg: Message to send, must be specified
notification_settings: Notification settings, must be specified
url: URL to NTFY server, can be None to use default
extra kwargs for ntfy:
title: Title of message, optional
tags: Tags to add to message, optional
notification_settings: Notification settings, must be specified
topic: Topic to send message to, must be specified
priority: Priority of message, optional
"""
user = kwargs.get("user", None)
notification_settings = kwargs.get(
"notification_settings", user.get_notification_settings().__dict__
)
if not notification_settings:
return
service = notification_settings.get("service")
if service == "none":
# Don't send anything
return
if service == "ntfy":
ntfy_sendmsg(**kwargs)
elif service == "webhook":
webhook_sendmsg(**kwargs)

360
core/lib/opensearch.py Normal file
View File

@ -0,0 +1,360 @@
from django.conf import settings
from opensearchpy import OpenSearch
from opensearchpy.exceptions import RequestError
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
def initialise_opensearch():
"""
Inititialise the OpenSearch API endpoint.
"""
auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
client = OpenSearch(
# fmt: off
hosts=[{"host": settings.OPENSEARCH_URL,
"port": settings.OPENSEARCH_PORT}],
http_compress=False, # enables gzip compression for request bodies
http_auth=auth,
# client_cert = client_cert_path,
# client_key = client_key_path,
use_ssl=settings.OPENSEARCH_TLS,
verify_certs=False,
ssl_assert_hostname=False,
ssl_show_warn=False,
# a_certs=ca_certs_path,
)
return client
client = initialise_opensearch()
def annotate_results(results_parsed):
"""
Accept a list of dict objects, search for the number of channels and users.
Add them to the object.
Mutate it in place. Does not return anything.
"""
# Figure out items with net (not discord)
nets = set()
for x in results_parsed:
if "net" in x:
nets.add(x["net"])
for net in nets:
# Annotate the online attribute from Threshold
nicks = [
x["nick"] for x in results_parsed if x["src"] == "irc" and x["net"] == net
]
channels = [
x["channel"]
for x in results_parsed
if x["src"] == "irc" and x["net"] == net
]
online_info = annotate_online(net, nicks)
# Annotate the number of users in the channel
num_users = annotate_num_users(net, channels)
# Annotate the number channels the user is on
num_chans = annotate_num_chans(net, nicks)
for item in results_parsed:
if "net" in item:
if item["net"] == net:
if "nick" in item:
if item["nick"] in online_info:
item["online"] = online_info[item["nick"]]
if "channel" in item:
if item["channel"] in num_users:
item["num_users"] = num_users[item["channel"]]
if "nick" in item:
if item["nick"] in num_chans:
item["num_chans"] = num_chans[item["nick"]]
def filter_blacklisted(user, response):
"""
Low level filter to take the raw OpenSearch response and remove
objects from it we want to keep secret.
Does not return, the object is mutated in place.
"""
response["redacted"] = 0
response["exemption"] = None
if user.is_superuser:
response["exemption"] = True
# is_anonymous = isinstance(user, AnonymousUser)
# For every hit from ES
for index, item in enumerate(list(response["hits"]["hits"])):
# For every blacklisted type
for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
# Check this field we are matching exists
if blacklisted_type in item["_source"].keys():
content = item["_source"][blacklisted_type]
# For every item in the blacklisted array for the type
for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
blacklisted_type
]:
if blacklisted_item == str(content):
# Remove the item
if item in response["hits"]["hits"]:
# Let the UI know something was redacted
if (
"exemption"
not in response["hits"]["hits"][index]["_source"]
):
response["redacted"] += 1
# Anonymous
if user.is_anonymous:
# Just set it to none so the index is not off
response["hits"]["hits"][index] = None
else:
if not user.is_superuser:
response["hits"]["hits"][index] = None
else:
response["hits"]["hits"][index]["_source"][
"exemption"
] = True
# Actually get rid of all the things we set to None
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
def run_main_query(client, user, query, custom_query=False, index=None, size=None):
"""
Low level helper to run an ES query.
Accept a user to pass it to the filter, so we can
avoid filtering for superusers.
Accept fields and size, for the fields we want to match and the
number of results to return.
"""
if not index:
index = settings.OPENSEARCH_INDEX_MAIN
if custom_query:
search_query = query
else:
search_query = construct_query(query, size)
try:
response = client.search(body=search_query, index=index)
except RequestError as err:
print("OpenSearch error", err)
return err
filter_blacklisted(user, response)
return response
def query_results(request, query_params, size=None):
"""
API helper to alter the OpenSearch return format into something
a bit better to parse.
Accept a HTTP request object. Run the query, and annotate the
results with the other data we have.
"""
# is_anonymous = isinstance(request.user, AnonymousUser)
message = None
message_class = None
add_bool = []
add_top = []
add_top_negative = []
sort = None
if request.user.is_anonymous:
sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
else:
sizes = settings.OPENSEARCH_MAIN_SIZES
if not size:
if "size" in query_params:
size = query_params["size"]
if size not in sizes:
message = "Size is not permitted"
message_class = "danger"
return {"message": message, "class": message_class}
if "source" in query_params:
source = query_params["source"]
if source not in settings.OPENSEARCH_MAIN_SOURCES:
message = "Invalid source"
message_class = "danger"
return {"message": message, "class": message_class}
if source != "all":
add_bool.append({"src": source})
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
query_params.keys()
):
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
print("from ts", from_ts)
print("to_ts", to_ts)
range_query = {
"range": {
"ts": {
"gt": from_ts,
"lt": to_ts,
}
}
}
add_top.append(range_query)
if "sorting" in query_params:
sorting = query_params["sorting"]
if sorting not in ("asc", "desc", "none"):
message = "Invalid sort"
message_class = "danger"
return {"message": message, "class": message_class}
if sorting in ("asc", "desc"):
sort = [
{
"ts": {
"order": sorting,
}
}
]
if "check_sentiment" in query_params:
if "sentiment_method" not in query_params:
message = "No sentiment method"
message_class = "danger"
return {"message": message, "class": message_class}
if "sentiment" in query_params:
sentiment = query_params["sentiment"]
try:
sentiment = float(sentiment)
except ValueError:
message = "Sentiment is not a float"
message_class = "danger"
return {"message": message, "class": message_class}
sentiment_method = query_params["sentiment_method"]
range_query_compare = {"range": {"sentiment": {}}}
range_query_precise = {
"match": {
"sentiment": None,
}
}
if sentiment_method == "below":
range_query_compare["range"]["sentiment"]["lt"] = sentiment
add_top.append(range_query_compare)
elif sentiment_method == "above":
range_query_compare["range"]["sentiment"]["gt"] = sentiment
add_top.append(range_query_compare)
elif sentiment_method == "exact":
range_query_precise["match"]["sentiment"] = sentiment
add_top.append(range_query_precise)
elif sentiment_method == "nonzero":
range_query_precise["match"]["sentiment"] = 0
add_top_negative.append(range_query_precise)
if "query" in query_params:
query = query_params["query"]
search_query = construct_query(query, size)
if add_bool:
for item in add_bool:
search_query["query"]["bool"]["must"].append({"match": item})
if add_top:
for item in add_top:
search_query["query"]["bool"]["must"].append(item)
if add_top_negative:
for item in add_top_negative:
if "must_not" in search_query["query"]["bool"]:
search_query["query"]["bool"]["must_not"].append(item)
else:
search_query["query"]["bool"]["must_not"] = [item]
if sort:
search_query["sort"] = sort
results = run_main_query(
client,
request.user, # passed through run_main_query to filter_blacklisted
search_query,
custom_query=True,
size=size,
)
if not results:
return False
if isinstance(results, RequestError):
message = results.info["error"]["root_cause"][0]["reason"]
message_class = "danger"
return {"message": message, "class": message_class}
if len(results["hits"]["hits"]) == 0:
message = "No results."
message_class = "danger"
return {"message": message, "class": message_class}
results_parsed = []
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
element["id"] = item["_id"]
# Split the timestamp into date and time
if "ts" not in element:
if "time" in element: # will fix data later
ts = element["time"]
del element["time"]
element["ts"] = ts
if "ts" in element:
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
results_parsed.append(element)
annotate_results(results_parsed)
context = {
"query": query,
"results": results_parsed,
"card": results["hits"]["total"]["value"],
"took": results["took"],
"redacted": results["redacted"],
"exemption": results["exemption"],
}
return context
def query_single_result(request):
context = query_results(request, 1)
dedup_set = {item["nick"] for item in context["results"]}
if dedup_set:
context["item"] = context["results"][0]
return (1, context)
def construct_query(query, size):
"""
Accept some query parameters and construct an OpenSearch query.
"""
if not size:
size = 5
query = {
"size": size,
"query": {
"bool": {
"must": [
{
"query_string": {
"query": query,
# "fields": fields,
# "default_field": "msg",
# "type": "best_fields",
"fuzziness": "AUTO",
"fuzzy_transpositions": True,
"fuzzy_max_expansions": 50,
"fuzzy_prefix_length": 0,
# "minimum_should_match": 1,
"default_operator": "or",
"analyzer": "standard",
"lenient": True,
"boost": 1,
"allow_leading_wildcard": True,
# "enable_position_increments": False,
"phrase_slop": 3,
# "max_determinized_states": 10000,
"quote_field_suffix": "",
"quote_analyzer": "standard",
"analyze_wildcard": False,
"auto_generate_synonyms_phrase_query": True,
}
}
]
}
},
}
return query

186
core/lib/parsing.py
View File

@ -1,186 +0,0 @@
from datetime import datetime
from django.conf import settings
from django.core.exceptions import ValidationError
from core.models import NotificationRule
class QueryError(Exception):
pass
def parse_rule(user, query_params):
"""
Parse a rule query.
"""
if "rule" in query_params:
try:
rule_object = NotificationRule.objects.filter(id=query_params["rule"])
except ValidationError:
message = "Rule is not a valid UUID"
message_class = "danger"
return {"message": message, "class": message_class}
if not rule_object.exists():
message = "Rule does not exist"
message_class = "danger"
return {"message": message, "class": message_class}
rule_object = rule_object.first()
if not rule_object.user == user:
message = "Rule does not belong to you"
message_class = "danger"
return {"message": message, "class": message_class}
return rule_object
else:
return None
def parse_size(query_params, sizes):
if "size" in query_params:
size = query_params["size"]
if size not in sizes:
message = "Size is not permitted"
message_class = "danger"
return {"message": message, "class": message_class}
size = int(size)
else:
size = 15
return size
def parse_index(user, query_params, raise_error=False):
if "index" in query_params:
index = query_params["index"]
if index == "main":
index = settings.INDEX_MAIN
else:
if not user.has_perm(f"core.index_{index}"):
message = f"Not permitted to search by this index: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
"class": message_class,
}
if index == "meta":
index = settings.INDEX_META
elif index == "internal":
index = settings.INDEX_INT
elif index == "restricted":
if not user.has_perm("core.restricted_sources"):
message = f"Not permitted to search by this index: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
"class": message_class,
}
index = settings.INDEX_RESTRICTED
else:
message = f"Index is not valid: {index}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {
"message": message,
"class": message_class,
}
else:
index = settings.INDEX_MAIN
return index
def parse_source(user, query_params, raise_error=False):
source = None
if "source" in query_params:
source = query_params["source"]
# Validate permissions for restricted sources
if source in settings.SOURCES_RESTRICTED:
if not user.has_perm("core.restricted_sources"):
message = f"Access denied: {source}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {"message": message, "class": message_class}
# Check validity of source
elif source not in settings.MAIN_SOURCES:
message = f"Invalid source: {source}"
if raise_error:
raise QueryError(message)
message_class = "danger"
return {"message": message, "class": message_class}
if source == "all":
source = None # the next block will populate it
if source:
sources = [source]
else:
# Here we need to populate what "all" means for the user.
# They may only have access to a subset of the sources.
# We build a custom source list with ones they have access
# to, and then remove "all" from the list.
sources = list(settings.MAIN_SOURCES)
if user.has_perm("core.restricted_sources"):
# If the user can use restricted sources, add them in.
for source_iter in settings.SOURCES_RESTRICTED:
sources.append(source_iter)
# Get rid of "all", it's just a meta-source
if "all" in sources:
sources.remove("all")
return sources
def parse_sort(query_params):
sort = None
if "sorting" in query_params:
sorting = query_params["sorting"]
if sorting not in ("asc", "desc", "none"):
message = "Invalid sort"
message_class = "danger"
return {"message": message, "class": message_class}
if sorting == "asc":
sort = "ascending"
elif sorting == "desc":
sort = "descending"
return sort
def parse_date_time(query_params):
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
query_params.keys()
):
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
return (from_ts, to_ts)
return (None, None)
def parse_sentiment(query_params):
sentiment = None
if "check_sentiment" in query_params:
if "sentiment_method" not in query_params:
message = "No sentiment method"
message_class = "danger"
return {"message": message, "class": message_class}
if "sentiment" in query_params:
sentiment = query_params["sentiment"]
try:
sentiment = float(sentiment)
except ValueError:
message = "Sentiment is not a float"
message_class = "danger"
return {"message": message, "class": message_class}
sentiment_method = query_params["sentiment_method"]
return (sentiment_method, sentiment)

787
core/lib/rules.py
View File

@ -1,787 +0,0 @@
from yaml import dump, load
from yaml.parser import ParserError
from yaml.scanner import ScannerError
try:
from yaml import CDumper as Dumper
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader, Dumper
import uuid
from copy import deepcopy
from datetime import datetime
import orjson
from siphashc import siphash
from core.lib.notify import sendmsg
from core.lib.parsing import parse_index, parse_source
from core.util import logs
log = logs.get_logger("rules")
SECONDS_PER_UNIT = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
MAX_WINDOW = 2592000
MAX_AMOUNT_NTFY = 10
MAX_AMOUNT_WEBHOOK = 1000
HIGH_FREQUENCY_MIN_SEC = 60
class RuleParseError(Exception):
def __init__(self, message, field):
super().__init__(message)
self.field = field
def format_ntfy(**kwargs):
"""
Format a message for ntfy.
If the message is a list, it will be joined with newlines.
If the message is None, it will be replaced with an empty string.
If specified, `matched` will be pretty-printed in the first line.
kwargs:
rule: The rule object, must be specified
index: The index the rule matched on, can be None
message: The message to send, can be None
meta:
matched: The matched fields, can be None
total_hits: The total number of matches, optional
"""
rule = kwargs.get("rule")
index = kwargs.get("index")
message = kwargs.get("message")
meta = kwargs.get("meta", {})
total_hits = meta.get("total_hits", 0)
matched = meta.get("matched")
if message:
# Dump the message in YAML for readability
messages_formatted = ""
if isinstance(message, list):
for message_iter in message:
messages_formatted += dump(
message_iter, Dumper=Dumper, default_flow_style=False
)
messages_formatted += "\n"
else:
messages_formatted = dump(message, Dumper=Dumper, default_flow_style=False)
else:
messages_formatted = ""
if matched:
matched = ", ".join([f"{k}: {v}" for k, v in matched.items()])
else:
matched = ""
notify_message = f"{rule.name} on {index}: {matched}\n{messages_formatted}"
notify_message += f"\nTotal hits: {total_hits}"
notify_message = notify_message.encode("utf-8", "replace")
return notify_message
def format_webhook(**kwargs):
"""
Format a message for a webhook.
Adds some metadata to the message that would normally be only in
notification_settings.
Dumps the message in JSON.
kwargs:
rule: The rule object, must be specified
index: The index the rule matched on, can be None
message: The message to send, can be None, but will be sent as None
meta:
matched: The matched fields, can be None, but will be sent as None
total_hits: The total number of matches, optional
notification_settings: The notification settings, must be specified
priority: The priority of the message, optional
topic: The topic of the message, optional
"""
# rule = kwargs.get("rule")
# index = kwargs.get("index")
message = kwargs.get("message")
meta = kwargs.get("meta")
notification_settings = kwargs.get("notification_settings")
notify_message = {
"data": message,
"meta": meta,
}
if "priority" in notification_settings:
notify_message["priority"] = notification_settings["priority"]
if "topic" in notification_settings:
notify_message["topic"] = notification_settings["topic"]
notify_message = orjson.dumps(notify_message)
return notify_message
def rule_notify(rule, index, message, meta=None):
"""
Send a notification for a matching rule.
Gets the notification settings for the rule.
Runs the formatting helpers for the service.
:param rule: The rule object, must be specified
:param index: The index the rule matched on, can be None
:param message: The message to send, can be None
:param meta: dict of metadata, contains `aggs` key for the matched fields
"""
# If there is no message, don't say anything matched
if message:
word = "match"
else:
word = "no match"
title = f"Rule {rule.name} {word} on {index}"
# The user notification settings are merged in with this
notification_settings = rule.get_notification_settings()
if not notification_settings:
# No/invalid notification settings, don't send anything
return
if notification_settings.get("service") == "none":
# Don't send anything
return
# double sigh
message_copy = deepcopy(message)
for index, _ in enumerate(message_copy):
if "meta" in message_copy[index]:
del message_copy[index]["meta"]
# Create a cast we can reuse for the formatting helpers and sendmsg
cast = {
"title": title,
"user": rule.user,
"rule": rule,
"index": index,
"message": message_copy,
"notification_settings": notification_settings,
}
if meta:
cast["meta"] = meta
if rule.service == "ntfy":
cast["msg"] = format_ntfy(**cast)
elif rule.service == "webhook":
cast["msg"] = format_webhook(**cast)
sendmsg(**cast)
class NotificationRuleData(object):
def __init__(self, user, cleaned_data, db):
self.user = user
self.object = None
# We are running live and have been passed a database object
if not isinstance(cleaned_data, dict):
self.object = cleaned_data
cleaned_data = cleaned_data.__dict__
self.cleaned_data = cleaned_data
self.db = db
self.data = self.cleaned_data.get("data")
self.window = self.cleaned_data.get("window")
self.policy = self.cleaned_data.get("policy")
self.parsed = None
self.aggs = {}
self.validate_user_permissions()
self.parse_data()
self.ensure_list()
self.validate_permissions()
self.validate_schedule_fields()
self.validate_time_fields()
if self.object is not None:
self.populate_matched()
def clear_database_matches(self):
"""
Delete all matches for this rule.
"""
rule_id = str(self.object.id)
self.db.delete_rule_entries(rule_id)
def populate_matched(self):
"""
On first creation, the match field is None. We need to populate it with
a dictionary containing the index names as keys and False as values.
"""
if self.object.match is None:
self.object.match = {}
for index in self.parsed["index"]:
if index not in self.object.match:
self.object.match[index] = False
self.object.save()
def format_matched(self, messages):
matched = {}
for message in messages:
for field, value in self.parsed.items():
if field == "msg":
# Allow partial matches for msg
for msg in value:
if "msg" in message:
if msg.lower() in message["msg"].lower():
matched[field] = msg
# Break out of the msg matching loop
break
# Continue to next field
continue
if field == "tokens":
# Allow partial matches for tokens
for token in value:
if "tokens" in message:
if token.lower() in [x.lower() for x in message["tokens"]]:
matched[field] = token
# Break out of the token matching loop
break
# Continue to next field
continue
if field in message and message[field] in value:
# Do exact matches for all other fields
matched[field] = message[field]
return matched
def store_match(self, index, match):
"""
Store a match result.
Accepts None for the index to set all indices.
:param index: the index to store the match for, can be None
:param match: the object that matched
"""
if match is not False:
# Dump match to JSON while sorting the keys
match_normalised = orjson.dumps(match, option=orjson.OPT_SORT_KEYS)
match = siphash(self.db.hash_key, match_normalised)
if self.object.match is None:
self.object.match = {}
if not isinstance(self.object.match, dict):
self.object.match = {}
if index is None:
for index_iter in self.parsed["index"]:
self.object.match[index_iter] = match
else:
self.object.match[index] = match
self.object.save()
log.debug(f"Stored match: {index} - {match}")
def get_match(self, index=None, match=None):
"""
Get a match result for an index.
If the index is None, it will return True if any index has a match.
:param index: the index to get the match for, can be None
"""
if self.object.match is None:
self.object.match = {}
self.object.save()
return None
if not isinstance(self.object.match, dict):
return None
if index is None:
# Check if we have any matches on all indices
values = self.object.match.values()
if not values:
return None
return any(values)
# Check if it's the same hash
if match is not None:
match_normalised = orjson.dumps(match, option=orjson.OPT_SORT_KEYS)
match = siphash(self.db.hash_key, match_normalised)
hash_matches = self.object.match.get(index) == match
return hash_matches
returned_match = self.object.match.get(index, None)
if type(returned_match) == int:
# We are getting a hash from the database,
# but we have nothing to check it against.
# In this instance, we are checking if we got a match
# at all last time. We can confidently say that since
# we have a hash, we did.
returned_match = True
return returned_match
def format_aggs(self, aggs):
"""
Format aggregations for the query.
We have self.aggs, which contains:
{"avg_sentiment": (">", 0.5)}
and aggs, which contains:
{"avg_sentiment": {"value": 0.6}}
It's matched already, we just need to format it like so:
{"avg_sentiment": "0.06>0.5"}
:param aggs: the aggregations to format
:return: the formatted aggregations
"""
new_aggs = {}
for agg_name, agg in aggs.items():
if agg_name in self.aggs:
op, value = self.aggs[agg_name]
new_aggs[agg_name] = f"{agg['value']}{op}{value}"
return new_aggs
def reform_matches(self, index, matches, meta, mode):
if not isinstance(matches, list):
matches = [matches]
matches_copy = matches.copy()
match_ts = datetime.utcnow().isoformat()
batch_id = uuid.uuid4()
# Filter empty fields in meta
meta = {k: v for k, v in meta.items() if v}
for match_index, _ in enumerate(matches_copy):
matches_copy[match_index]["index"] = index
matches_copy[match_index]["rule_id"] = str(self.object.id)
matches_copy[match_index]["meta"] = meta
matches_copy[match_index]["match_ts"] = match_ts
matches_copy[match_index]["mode"] = mode
matches_copy[match_index]["batch_id"] = str(batch_id)
return matches_copy
async def ingest_matches(self, index, matches, meta, mode):
"""
Store all matches for an index.
:param index: the index to store the matches for
:param matches: the matches to store
"""
# new_matches = self.reform_matches(index, matches, meta, mode)
if self.object.ingest:
await self.db.async_store_matches(matches)
def ingest_matches_sync(self, index, matches, meta, mode):
"""
Store all matches for an index.
:param index: the index to store the matches for
:param matches: the matches to store
"""
# new_matches = self.reform_matches(index, matches, meta, mode)
if self.object.ingest:
self.db.store_matches(matches)
async def rule_matched(self, index, message, meta, mode):
"""
A rule has matched.
If the previous run did not match, send a notification after formatting
the aggregations.
:param index: the index the rule matched on
:param message: the message object that matched
:param aggs: the aggregations that matched
"""
current_match = self.get_match(index, message)
log.debug(f"Rule matched: {index} - current match: {current_match}")
last_run_had_matches = current_match is True
if self.policy in ["change", "default"]:
# Change or Default policy, notifying only on new results
if last_run_had_matches:
# Last run had matches, and this one did too
# We don't need to notify
return
elif self.policy == "always":
# Only here for completeness, we notify below by default
pass
# We hit the return above if we don't need to notify
if "matched" not in meta:
meta["matched"] = self.format_matched(message)
if "aggs" in meta:
aggs_formatted = self.format_aggs(meta["aggs"])
if aggs_formatted:
meta["matched_aggs"] = aggs_formatted
meta["is_match"] = True
self.store_match(index, message)
message = self.reform_matches(index, message, meta, mode)
rule_notify(self.object, index, message, meta)
await self.ingest_matches(index, message, meta, mode)
def rule_matched_sync(self, index, message, meta, mode):
"""
A rule has matched.
If the previous run did not match, send a notification after formatting
the aggregations.
:param index: the index the rule matched on
:param message: the message object that matched
:param aggs: the aggregations that matched
"""
current_match = self.get_match(index, message)
log.debug(f"Rule matched: {index} - current match: {current_match}")
last_run_had_matches = current_match is True
if self.policy in ["change", "default"]:
# Change or Default policy, notifying only on new results
if last_run_had_matches:
# Last run had matches, and this one did too
# We don't need to notify
return
elif self.policy == "always":
# Only here for completeness, we notify below by default
pass
# We hit the return above if we don't need to notify
if "matched" not in meta:
meta["matched"] = self.format_matched(message)
if "aggs" in meta:
aggs_formatted = self.format_aggs(meta["aggs"])
if aggs_formatted:
meta["matched_aggs"] = aggs_formatted
meta["is_match"] = True
self.store_match(index, message)
message = self.reform_matches(index, message, meta, mode)
rule_notify(self.object, index, message, meta)
self.ingest_matches_sync(index, message, meta, mode)
# No async helper for this one as we only need it for schedules
async def rule_no_match(self, index=None, message=None, mode=None):
"""
A rule has not matched.
If the previous run did match, send a notification if configured to notify
for empty matches.
:param index: the index the rule did not match on, can be None
"""
current_match = self.get_match(index)
log.debug(
f"Rule not matched: {index} - current match: {current_match}: {message}"
)
last_run_had_matches = current_match is True
initial = current_match is None
self.store_match(index, False)
if self.policy != "always":
# We hit the return above if we don't need to notify
if self.policy in ["change", "default"]:
if not last_run_had_matches and not initial:
# We don't need to notify if the last run didn't have matches
return
if self.policy in ["always", "change"]:
# Never notify for empty matches on default policy
meta = {"msg": message, "is_match": False}
matches = [{"msg": None}]
message = self.reform_matches(index, matches, meta, mode)
rule_notify(self.object, index, matches, meta)
await self.ingest_matches(
index=index,
matches=matches,
meta=meta,
mode="schedule",
)
async def run_schedule(self):
"""
Run the schedule query.
Get the results from the database, and check if the rule has matched.
Check if all of the required aggregations have matched.
"""
response = await self.db.schedule_query_results(self)
if not response:
# No results in the result_map
await self.rule_no_match(
message="No response from database", mode="schedule"
)
return
for index, (meta, results) in response.items():
if not results:
# Falsy results, no matches
await self.rule_no_match(
index, message="No results for index", mode="schedule"
)
continue
# Add the match values of all aggregations to a list
aggs_for_index = []
for agg_name in self.aggs.keys():
if agg_name in meta["aggs"]:
if "match" in meta["aggs"][agg_name]:
aggs_for_index.append(meta["aggs"][agg_name]["match"])
# All required aggs are present
if len(aggs_for_index) == len(self.aggs.keys()):
if all(aggs_for_index):
# All aggs have matched
await self.rule_matched(
index, results[: self.object.amount], meta, mode="schedule"
)
continue
# Default branch, since the happy path has a continue keyword
await self.rule_no_match(
index, message="Aggregation did not match", mode="schedule"
)
def test_schedule(self):
"""
Test the schedule query to ensure it is valid.
Raises an exception if the query is invalid.
"""
if self.db:
self.db.schedule_query_results_test_sync(self)
def validate_schedule_fields(self):
"""
Ensure schedule fields are valid.
index: can be a list, it will schedule one search per index.
source: can be a list, it will be the filter for each search.
tokens: can be list, it will ensure the message matches any token.
msg: can be a list, it will ensure the message contains any msg.
No other fields can be lists containing more than one item.
:raises RuleParseError: if the fields are invalid
"""
is_schedule = self.is_schedule
if is_schedule:
allowed_list_fields = ["index", "source", "tokens", "msg"]
for field, value in self.parsed.items():
if field not in allowed_list_fields:
if len(value) > 1:
raise RuleParseError(
(
f"For scheduled rules, field {field} cannot contain "
"more than one item"
),
"data",
)
if len(str(value[0])) == 0:
raise RuleParseError(f"Field {field} cannot be empty", "data")
if "sentiment" in self.parsed:
sentiment = str(self.parsed["sentiment"][0])
sentiment = sentiment.strip()
if sentiment[0] not in [">", "<", "="]:
raise RuleParseError(
(
"Sentiment field must be a comparison operator and then a "
"float: >0.02"
),
"data",
)
operator = sentiment[0]
number = sentiment[1:]
try:
number = float(number)
except ValueError:
raise RuleParseError(
(
"Sentiment field must be a comparison operator and then a "
"float: >0.02"
),
"data",
)
self.aggs["avg_sentiment"] = (operator, number)
else:
if "query" in self.parsed:
raise RuleParseError(
"Field query cannot be used with on-demand rules", "data"
)
if "tags" in self.parsed:
raise RuleParseError(
"Field tags cannot be used with on-demand rules", "data"
)
if self.policy != "default":
raise RuleParseError(
(
f"Cannot use {self.cleaned_data['policy']} policy with "
"on-demand rules"
),
"policy",
)
@property
def is_schedule(self):
"""
Check if the rule is a schedule rule.
:return: True if the rule is a schedule rule, False otherwise
"""
if "interval" in self.cleaned_data:
if self.cleaned_data["interval"] != 0:
return True
return False
def ensure_list(self):
"""
Ensure all values in the data field are lists.
Convert all strings to lists with one item.
"""
for field, value in self.parsed.items():
if not isinstance(value, list):
self.parsed[field] = [value]
def validate_user_permissions(self):
"""
Ensure the user can use notification rules.
:raises RuleParseError: if the user does not have permission
"""
if not self.user.has_perm("core.use_rules"):
raise RuleParseError("User does not have permission to use rules", "data")
def validate_time_fields(self):
"""
Validate the interval and window fields.
Prohibit window being specified with an ondemand interval.
Prohibit window not being specified with a non-ondemand interval.
Prohibit amount being specified with an on-demand interval.
Prohibut amount not being specified with a non-ondemand interval.
Validate window field.
Validate window unit and enforce maximum.
:raises RuleParseError: if the fields are invalid
"""
interval = self.cleaned_data.get("interval")
window = self.cleaned_data.get("window")
amount = self.cleaned_data.get("amount")
service = self.cleaned_data.get("service")
on_demand = interval == 0
# Not on demand and interval is too low
if not on_demand and interval <= HIGH_FREQUENCY_MIN_SEC:
if not self.user.has_perm("core.rules_high_frequency"):
raise RuleParseError(
"User does not have permission to use high frequency rules", "data"
)
if not on_demand:
if not self.user.has_perm("core.rules_scheduled"):
raise RuleParseError(
"User does not have permission to use scheduled rules", "data"
)
if on_demand and window is not None:
# Interval is on demand and window is specified
# We can't have a window with on-demand rules
raise RuleParseError(
"Window cannot be specified with on-demand interval", "window"
)
if not on_demand and window is None:
# Interval is not on demand and window is not specified
# We can't have a non-on-demand interval without a window
raise RuleParseError(
"Window must be specified with non-on-demand interval", "window"
)
if not on_demand and amount is None:
# Interval is not on demand and amount is not specified
# We can't have a non-on-demand interval without an amount
raise RuleParseError(
"Amount must be specified with non-on-demand interval", "amount"
)
if on_demand and amount is not None:
# Interval is on demand and amount is specified
# We can't have an amount with on-demand rules
raise RuleParseError(
"Amount cannot be specified with on-demand interval", "amount"
)
if window is not None:
window_number = window[:-1]
if not window_number.isdigit():
raise RuleParseError("Window prefix must be a number", "window")
window_number = int(window_number)
window_unit = window[-1]
if window_unit not in SECONDS_PER_UNIT:
raise RuleParseError(
(
"Window unit must be one of "
f"{', '.join(SECONDS_PER_UNIT.keys())},"
f" not '{window_unit}'"
),
"window",
)
window_seconds = window_number * SECONDS_PER_UNIT[window_unit]
if window_seconds > MAX_WINDOW:
raise RuleParseError(
f"Window cannot be larger than {MAX_WINDOW} seconds (30 days)",
"window",
)
if amount is not None:
if service == "ntfy":
if amount > MAX_AMOUNT_NTFY:
raise RuleParseError(
f"Amount cannot be larger than {MAX_AMOUNT_NTFY} for ntfy",
"amount",
)
else:
if amount > MAX_AMOUNT_WEBHOOK:
raise RuleParseError(
(
f"Amount cannot be larger than {MAX_AMOUNT_WEBHOOK} for "
f"{service}"
),
"amount",
)
def validate_permissions(self):
"""
Validate permissions for the source and index variables.
Also set the default values for the user if not present.
Stores the default or expanded values in the parsed field.
:raises QueryError: if the user does not have permission to use the source
"""
if "index" in self.parsed:
index = self.parsed["index"]
if type(index) == list:
for i in index:
parse_index(self.user, {"index": i}, raise_error=True)
# else:
# db.parse_index(self.user, {"index": index}, raise_error=True)
else:
# Get the default value for the user if not present
index = parse_index(self.user, {}, raise_error=True)
self.parsed["index"] = [index]
if "source" in self.parsed:
source = self.parsed["source"]
if type(source) == list:
for i in source:
parse_source(self.user, {"source": i}, raise_error=True)
# else:
# parse_source(self.user, {"source": source}, raise_error=True)
else:
# Get the default value for the user if not present
source = parse_source(self.user, {}, raise_error=True)
self.parsed["source"] = source
def parse_data(self):
"""
Parse the data in the text field to YAML.
:raises RuleParseError: if the data is invalid
"""
try:
self.parsed = load(self.data, Loader=Loader)
except (ScannerError, ParserError) as e:
raise RuleParseError(f"Invalid YAML: {e}", "data")
def __str__(self):
"""
Get a YAML representation of the data field of the rule.
"""
return dump(self.parsed, Dumper=Dumper)
def get_data(self):
"""
Return the data field as a dictionary.
"""
return self.parsed

0
core/management/__init__.py
View File

0
core/management/commands/__init__.py
View File

107
core/management/commands/processing.py
View File

@ -1,107 +0,0 @@
import msgpack
from django.core.management.base import BaseCommand
from redis import StrictRedis
from core.db.storage import db
from core.lib.rules import NotificationRuleData
from core.models import NotificationRule
from core.util import logs
log = logs.get_logger("processing")
def process_rules(data):
all_rules = NotificationRule.objects.filter(enabled=True, interval=0)
for index, index_messages in data.items():
for message in index_messages:
for rule in all_rules:
# Quicker helper to get the data without spinning
# up a NotificationRuleData object
parsed_rule = rule.parse()
matched = {}
# Rule is invalid, this shouldn't happen
if "index" not in parsed_rule:
continue
if "source" not in parsed_rule:
continue
rule_index = parsed_rule["index"]
rule_source = parsed_rule["source"]
# if not type(rule_index) == list:
# rule_index = [rule_index]
# if not type(rule_source) == list:
# rule_source = [rule_source]
if index not in rule_index:
# We don't care about this index, go to the next one
continue
if message["src"] not in rule_source:
# We don't care about this source, go to the next one
continue
matched["index"] = index
matched["source"] = message["src"]
rule_field_length = len(parsed_rule.keys())
matched_field_number = 0
for field, value in parsed_rule.items():
# if not type(value) == list:
# value = [value]
if field == "src":
# We already checked this
continue
if field == "tokens":
# Check if tokens are in the rule
# We only check if *at least one* token matches
for token in value:
if "tokens" in message:
if token in message["tokens"]:
matched_field_number += 1
matched[field] = token
# Break out of the token matching loop
break
# Continue to next field
continue
if field == "msg":
# Allow partial matches for msg
for msg in value:
if "msg" in message:
if msg.lower() in message["msg"].lower():
matched_field_number += 1
matched[field] = msg
# Break out of the msg matching loop
break
# Continue to next field
continue
if field in message and message[field] in value:
# Do exact matches for all other fields
matched_field_number += 1
matched[field] = message[field]
# Subtract 2, 1 for source and 1 for index
if matched_field_number == rule_field_length - 2:
meta = {"matched": matched, "total_hits": 1}
# Parse the rule, we saved some work above to avoid doing this,
# but it makes delivering messages significantly easier as we can
# use the same code as for scheduling.
rule_data_object = NotificationRuleData(rule.user, rule, db=db)
# rule_notify(rule, index, message, meta=meta)
rule_data_object.rule_matched_sync(
index, message, meta=meta, mode="ondemand"
)
class Command(BaseCommand):
def handle(self, *args, **options):
r = StrictRedis(unix_socket_path="/var/run/socks/redis.sock", db=0)
p = r.pubsub()
p.psubscribe("messages")
for message in p.listen():
if message:
if message["channel"] == b"messages":
data = message["data"]
try:
unpacked = msgpack.unpackb(data, raw=False)
except TypeError:
continue
process_rules(unpacked)

54
core/management/commands/scheduling.py
View File

@ -1,54 +0,0 @@
import asyncio
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from asgiref.sync import sync_to_async
from django.core.management.base import BaseCommand
from core.db.storage import db
from core.lib.parsing import QueryError
from core.lib.rules import NotificationRuleData, RuleParseError
from core.models import NotificationRule
from core.util import logs
log = logs.get_logger("scheduling")
INTERVALS = [5, 60, 900, 1800, 3600, 14400, 86400]
async def job(interval_seconds):
"""
Run all schedules matching the given interval.
:param interval_seconds: The interval to run.
"""
matching_rules = await sync_to_async(list)(
NotificationRule.objects.filter(enabled=True, interval=interval_seconds)
)
for rule in matching_rules:
log.debug(f"Running rule {rule}")
try:
rule = NotificationRuleData(rule.user, rule, db=db)
await rule.run_schedule()
# results = await db.schedule_query_results(rule.user, rule)
except QueryError as e:
log.error(f"Error running rule {rule}: {e}")
except RuleParseError as e:
log.error(f"Error parsing rule {rule}: {e}")
class Command(BaseCommand):
def handle(self, *args, **options):
"""
Start the scheduling process.
"""
scheduler = AsyncIOScheduler()
for interval in INTERVALS:
log.debug(f"Scheduling {interval} second job")
scheduler.add_job(job, "interval", seconds=interval, args=[interval])
scheduler.start()
loop = asyncio.get_event_loop()
try:
loop.run_forever()
except (KeyboardInterrupt, SystemExit):
log.info("Process terminating")
finally:
loop.close()

22
core/migrations/0007_perms.py
View File

@ -1,22 +0,0 @@
# Generated by Django 4.0.6 on 2022-08-16 18:04
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0006_contentblock_page'),
]
operations = [
migrations.CreateModel(
name='Perms',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
],
options={
'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord')),
},
),
]

17
core/migrations/0008_alter_perms_options.py
View File

@ -1,17 +0,0 @@
# Generated by Django 4.0.6 on 2022-08-27 11:50
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0007_perms'),
]
operations = [
migrations.AlterModelOptions(
name='perms',
options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))},
),
]

17
core/migrations/0009_alter_perms_options.py
View File

@ -1,17 +0,0 @@
# Generated by Django 4.0.6 on 2022-08-27 12:05
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0008_alter_perms_options'),
]
operations = [
migrations.AlterModelOptions(
name='perms',
options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))},
),
]

17
core/migrations/0010_alter_perms_options.py
View File

@ -1,17 +0,0 @@
# Generated by Django 4.1 on 2022-09-01 11:07
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0009_alter_perms_options'),
]
operations = [
migrations.AlterModelOptions(
name='perms',
options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('restricted_sources', 'Can access restricted sources'))},
),
]

17
core/migrations/0011_alter_perms_options.py
View File

@ -1,17 +0,0 @@
# Generated by Django 4.1.3 on 2022-11-29 12:04
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0010_alter_perms_options'),
]
operations = [
migrations.AlterModelOptions(
name='perms',
options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
),
]

25
core/migrations/0012_notificationrule.py
View File

@ -1,25 +0,0 @@
# Generated by Django 4.1.3 on 2023-01-12 15:12
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0011_alter_perms_options'),
]
operations = [
migrations.CreateModel(
name='NotificationRule',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=255)),
('enabled', models.BooleanField(default=True)),
('data', models.TextField()),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
),
]

24
core/migrations/0013_notificationsettings.py
View File

@ -1,24 +0,0 @@
# Generated by Django 4.1.3 on 2023-01-12 15:25
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0012_notificationrule'),
]
operations = [
migrations.CreateModel(
name='NotificationSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('ntfy_topic', models.CharField(blank=True, max_length=255, null=True)),
('ntfy_url', models.CharField(blank=True, max_length=255, null=True)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
),
]

18
core/migrations/0014_notificationrule_priority.py
View File

@ -1,18 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-12 18:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0013_notificationsettings'),
]
operations = [
migrations.AddField(
model_name='notificationrule',
name='priority',
field=models.IntegerField(choices=[(1, 'min'), (2, 'low'), (3, 'default'), (4, 'high'), (5, 'max')], default=1),
),
]

18
core/migrations/0015_notificationrule_topic.py
View File

@ -1,18 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-12 18:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0014_notificationrule_priority'),
]
operations = [
migrations.AddField(
model_name='notificationrule',
name='topic',
field=models.CharField(blank=True, max_length=255, null=True),
),
]

23
core/migrations/0016_notificationrule_interval_notificationrule_window.py
View File

@ -1,23 +0,0 @@
# Generated by Django 4.1.3 on 2023-01-14 14:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0015_notificationrule_topic'),
]
operations = [
migrations.AddField(
model_name='notificationrule',
name='interval',
field=models.CharField(choices=[('ondemand', 'On demand'), ('minute', 'Every minute'), ('15m', 'Every 15 minutes'), ('30m', 'Every 30 minutes'), ('hour', 'Every hour'), ('4h', 'Every 4 hours'), ('day', 'Every day'), ('week', 'Every week'), ('month', 'Every month')], default='ondemand', max_length=255),
),
migrations.AddField(
model_name='notificationrule',
name='window',
field=models.CharField(blank=True, max_length=255, null=True),
),
]

18
core/migrations/0017_alter_notificationrule_interval.py
View File

@ -1,18 +0,0 @@
# Generated by Django 4.1.3 on 2023-01-14 14:54
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0016_notificationrule_interval_notificationrule_window'),
]
operations = [
migrations.AlterField(
model_name='notificationrule',
name='interval',
field=models.IntegerField(choices=[(0, 'On demand'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=0),
),
]

27
core/migrations/0018_alter_perms_options_notificationrule_match_and_more.py
View File

@ -1,27 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-15 00:58
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0017_alter_notificationrule_interval'),
]
operations = [
migrations.AlterModelOptions(
name='perms',
options={'permissions': (('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('use_insights', 'Can use the Insights page'), ('use_rules', 'Can use the Rules page'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
),
migrations.AddField(
model_name='notificationrule',
name='match',
field=models.BooleanField(default=False),
),
migrations.AlterField(
model_name='notificationrule',
name='interval',
field=models.IntegerField(choices=[(0, 'On demand'), (5, 'Every 5 seconds'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=0),
),
]

18
core/migrations/0019_alter_notificationrule_match.py
View File

@ -1,18 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-15 01:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0018_alter_perms_options_notificationrule_match_and_more'),
]
operations = [
migrations.AlterField(
model_name='notificationrule',
name='match',
field=models.JSONField(blank=True, null=True),
),
]

42
core/migrations/0020_rename_ntfy_topic_notificationsettings_topic_and_more.py
View File

@ -1,42 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-15 18:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0019_alter_notificationrule_match'),
]
operations = [
migrations.RenameField(
model_name='notificationsettings',
old_name='ntfy_topic',
new_name='topic',
),
migrations.RemoveField(
model_name='notificationsettings',
name='ntfy_url',
),
migrations.AddField(
model_name='notificationrule',
name='service',
field=models.CharField(blank=True, choices=[('ntfy', 'NTFY'), ('wehbook', 'Custom webhook')], max_length=255, null=True),
),
migrations.AddField(
model_name='notificationrule',
name='url',
field=models.CharField(blank=True, max_length=1024, null=True),
),
migrations.AddField(
model_name='notificationsettings',
name='service',
field=models.CharField(blank=True, choices=[('ntfy', 'NTFY'), ('wehbook', 'Custom webhook')], max_length=255, null=True),
),
migrations.AddField(
model_name='notificationsettings',
name='url',
field=models.CharField(blank=True, max_length=1024, null=True),
),
]

28
core/migrations/0021_notificationrule_amount_and_more.py
View File

@ -1,28 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-15 20:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0020_rename_ntfy_topic_notificationsettings_topic_and_more'),
]
operations = [
migrations.AddField(
model_name='notificationrule',
name='amount',
field=models.IntegerField(blank=True, default=1, null=True),
),
migrations.AlterField(
model_name='notificationrule',
name='service',
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook')], default='ntfy', max_length=255),
),
migrations.AlterField(
model_name='notificationsettings',
name='service',
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook')], default='ntfy', max_length=255),
),
]

23
core/migrations/0022_notificationrule_send_empty_and_more.py
View File

@ -1,23 +0,0 @@
# Generated by Django 4.1.5 on 2023-01-15 23:34
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0021_notificationrule_amount_and_more'),
]
operations = [
migrations.AddField(
model_name='notificationrule',
name='send_empty',
field=models.BooleanField(default=False),
),
migrations.AlterField(
model_name='notificationrule',
name='amount',
field=models.PositiveIntegerField(blank=True, default=1, null=True),
),
]

17
core/migrations/0023_alter_perms_options.py
View File

@ -1,17 +0,0 @@
# Generated by Django 4.1.5 on 2023-02-02 19:07
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0022_notificationrule_send_empty_and_more'),
]
operations = [
migrations.AlterModelOptions(
name='perms',
options={'permissions': (('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('use_insights', 'Can use the Insights page'), ('use_rules', 'Can use the Rules page'), ('rules_scheduled', 'Can use the scheduled rules'), ('rules_high_frequency', 'Can use the high frequency rules'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
),
]

20
core/migrations/0024_alter_notificationrule_id.py
View File

@ -1,20 +0,0 @@
# Generated by Django 4.1.5 on 2023-02-02 19:08
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0023_alter_perms_options'),
]
operations = [
migrations.AlterField(
model_name='notificationrule',
name='id',
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
),
]

20
core/migrations/0025_alter_notificationrule_id.py
View File

@ -1,20 +0,0 @@
# Generated by Django 4.1.5 on 2023-02-02 19:35
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0024_alter_notificationrule_id'),
]
operations = [
migrations.AlterField(
model_name='notificationrule',
name='id',
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
),
]

28
core/migrations/0026_notificationrule_policy_and_more.py
View File

@ -1,28 +0,0 @@
# Generated by Django 4.1.5 on 2023-02-09 14:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0025_alter_notificationrule_id'),
]
operations = [
migrations.AddField(
model_name='notificationrule',
name='policy',
field=models.CharField(choices=[('default', 'Only trigger for matched events'), ('change', 'Trigger only if no results found when they were last run'), ('always', 'Always trigger regardless of whether results are found')], default='default', max_length=255),
),
migrations.AlterField(
model_name='notificationrule',
name='service',
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook'), ('none', 'Disabled')], default='ntfy', max_length=255),
),
migrations.AlterField(
model_name='notificationsettings',
name='service',
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook'), ('none', 'Disabled')], default='ntfy', max_length=255),
),
]

28
core/migrations/0027_alter_notificationrule_policy_and_more.py
View File

@ -1,28 +0,0 @@
# Generated by Django 4.1.6 on 2023-02-13 10:18
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0026_notificationrule_policy_and_more'),
]
operations = [
migrations.AlterField(
model_name='notificationrule',
name='policy',
field=models.CharField(choices=[('default', 'Default: Trigger only when there were no results last time'), ('change', 'Change: Default + trigger when there are no results (if there were before)'), ('always', 'Always: Trigger on every run (not recommended for low intervals)')], default='default', max_length=255),
),
migrations.AlterField(
model_name='notificationrule',
name='topic',
field=models.CharField(blank=True, max_length=2048, null=True),
),
migrations.AlterField(
model_name='notificationsettings',
name='topic',
field=models.CharField(blank=True, max_length=2048, null=True),
),
]

33
core/migrations/0028_rename_send_empty_notificationrule_ingest_and_more.py
View File

@ -1,33 +0,0 @@
# Generated by Django 4.1.6 on 2023-02-13 21:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0027_alter_notificationrule_policy_and_more'),
]
operations = [
migrations.RenameField(
model_name='notificationrule',
old_name='send_empty',
new_name='ingest',
),
migrations.AlterField(
model_name='notificationrule',
name='interval',
field=models.IntegerField(choices=[(0, 'On demand'), (5, 'Every 5 seconds'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=60),
),
migrations.AlterField(
model_name='notificationrule',
name='service',
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook'), ('none', 'Disabled')], default='webhook', max_length=255),
),
migrations.AlterField(
model_name='notificationrule',
name='window',
field=models.CharField(blank=True, default='30d', max_length=255, null=True),
),
]

155
core/models.py
View File

@ -1,56 +1,13 @@
import logging
import uuid
import stripe
from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.db import models
from yaml import load
from yaml.parser import ParserError
from yaml.scanner import ScannerError
from core.lib.customers import get_or_create, update_customer_fields
try:
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
logger = logging.getLogger(__name__)
PRIORITY_CHOICES = (
(1, "min"),
(2, "low"),
(3, "default"),
(4, "high"),
(5, "max"),
)
INTERVAL_CHOICES = (
(0, "On demand"),
(5, "Every 5 seconds"),
(60, "Every minute"),
(900, "Every 15 minutes"),
(1800, "Every 30 minutes"),
(3600, "Every hour"),
(14400, "Every 4 hours"),
(86400, "Every day"),
)
SERVICE_CHOICES = (
("ntfy", "NTFY"),
("webhook", "Custom webhook"),
("none", "Disabled"),
)
POLICY_CHOICES = (
("default", "Default: Trigger only when there were no results last time"),
(
"change",
"Change: Default + trigger when there are no results (if there were before)",
),
("always", "Always: Trigger on every run (not recommended for low intervals)"),
)
class Plan(models.Model):
name = models.CharField(max_length=255, unique=True)
@ -103,28 +60,6 @@ class User(AbstractUser):
plan_list = [plan.name for plan in self.plans.all()]
return plan in plan_list
def get_notification_settings(self, check=True):
sets = NotificationSettings.objects.get_or_create(user=self)[0]
if check:
if sets.service == "ntfy" and sets.topic is None:
return None
if sets.service == "webhook" and sets.url is None:
return None
return sets
@property
def allowed_indices(self):
indices = [settings.INDEX_MAIN]
if self.has_perm("core.index_meta"):
indices.append(settings.INDEX_META)
if self.has_perm("core.index_internal"):
indices.append(settings.INDEX_INT)
if self.has_perm("core.index_restricted"):
if self.has_perm("core.restricted_sources"):
indices.append(settings.INDEX_RESTRICTED)
return indices
class Session(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE)
@ -167,93 +102,3 @@ class ContentBlock(models.Model):
self.image3 = None
super().save(*args, **kwargs)
class Perms(models.Model):
class Meta:
permissions = (
("post_irc", "Can post to IRC"),
("post_discord", "Can post to Discord"),
("use_insights", "Can use the Insights page"),
("use_rules", "Can use the Rules page"),
("rules_scheduled", "Can use the scheduled rules"),
("rules_high_frequency", "Can use the high frequency rules"),
("index_internal", "Can use the internal index"),
("index_meta", "Can use the meta index"),
("index_restricted", "Can use the restricted index"),
("restricted_sources", "Can access restricted sources"),
)
class NotificationRule(models.Model):
id = models.UUIDField(
default=uuid.uuid4, primary_key=True, editable=False, unique=True
)
user = models.ForeignKey(User, on_delete=models.CASCADE)
name = models.CharField(max_length=255)
priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1)
topic = models.CharField(max_length=2048, null=True, blank=True)
url = models.CharField(max_length=1024, null=True, blank=True)
interval = models.IntegerField(choices=INTERVAL_CHOICES, default=60)
window = models.CharField(max_length=255, default="30d", null=True, blank=True)
amount = models.PositiveIntegerField(default=1, null=True, blank=True)
enabled = models.BooleanField(default=True)
data = models.TextField()
match = models.JSONField(null=True, blank=True)
service = models.CharField(
choices=SERVICE_CHOICES, max_length=255, default="webhook"
)
ingest = models.BooleanField(default=False)
policy = models.CharField(choices=POLICY_CHOICES, max_length=255, default="default")
def __str__(self):
return f"{self.user} - {self.name}"
def parse(self):
try:
parsed = load(self.data, Loader=Loader)
except (ScannerError, ParserError) as e:
raise ValueError(f"Invalid YAML: {e}")
return parsed
@property
def matches(self):
"""
Get the total number of matches for this rule.
"""
if isinstance(self.match, dict):
truthy_values = [x for x in self.match.values() if x is not False]
return f"{len(truthy_values)}/{len(self.match)}"
def get_notification_settings(self, check=True):
"""
Get the notification settings for this rule.
Notification rule settings take priority.
"""
user_settings = self.user.get_notification_settings(check=False)
user_settings = user_settings.__dict__
if self.priority is not None:
user_settings["priority"] = str(self.priority)
if self.topic is not None:
user_settings["topic"] = self.topic
if self.url is not None:
user_settings["url"] = self.url
if self.service is not None:
user_settings["service"] = self.service
if check:
if user_settings["service"] == "ntfy" and user_settings["topic"] is None:
return None
if user_settings["service"] == "webhook" and user_settings["url"] is None:
return None
return user_settings
class NotificationSettings(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
topic = models.CharField(max_length=2048, null=True, blank=True)
url = models.CharField(max_length=1024, null=True, blank=True)
service = models.CharField(choices=SERVICE_CHOICES, max_length=255, default="ntfy")
def __str__(self):
return f"Notification settings for {self.user}"

BIN
core/static/bitcoin-sentiment.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 25 KiB

BIN
core/static/chart-bitcoin.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 29 KiB

16
core/static/chart.js
View File

@ -4,7 +4,7 @@ function loadJson(selector) {
var jsonData = loadJson('#jsonData');
var full_data = jsonData.map((item) => item);
var ctx = document.getElementById('sentiment-chart').getContext("2d");
var ctx = document.getElementById('volume').getContext("2d");
new Chart(ctx, {
type: 'line',
data: {
@ -30,18 +30,14 @@ new Chart(ctx, {
plugins: {
tooltip: {
callbacks: {
beforeFooter: function(context) {
return "Nick: " + full_data[context[0].dataIndex].nick;
},
footer: function(context) {
var foot = "Text: " + full_data[context[0].dataIndex].text + "\n";
foot += "Nick: " + full_data[context[0].dataIndex].nick + "\n";
foot += "Channel: " + full_data[context[0].dataIndex].channel + "\n";
foot += "Net: " + full_data[context[0].dataIndex].net;
return foot;
return "Msg: " + full_data[context[0].dataIndex].text;
}
}
}
},
legend: {
display: false,
},
}
}
});

2
core/static/css/bulma-slider.min.css vendored
View File

File diff suppressed because one or more lines are too long

1
core/static/css/bulma-switch.min.css vendored
View File

File diff suppressed because one or more lines are too long

2
core/static/css/bulma-tooltip.min.css vendored
View File

File diff suppressed because one or more lines are too long

1
core/static/css/gridstack.min.css vendored
View File

File diff suppressed because one or more lines are too long

259
core/static/js/column-shifter.js
View File

@ -1,259 +0,0 @@
// Author: Grzegorz Tężycki
$(document).ready(function(){
// In web storage is saved structure like that:
// localStorage['django_tables2_column_shifter'] = {
// 'table_class_container1' : {
// 'id' : 'on',
// 'col1' : 'off',
// 'col2' : 'on',
// 'col3' : 'on',
// },
// 'table_class_container2' : {
// 'id' : 'on',
// 'col1' : 'on'
// },
// }
// main name for key in web storage
var COLUMN_SHIFTER_STORAGE_ACCESOR = "django_tables2_column_shifter";
// Return storage structure for shifter
// If structure does'n exist in web storage
// will be return empty object
var get_column_shifter_storage = function(){
var storage = localStorage.getItem(COLUMN_SHIFTER_STORAGE_ACCESOR);
if (storage === null) {
storage = {
"drilldown-table": {
"date": "off",
"time": "off",
"id": "off",
"host": "off",
"ident": "off",
"channel": "off",
"net": "off",
"num": "off",
"channel_nsfw": "off",
"channel_category": "off",
"channel_category_id": "off",
"channel_category_nsfw": "off",
"channel_id": "off",
"guild_member_count": "off",
"bot": "off",
"msg_id": "off",
"user": "off",
"net_id": "off",
"user_id": "off",
"nick_id": "off",
"status": "off",
"num_users": "off",
"num_chans": "off",
"exemption": "off",
// "version_sentiment": "off",
"sentiment": "off",
"num": "off",
"online": "off",
"mtype": "off",
"realname": "off",
"server": "off",
"mtype": "off",
"hidden": "off",
"filename": "off",
"file_md5": "off",
"file_ext": "off",
"file_size": "off",
"lang_code": "off",
"tokens": "off",
"rule_id": "off",
"index": "off",
"meta": "off",
"match_ts": "off",
"batch_id": "off"
//"lang_name": "off",
// "words_noun": "off",
// "words_adj": "off",
// "words_verb": "off",
// "words_adv": "off"
},
};
} else {
storage = JSON.parse(storage);
}
return storage;
};
// Save structure in web storage
var set_column_shifter_storage = function(storage){
var json_storage = JSON.stringify(storage)
localStorage.setItem(COLUMN_SHIFTER_STORAGE_ACCESOR, json_storage);
};
// Remember state for single button
var save_btn_state = function($btn){
// Take css class for container with table
var table_class_container = $btn.data("table-class-container");
// Take html object with table
var $table_class_container = $("#" + table_class_container);
// Take single button statne ("on" / "off")
var state = $btn.data("state");
// td-class is a real column name in table
var td_class = $btn.data("td-class");
var storage = get_column_shifter_storage();
// Table id
var id = $table_class_container.attr("id");
// Checking if the ID is already in storage
if (id in storage) {
data = storage[id]
} else {
data = {}
storage[id] = data;
}
// Save state for table column in storage
data[td_class] = state;
set_column_shifter_storage(storage);
};
// Load states for buttons from storage for single tabel
var load_states = function($table_class_container) {
var storage = get_column_shifter_storage();
// Table id
var id = $table_class_container.attr("id");
var data = {};
// Checking if the ID is already in storage
if (id in storage) {
data = storage[id]
// For each shifter button set state
$table_class_container.find(".btn-shift-column").each(function(){
var $btn = $(this);
var td_class = $btn.data("td-class");
// If name of column is in store then get state
// and set state
if (td_class in data) {
var state = data[td_class]
set_btn_state($btn, state);
}
});
}
};
// Show table content and hide spiner
var show_table_content = function($table_class_container){
$table_class_container.find("#loader").hide();
$table_class_container.find("#table-container").show();
};
// Load buttons states for all button in page
var load_state_for_all_containters = function(){
$(".column-shifter-container").each(function(){
$table_class_container = $(this);
// Load states for all buttons in single container
load_states($table_class_container);
// When states was loaded then table must be show and
// loader (spiner) must be hide
show_table_content($table_class_container);
});
};
// change visibility column for single button
// if button has state "on" then show column
// else then column will be hide
shift_column = function( $btn ){
// button state
var state = $btn.data("state");
// td-class is a real column name in table
var td_class = $btn.data("td-class");
var table_class_container = $btn.data("table-class-container");
var $table_class_container = $("#" + table_class_container);
var $table = $table_class_container.find("table");
var $cels = $table.find("." + td_class);
if ( state === "on" ) {
$cels.show();
} else {
$cels.hide();
}
};
// Shift visibility for all columns
shift_columns = function(){
var cols = $(".btn-shift-column");
var i, len = cols.length;
for (i=0; i < len; i++) {
shift_column($(cols[i]));
}
};
// Set icon imgae visibility for button state
var set_icon_for_state = function( $btn, state ) {
if (state === "on") {
$btn.find("span.uncheck").hide();
$btn.find("span.check").show();
} else {
$btn.find("span.check").hide();
$btn.find("span.uncheck").show();
}
};
// Set state for single button
var set_btn_state = function($btn, state){
$btn.data('state', state);
set_icon_for_state($btn, state);
}
// Change state for single button
var change_btn_state = function($btn){
var state = $btn.data("state");
if (state === "on") {
state = "off"
} else {
state = "on"
}
set_btn_state($btn, state);
};
// Run show/hide when click on button
$(".btn-shift-column").on("click", function(event){
var $btn = $(this);
event.stopPropagation();
change_btn_state($btn);
shift_column($btn);
save_btn_state($btn);
});
// Load saved states for all tables
load_state_for_all_containters();
// show or hide columns based on data from web storage
shift_columns();
// Add API method for retrieving non-visible cols for table
// Pass the 0-based index of the table or leave the parameter
// empty to return the hidden cols for the 1st table found
$.django_tables2_column_shifter_hidden = function(idx) {
if(idx==undefined) {
idx = 0;
}
return $('#table-container').eq(idx).find('.btn-shift-column').filter(function(z) {
return $(this).data('state')=='off'
}).map(function(z) {
return $(this).data('td-class')
}).toArray();
}
const event = new Event('restore-scroll');
document.dispatchEvent(event);
const event2 = new Event('load-widget-results');
document.dispatchEvent(event2);
});

3
core/static/js/gridstack-all.js
View File

File diff suppressed because one or more lines are too long

16
core/static/js/gridstack.min.js vendored
View File

File diff suppressed because one or more lines are too long

2
core/static/js/jquery.min.js vendored
View File

File diff suppressed because one or more lines are too long

2
core/static/js/magnet.min.js vendored
View File

File diff suppressed because one or more lines are too long

21
core/static/modal.js
View File

@ -1,21 +1,12 @@
// var modal = document.querySelector('.modal'); // assuming you have only 1
var modal = document.getElementById("modal");
var modal = document.querySelector('.modal'); // assuming you have only 1
var html = document.querySelector('html');
var disableModal = function() {
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
var modal_refresh = document.getElementsByClassName("modal-refresh");
for(var i = 0; i < modal_refresh.length; i++) {
modal_refresh[i].remove();
}
}
var elements = document.querySelectorAll('.modal-background');
for(var i = 0; i < elements.length; i++) {
elements[i].addEventListener('click', function(e) {
// elements[i].preventDefault();
disableModal();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
});
}
@ -23,7 +14,8 @@ var elements = document.querySelectorAll('.modal-close');
for(var i = 0; i < elements.length; i++) {
elements[i].addEventListener('click', function(e) {
// elements[i].preventDefault();
disableModal();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
});
}
@ -32,7 +24,8 @@ function activateButtons() {
for(var i = 0; i < elements.length; i++) {
elements[i].addEventListener('click', function(e) {
// elements[i].preventDefault();
disableModal();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
});
}
}

7
core/static/tabs.css Normal file
View File

@ -0,0 +1,7 @@
#tab-content div {
display: none;
}
#tab-content div.is-active {
display: block;
}

21
core/static/tabs.js
View File

@ -1,18 +1,15 @@
function initTabs(unique) {
var tabs_selector = '#tabs-'+unique+' li';
var TABS = [...document.querySelectorAll(tabs_selector)];
var CONTENT = [...document.querySelectorAll('#tab-content-'+unique+' div')];
var ACTIVE_CLASS = 'is-active';
// tabbed browsing for the modal
function initTabs() {
TABS.forEach((tab) => {
tab.addEventListener('click', (e) => {
let selected = tab.getAttribute('data-tab');
updateActiveTab(TABS, ACTIVE_CLASS, tab);
updateActiveContent(CONTENT, ACTIVE_CLASS, selected);
updateActiveTab(tab);
updateActiveContent(selected);
})
})
}
function updateActiveTab(TABS, ACTIVE_CLASS, selected) {
function updateActiveTab(selected) {
TABS.forEach((tab) => {
if (tab && tab.classList.contains(ACTIVE_CLASS)) {
tab.classList.remove(ACTIVE_CLASS);
@ -21,7 +18,7 @@ function updateActiveTab(TABS, ACTIVE_CLASS, selected) {
selected.classList.add(ACTIVE_CLASS);
}
function updateActiveContent(CONTENT, ACTIVE_CLASS, selected) {
function updateActiveContent(selected) {
CONTENT.forEach((item) => {
if (item && item.classList.contains(ACTIVE_CLASS)) {
item.classList.remove(ACTIVE_CLASS);
@ -32,5 +29,7 @@ function updateActiveContent(CONTENT, ACTIVE_CLASS, selected) {
}
});
}
// initTabs();
var TABS = [...document.querySelectorAll('#tabs li')];
var CONTENT = [...document.querySelectorAll('#tab-content div')];
var ACTIVE_CLASS = 'is-active';
initTabs();

229
core/templates/base.html
View File

@ -1,10 +1,8 @@
{% load static %}
{% load has_plan %}
{% load cache %}
<!DOCTYPE html>
<html lang="en-GB">
{% cache 600 head request.path_info %}
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
@ -17,30 +15,12 @@
<link rel="stylesheet" href="{% static 'css/bulma-slider.min.css' %}">
<link rel="stylesheet" href="{% static 'css/bulma-calendar.min.css' %}">
<link rel="stylesheet" href="{% static 'css/bulma-tagsinput.min.css' %}">
<link rel="stylesheet" href="{% static 'css/bulma-switch.min.css' %}">
<link rel="stylesheet" href="{% static 'css/gridstack.min.css' %}">
<script src="{% static 'js/bulma-calendar.min.js' %}" integrity="sha384-DThNif0xGXbopX7+PE+UabkuClfI/zELNhaVqoGLutaWB76dyMw0vIQBGmUxSfVQ" crossorigin="anonymous"></script>
<script src="{% static 'js/bulma-slider.min.js' %}" integrity="sha384-wbyps8iLG8QzJE02viYc/27BtT5HSa11+b5V7QPR1/huVuA8f4LRTNGc82qAIeIZ" crossorigin="anonymous"></script>
<script src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
<script defer src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
<script defer src="{% static 'js/remove-me.js' %}" integrity="sha384-6fHcFNoQ8QEI3ZDgw9Z/A6Brk64gF7AnFbLgdrumo8/kBbsKQ/wo7wPegj5WkzuG" crossorigin="anonymous"></script>
<script defer src="{% static 'js/hyperscript.min.js' %}" integrity="sha384-6GYN8BDHOJkkru6zcpGOUa//1mn+5iZ/MyT6mq34WFIpuOeLF52kSi721q0SsYF9" crossorigin="anonymous"></script>
<script src="{% static 'js/bulma-tagsinput.min.js' %}"></script>
<script src="{% static 'js/jquery.min.js' %}"></script>
<script src="{% static 'js/gridstack-all.js' %}"></script>
<script defer src="{% static 'js/magnet.min.js' %}"></script>
<script>
document.addEventListener("restore-scroll", function(event) {
var scrollpos = localStorage.getItem('scrollpos');
if (scrollpos) {
window.scrollTo(0, scrollpos)
};
});
document.addEventListener("htmx:beforeSwap", function(event) {
localStorage.setItem('scrollpos', window.scrollY);
});
</script>
<script src="{% static 'js/bulma-tagsinput.min.js' %}" integrity="sha384-GmnKCsPJIPPZbNVXpkGRmKdxOa0PQLnOM/hQLIHvMRERySuyvFqKGc76iHTGUY+d" crossorigin="anonymous"></script>
<script>
document.addEventListener('DOMContentLoaded', () => {
@ -67,11 +47,11 @@
<style>
.icon { border-bottom: 0px !important;}
.wrap {
/* white-space: pre-wrap;
white-space: -moz-pre-wrap;
white-space: -pre-wrap;
white-space: -o-pre-wrap; */
word-wrap: break-word;
white-space: pre-wrap; /* CSS3 */
white-space: -moz-pre-wrap; /* Firefox */
white-space: -pre-wrap; /* Opera <7 */
white-space: -o-pre-wrap; /* Opera 7 */
word-wrap: break-word; /* IE */
}
.nowrap-parent {
white-space: nowrap;
@ -92,154 +72,12 @@
.htmx-request.htmx-indicator{
opacity:1
}
.dropdown-content {
height: 20em;
overflow: auto;
}
table.relays-table tr:nth-of-type(2n) td {
border-bottom: 3px solid grey;
}
.tooltiptext {
visibility: hidden;
background-color: black;
color: #fff;
text-align: center;
padding: 5px 0;
border-radius: 6px;
position: absolute;
z-index: 1;
}
.rounded-tooltip:hover .tooltiptext {
visibility: visible;
}
#sentiment-container {
position: fixed;
top: 0;
right: 0;
bottom: 0;
left: 0;
height: 100vh;
width: 100vw;
z-index: -2;
}
.table {
background: transparent !important;
}
tr {
transition: all 0.2s ease-in-out;
}
tr:hover {
cursor:pointer;
background-color:rgba(221, 224, 255, 0.3) !important;
}
a.panel-block {
transition: all 0.2s ease-in-out;
}
a.panel-block:hover {
cursor:pointer;
background-color:rgba(221, 224, 255, 0.3) !important;
}
.panel, .box, .modal {
background-color:rgba(250, 250, 250, 0.5) !important;
}
.modal, .modal.box{
background-color:rgba(210, 210, 210, 0.9) !important;
}
.modal-background{
background-color:rgba(255, 255, 255, 0.3) !important;
}
.has-background-grey-lighter{
background-color:rgba(219, 219, 219, 0.5) !important;
}
.navbar {
background-color:rgba(0, 0, 0, 0.03) !important;
}
.grid-stack-item-content {
display: flex !important;
flex-direction: column !important;
overflow-x: hidden !important;
overflow-y: hidden !important;
}
.panel {
display: flex !important;
flex-direction: column !important;
overflow: hidden;
}
.panel-block {
overflow-y:auto;
overflow-x:auto;
min-height: 90%;
display: block;
}
.floating-window {
/* background-color:rgba(210, 210, 210, 0.6) !important; */
display: flex !important;
flex-direction: column !important;
overflow-x: hidden !important;
overflow-y: hidden !important;
max-height: 300px;
z-index: 9000;
position: absolute;
top: 50px;
left: 50px;
}
.floating-window .panel {
background-color:rgba(250, 250, 250, 0.8) !important;
}
.float-right {
float: right;
padding-right: 5px;
padding-left: 5px;
}
.grid-stack-item:hover .ui-resizable-handle {
display: block !important;
}
.ui-resizable-handle {
z-index: 39 !important;
}
.small-field {
overflow: hidden;
text-overflow: ellipsis;
overflow-y: hidden;
}
</style>
<!-- Piwik --> {# Yes it's in the source, fight me #}
<script type="text/javascript">
var _paq = _paq || [];
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
_paq.push(['setTrackerUrl', 'https://api-a6fe73d3464641fe99ba77e5fdafa19c.s.zm.is']);
_paq.push(['setSiteId', 4]);
_paq.push(['setApiToken', 'je4TjsrunIM9uD4jrr_DGXJP4_b_Kq6ABhulOLo_Old']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.type='text/javascript'; g.async=true; g.defer=true; g.src='https://c87zpt9a74m181wto33r.s.zm.is/embed.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Piwik Code -->
</head>
{% endcache %}
<body>
{% cache 600 nav request.user.id %}
<nav class="navbar" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="{% url 'home' %}">
@ -258,24 +96,13 @@
<a class="navbar-item" href="{% url 'home' %}">
Search
</a>
<a class="navbar-item" href="{% url 'rules' type='page' %}">
Rules
<a class="navbar-item" href="{% url 'about' %}">
About
</a>
{% if user.is_authenticated %}
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link">
Account
</a>
<div class="navbar-dropdown">
<a class="navbar-item" href="{% url 'billing' %}">
Billing
</a>
<a class="navbar-item" href="{% url 'notifications_update' type='page' %}">
Notifications
</a>
</div>
</div>
{% endif %}
{% if user.is_superuser %}
<div class="navbar-item has-dropdown is-hoverable">
@ -294,22 +121,12 @@
</div>
{% endif %}
{% if perms.core.use_insights %}
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link">
{% if user.is_authenticated %}
{% if user|has_plan:'drilldown' %}
<a class="navbar-item" href="{% url 'insights' %}">
Insights
</a>
<div class="navbar-dropdown">
{% for index in user.allowed_indices %}
{% if index != "meta" %}
<a class="navbar-item" href="{% url 'insights' index=index %}">
{{ index }}
</a>
{% endif %}
{% endfor %}
</div>
</div>
{% endif %}
<a class="navbar-item add-button">
Install
@ -321,23 +138,23 @@
<div class="buttons">
{% if not user.is_authenticated %}
<a class="button is-info" href="{% url 'signup' %}">
Sign up
<strong>Sign up</strong>
</a>
<a class="button" href="{% url 'login' %}">
<a class="button is-light" href="{% url 'login' %}">
Log in
</a>
{% endif %}
{% if user.is_authenticated %}
<a class="button" href="{% url 'logout' %}">Logout</a>
<a class="button is-dark" href="{% url 'logout' %}">Logout</a>
{% endif %}
</div>
</div>
</div>
</div>
</nav>
{% endcache %}
<script>
let deferredPrompt;
const addBtn = document.querySelector('.add-button');
@ -367,22 +184,10 @@
});
});
</script>
{% block outer_content %}
{% endblock %}
<section class="section">
<div class="container">
{% block content_wrapper %}
{% block content %}
{% endblock %}
{% endblock %}
<div id="modals-here">
</div>
<div id="windows-here">
</div>
<div id="widgets-here" style="display: none;">
{% block widgets %}
{% endblock %}
</div>
</div>
</section>
</body>

12
core/templates/billing.html
View File

@ -31,16 +31,6 @@
Subscription management
</a>
</article>
<div class="box">
<h1 class="subtitle">
This product is currently free. You may cancel any plans above.
</h1>
</div>
<div class="box">
<h1 class="subtitle">
You cannot pay for access to the raw data. It is hashed to preserve privacy.
</h1>
</div>
{# {% include "partials/product-list.html" %} #}
{% include "partials/product-list.html" %}
{% endblock %}

188
core/templates/index.html
View File

@ -1,152 +1,48 @@
{% extends 'base.html' %}
{% extends "base.html" %}
{% load static %}
{% load joinsep %}
{% block outer_content %}
{% if params.modal == 'context' %}
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "{{ params.net|escapejs }}",
"num": "{{ params.num|escapejs }}",
"source": "{{ params.source|escapejs }}",
"channel": "{{ params.channel|escapejs }}",
"time": "{{ params.time|escapejs }}",
"date": "{{ params.date|escapejs }}",
"index": "{{ params.index }}",
"type": "{{ params.type|escapejs }}",
"mtype": "{{ params.mtype|escapejs }}",
"nick": "{{ params.nick|escapejs }}"}'
hx-target="#modals-here"
hx-trigger="load">
{% block content %}
<div class="block">
{% for block in blocks %}
{% if block.title is not None %}
<h1 class="title">{{ block.title }}</h1>
{% endif %}
<div class="box">
<div class="columns">
{% if block.column1 is not None %}
<div class="column">
{{ block.column1 }}
</div>
{% endif %}
<script src="{% static 'js/chart.js' %}"></script>
<script src="{% static 'tabs.js' %}"></script>
<script>
function setupTags() {
var inputTags = document.getElementById('tags');
new BulmaTagsInput(inputTags);
inputTags.BulmaTagsInput().on('before.add', function(item) {
if (item.includes(": ")) {
var spl = item.split(": ");
} else {
var spl = item.split(":");
}
var field = spl[0];
try {
var value = JSON.parse(spl[1]);
} catch {
var value = spl[1];
}
return `${field}: ${value}`;
});
inputTags.BulmaTagsInput().on('after.remove', function(item) {
var spl = item.split(": ");
var field = spl[0];
var value = spl[1].trim();
});
}
function populateSearch(field, value) {
var inputTags = document.getElementById('tags');
inputTags.BulmaTagsInput().add(field+": "+value);
//htmx.trigger("#search", "click");
}
</script>
<div class="grid-stack" id="grid-stack-main">
<div class="grid-stack-item" gs-w="7" gs-h="10" gs-y="0" gs-x="1">
<div class="grid-stack-item-content">
<nav class="panel">
<p class="panel-heading" style="padding: .2em; line-height: .5em;">
<i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
Search
</p>
<article class="panel-block is-active">
{% include 'window-content/search.html' %}
</article>
</nav>
{% if block.column2 is not None %}
<div class="column">
{{ block.column2 }}
</div>
</div>
</div>
<script>
var grid = GridStack.init({
cellHeight: 20,
cellWidth: 50,
cellHeightUnit: 'px',
auto: true,
float: true,
draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
removable: false,
animate: true,
});
// GridStack.init();
setupTags();
// a widget is ready to be loaded
document.addEventListener('load-widget', function(event) {
let container = htmx.find('#widget');
// get the scripts, they won't be run on the new element so we need to eval them
var scripts = htmx.findAll(container, "script");
let widgetelement = container.firstElementChild.cloneNode(true);
var new_id = widgetelement.id;
// check if there's an existing element like the one we want to swap
let grid_element = htmx.find('#grid-stack-main');
let existing_widget = htmx.find(grid_element, "#"+new_id);
// get the size and position attributes
if (existing_widget) {
let attrs = existing_widget.getAttributeNames();
for (let i = 0, len = attrs.length; i < len; i++) {
if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
}
}
}
// clear the queue element
container.outerHTML = "";
// temporary workaround, other widgets can be duplicated, but not results
if (widgetelement.id == 'widget-results') {
grid.removeWidget("widget-results");
}
grid.addWidget(widgetelement);
// re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
htmx.process(widgetelement);
// update size when the widget is loaded
document.addEventListener('load-widget-results', function(evt) {
var added_widget = htmx.find(grid_element, '#widget-results');
var itemContent = htmx.find(added_widget, ".control");
var scrollheight = itemContent.scrollHeight+80;
var verticalmargin = 0;
var cellheight = grid.opts.cellHeight;
var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
var opts = {
h: height,
}
grid.update(
added_widget,
opts
);
});
// run the JS scripts inside the added element again
// for instance, this will fix the dropdown
for (var i = 0; i < scripts.length; i++) {
eval(scripts[i].innerHTML);
}
});
</script>
{% endblock %}
{% block widgets %}
{% if table or message is not None %}
{% include 'partials/results_load.html' %}
{% endif %}
{% if block.column3 is not None %}
<div class="column">
{{ block.column3 }}
</div>
{% endif %}
</div>
<div class="columns">
{% if block.image1 is not None %}
<div class="column">
<img src="{% static block.image1 %}">
</div>
{% endif %}
{% if block.image2 is not None %}
<div class="column">
<img src="{% static block.image2 %}">
</div>
{% endif %}
{% if block.image3 is not None %}
<div class="column">
<img src="{% static block.image3 %}">
</div>
{% endif %}
</div>
</div>
{% endfor %}
</div>
{% endblock %}

16
core/templates/manage/threshold/irc/network/actions.html
View File

@ -31,20 +31,7 @@
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_actions_registration_net' net %}"
hx-trigger="click"
hx-target="#modals-here"
class="button is-info">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-list"></i>
</span>
<span>Registration</span>
</span>
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'threshold_irc_network_list' net %}"
hx-post="{% url 'threshold_irc_network_actions_list' net %}"
hx-trigger="click"
hx-target="#actions"
hx-swap="outerHTML"
@ -78,4 +65,5 @@
</div>
</div>
</form>
</div>

15
core/templates/manage/threshold/irc/network/channels.html
View File

@ -6,26 +6,21 @@
<table class="table is-fullwidth is-hoverable">
<thead>
<th>channel</th>
<th>num</th>
<th>actions</th>
</thead>
<tbody>
{% for channel in channels %}
{% for channel, info in channels.items %}
<tr>
<td>
{{ channel.name }}
{{ channel }}
<span class="tag">
{{ channel.users }}
{{ info }}
</span>
</td>
<td>
{{ channel.num }}
</td>
<td>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}", "Content-Type": "application/json"}'
hx-delete="{% url 'threshold_irc_network_channel_json' net %}"
hx-vals='{"channel": "{{ channel.name }}"}'
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-delete="{% url 'threshold_irc_network_channel' net channel %}"
hx-target="#channels"
hx-swap="outerHTML"
class="button is-danger is-small">

99
core/templates/manage/threshold/irc/network/modals/registration.html
View File

@ -1,99 +0,0 @@
{% load index %}
{% load static %}
<script src="{% static 'modal.js' %}"></script>
<div id="modal" class="modal is-active is-clipped">
<div class="modal-background"></div>
<div class="modal-content">
<div class="box">
{% include 'manage/threshold/partials/notify.html' %}
<h4 class="subtitle is-4">Registration</h4>
<div class="buttons">
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'threshold_irc_actions_registration_auth' %}"
hx-vals='{"net": "{{ net }}", "func": "recheckauth"}'
hx-trigger="click"
hx-target="#modals-here"
class="button is-info">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-wrench"></i>
</span>
<span>Check auth</span>
</span>
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'threshold_irc_actions_registration_auth' %}"
hx-vals='{"net": "{{ net }}", "func": "resetauth"}'
hx-trigger="click"
hx-target="#modals-here"
class="button is-info">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-wrench"></i>
</span>
<span>Reset auth</span>
</span>
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'threshold_irc_actions_registration_auth' %}"
hx-vals='{"net": "{{ net }}", "func": "register"}'
hx-trigger="click"
hx-target="#modals-here"
class="button is-info">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-wrench"></i>
</span>
<span>Register</span>
</span>
</button>
</div>
{% if unreg is not None %}
<form
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-put="{% url 'threshold_irc_actions_registration_net' net %}"
hx-target="#actions"
hx-swap="outerHTML">
{% for network, items in unreg.items %}
<h4 class="title is-4">{{ network }}</h4>
{% if items is not False %}
{% for nick, num in items %}
<div class="field">
<label class="label">{{ nick }}/{{ num }}</label>
<div class="control">
<input class="input" type="text" name="{{ network }}|{{ num }}" placeholder="Enter token">
</div>
</div>
{% endfor %}
{% else %}
<p>Error getting information for {{ network }}.</p>
{% endif %}
{% endfor %}
<button
type="button"
class="button is-light modal-close-button">
Cancel
</button>
<button type="submit" class="button is-info modal-close-button">Submit</button>
{# <script>activateButtons();</script> #}
</form>
{% else %}
<p>No unregistered relays.</p>
{% endif %}
</div>
<button class="modal-close is-large" aria-label="close"></button>
</div>
</div>

59
core/templates/manage/threshold/irc/network/network.html
View File

@ -1,33 +1,11 @@
{% extends "base.html" %}
{% block content %}
<script>
document.addEventListener("restore-relay-scroll", function(event) {
var modalContent = document.getElementsByClassName("relay_table_container")[0];
var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
var scrollpos = localStorage.getItem('scrollpos_relays_table');
if (scrollpos == 'BOTTOM') {
modalContent.scrollTop = maxScroll;
} else if (scrollpos) {
modalContent.scrollTop = scrollpos;
};
});
document.addEventListener("htmx:beforeSwap", function(event) {
var modalContent = document.getElementsByClassName("relay_table_container")[0];
var scrollpos = modalContent.scrollTop;
if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
localStorage.setItem('scrollpos_relays_table', 'BOTTOM');
} else {
localStorage.setItem('scrollpos_relays_table', scrollpos);
}
});
</script>
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_network_info' net %}"
hx-trigger="load, every 60s"
hx-trigger="load, every 5s"
hx-target="#info"
hx-swap="outerHTML">
</div>
@ -36,17 +14,16 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_network_relays' net %}"
hx-trigger="load, every 60s"
hx-trigger="load, every 5s"
hx-target="#relays"
{# hx-swap="innerHTML" #}
>
hx-swap="outerHTML">
</div>
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_network_channels' net %}"
hx-trigger="load, every 60s"
hx-trigger="load, every 5s"
hx-target="#channels"
hx-swap="outerHTML">
</div>
@ -60,16 +37,6 @@
hx-swap="outerHTML">
</div>
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_network_list' net %}"
hx-trigger="load"
hx-target="#stats"
hx-swap="outerHTML">
</div>
<div class="columns">
<div class="column">
<div class="box">
@ -77,19 +44,9 @@
</div>
</div>
</div>
<!-- <div class="column">
<div class="box">
<div id="relays">
</div>
</div>
</div> -->
<div class="column">
<div class="box">
<div>
<div class="content" style="max-height: 30em; overflow: auto;">
<div class="table-container relay_table_container" id="relays">
</div>
</div>
<div id="relays">
</div>
</div>
</div>
@ -104,8 +61,8 @@
</div>
<div class="column">
<div class="box">
<div id="stats">
Stats here
<div id="alerts">
Alerts here
</div>
</div>
</div>
@ -145,6 +102,4 @@
</div>
</div>
</div>
<div id="modals-here">
</div>
{% endblock %}

189
core/templates/manage/threshold/irc/network/relays.html
View File

@ -1,43 +1,23 @@
{% load index %}
<div class="table-container relay_table_container" id="relays">
<table class="table is-fullwidth is-hoverable relays-table">
<div id="relays">
{% include 'manage/threshold/partials/notify.html' %}
{% if relays is not None %}
<div class="content" style="max-height: 30em; overflow: auto;">
<div class="table-container">
<table class="table is-fullwidth is-hoverable">
<thead>
<th>id</th>
<th>reg</th>
<th>on</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Registered">
<i class="fa-solid fa-seal"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Authenticated">
<i class="fa-solid fa-passport"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Connected">
<i class="fa-solid fa-cloud-question"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Enabled">
<i class="fa-solid fa-toggle-on"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Channels">
<span class="icon">
<i class="fa-solid fa-hashtag"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Chanlimit">
<i class="fa-solid fa-list-ol"></i>
</span>
</th>
<th>nick</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Actions">
<i class="fa-solid fa-wrench"></i>
<span class="icon">
<i class="fa-solid fa-wrench" aria-hidden="true"></i>
</span>
</th>
</thead>
@ -47,44 +27,22 @@
<td>{{ relay.id }}</td>
<td>
{% if relay.registered %}
<span class="icon has-text-success">
<span class="icon">
<i class="fa-solid fa-check" aria-hidden="true"></i>
</span>
{% else %}
<span class="icon has-text-danger">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
{% endif %}
</td>
<td>
{% if relay.authed %}
<span class="icon has-text-success">
<i class="fa-solid fa-check" aria-hidden="true"></i>
</span>
{% else %}
<span class="icon has-text-danger">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
{% endif %}
</td>
<td>
{% if relay.conn %}
<span class="icon has-text-success">
<i class="fa-solid fa-check" aria-hidden="true"></i>
</span>
{% else %}
<span class="icon has-text-danger">
<span class="icon">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
{% endif %}
</td>
<td>
{% if relay.enabled %}
<span class="icon has-text-success">
<span class="icon">
<i class="fa-solid fa-check" aria-hidden="true"></i>
</span>
{% else %}
<span class="icon has-text-danger">
<span class="icon">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
{% endif %}
@ -92,130 +50,51 @@
<td>
{{ relay.chans }}
</td>
<td>{{ relay.limit }}</td>
<td>
{{ relay.nick }}
</td>
<td>
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "{{ net }}",
"num": "{{ relay.id }}",
"source": "irc",
"channel": "*status",
"time": "None",
"date": "None",
"index": "internal",
"type": "znc",
"mtype": "None",
"nick": "*status",
"dedup": "on"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small has-background-info has-text-white">
<span class="icon has-tooltip-left" data-tooltip="ZNC context">
<i class="fa-brands fa-unity" aria-hidden="true"></i>
</span>
</a>
</td>
</tr>
<tr>
<td>
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-delete="{% url 'threshold_irc_network_relay_del' relay|index:'net' relay|index:'id' %}"
hx-target="#relays"
hx-swap="outerHTML"
class="button is-small has-background-danger has-text-white">
<span class="icon" data-tooltip="Delete">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
</a>
</td>
<td>
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'threshold_irc_network_relay_provision' relay|index:'net' relay|index:'id' %}"
hx-target="#relays"
hx-swap="outerHTML"
class="button is-small has-background-info has-text-white">
<span class="icon" data-tooltip="Provision">
<i class="fa-solid fa-wrench" aria-hidden="true"></i>
</span>
</a>
</td>
<td>
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'threshold_irc_network_relay_auth' relay|index:'net' relay|index:'id' %}"
hx-target="#relays"
hx-swap="outerHTML"
class="button is-small has-background-info has-text-white">
<span class="icon" data-tooltip="Enable authentication">
<i class="fa-solid fa-passport" aria-hidden="true"></i>
</span>
</a>
</td>
<td></td>
<td>
<div class="buttons">
{% if relay.enabled %}
<a
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_network_relay_status' relay|index:'net' relay|index:'id' 0 %}"
hx-target="#relays"
hx-swap="outerHTML"
class="button is-small has-background-warning">
class="button is-danger is-small">
<span class="icon" data-tooltip="Disable">
<i class="fa-solid fa-wifi-slash" aria-hidden="true"></i>
</span>
</a>
</button>
{% else %}
<a
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_network_relay_status' relay|index:'net' relay|index:'id' 1 %}"
hx-target="#relays"
hx-swap="outerHTML"
class="button is-small has-background-success has-text-white">
class="button is-success is-small">
<span class="icon" data-tooltip="Enable">
<i class="fa-solid fa-wifi" aria-hidden="true"></i>
</span>
</a>
</button>
{% endif %}
</td>
<td></td>
<td></td>
<td></td>
<td>
<a
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "{{ net }}",
"num": "{{ relay.id }}",
"source": "irc",
"channel": "{{ sinst.entity }}",
"time": "None",
"date": "None",
"index": "internal",
"type": "auth",
"mtype": "None",
"nick": "{{ sinst.entity }}",
"dedup": "on"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small has-background-info has-text-white">
<span class="icon has-tooltip-left" data-tooltip="Auth ({{ sinst.entity }})">
<i class="fa-solid fa-signature" aria-hidden="true"></i>
hx-delete="{% url 'threshold_irc_network_relay_del' relay|index:'net' relay|index:'id' %}"
hx-target="#relays"
hx-swap="outerHTML"
class="button is-danger is-small">
<span class="icon" data-tooltip="Delete">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
</a>
</button>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<script>
var modal_event = new Event('restore-relay-scroll');
document.dispatchEvent(modal_event);
</script>
{% include 'manage/threshold/partials/notify.html' %}
</div>
</div>
{% endif %}
</div>

27
core/templates/manage/threshold/irc/network/stats.html
View File

@ -1,27 +0,0 @@
<div id="stats">
{% include 'manage/threshold/partials/notify.html' %}
{% if list is not None %}
<div class="content" style="max-height: 30em; overflow: auto;">
<div class="table-container">
<table class="table is-fullwidth is-hoverable">
<thead>
<th>attribute</th>
<th>value</th>
</thead>
<tbody>
{% for key, item in list.items %}
<tr>
<th>{{ key }}</th>
<td>
{% if item is not None %}
{{ item }}
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endif %}
</div>

3
core/templates/manage/threshold/irc/overview/actions.html
View File

@ -30,9 +30,6 @@
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_actions_registration' %}"
hx-trigger="click"
hx-target="#modals-here"
class="button is-info">
<span class="icon-text">
<span class="icon">

94
core/templates/manage/threshold/irc/overview/alerts.html Normal file
View File

@ -0,0 +1,94 @@
<div id="alerts">
{% include 'manage/threshold/partials/notify.html' %}
{% if alerts is not None %}
<div class="icons">
<button
class="button is-small">
<span class="icon" data-tooltip="Conn">
<i class="fa-solid fa-cloud-exclamation"></i>
</span>
</button>
<button
class="button is-small">
<span class="icon" data-tooltip="Highlight">
<i class="fa-solid fa-square-quote"></i>
</span>
</button>
<button
class="button is-small">
<span class="icon" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
</button>
<button
class="button is-small">
<span class="icon" data-tooltip="Query">
<i class="fa-solid fa-inbox"></i>
</span>
</button>
<button class="button is-small">
<span class="icon" data-tooltip="Self">
<i class="fa-solid fa-message-bot"></i>
</span>
</button>
</div>
<div class="content" style="max-height: 30em; overflow: auto;">
<div class="table-container">
<table class="table is-fullwidth is-hoverable">
<thead>
<th>ts</th>
<th>name</th>
<th>type</th>
<th>msg</th>
</thead>
<tbody>
{% for alert in alerts %}
<tr>
<td>
<p>{{ alert.date }}</p>
<p>{{ alert.time }}</p>
</td>
<td>
{{ alert.net }}/{{ alert.num }}
</td>
<td>
{% if alert.type == 'conn' %}
<span class="icon" data-tooltip="Conn">
<i class="fa-solid fa-cloud-exclamation"></i>
</span>
{% elif alert.type == 'highlight' %}
<span class="icon" data-tooltip="Highlight">
<i class="fa-solid fa-square-quote"></i>
</span>
{% elif alert.type == 'znc' %}
<span class="icon" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
{% elif alert.type == 'query' %}
<span class="icon" data-tooltip="Query">
<i class="fa-solid fa-inbox"></i>
</span>
{% elif alert.type == 'self' %}
<span class="icon" data-tooltip="Self">
<i class="fa-solid fa-message-bot"></i>
</span>
{% else %}
{{ alert.type }}
{% endif %}
</td>
<td class="wrap" style="max-width: 10em">
{{ alert.msg }}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endif %}
</div>

4
core/templates/manage/threshold/irc/overview/modals/add-network.html
View File

@ -2,7 +2,7 @@
{% load static %}
<script src="{% static 'modal.js' %}"></script>
<div id="modal" class="modal is-active is-clipped">
<div class="modal is-active is-clipped">
<div class="modal-background"></div>
<div class="modal-content">
<div class="box">
@ -68,7 +68,7 @@
Cancel
</button>
<button type="submit" class="button is-info modal-close-button">Submit</button>
{# <script>activateButtons();</script> #}
<script>activateButtons();</script>
</form>
</div>

2
core/templates/manage/threshold/irc/overview/modals/edit-aliases.html
View File

@ -3,7 +3,7 @@
{% load nsep %}
<script src="{% static 'modal.js' %}"></script>
<div id="modal" class="modal is-active is-clipped">
<div class="modal is-active is-clipped">
<div class="modal-background"></div>
<div class="modal-content">
<div class="box">

42
core/templates/manage/threshold/irc/overview/networks.html
View File

@ -4,28 +4,15 @@
<table class="table is-fullwidth is-hoverable">
<thead>
<th>net</th>
<th>relays</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Relays">
<i class="fa-brands fa-unity"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Active">
<i class="fa-solid fa-signal-bars-good"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Channels">
<span class="icon">
<i class="fa-solid fa-hashtag"></i>
</span>
</th>
<th>records</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Records">
<i class="fa-solid fa-album"></i>
</span>
</th>
<th>
<span class="icon has-tooltip-bottom" data-tooltip="Actions">
<span class="icon">
<i class="fa-solid fa-wrench" aria-hidden="true"></i>
</span>
</th>
@ -35,24 +22,21 @@
<th><a href="{% url 'threshold_irc_network' key %}">{{ key }}</a></th>
<td>
<span class="icon">
<i class="fa-brands fa-unity"></i>
</span>
{{ net.relays }}
</td>
<td>
{% if net.active %}
<span class="icon has-text-success">
<i class="fa-solid fa-check" aria-hidden="true"></i>
<span class="icon">
<i class="fa-solid fa-hashtag"></i>
</span>
{% else %}
<span class="icon has-text-danger">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>
{% endif %}
</td>
<td>
{{ net.channels }}
</td>
<td>
<span class="icon">
<i class="fa-solid fa-album"></i>
</span>
{{ net.records }}
</td>
<td>
@ -61,7 +45,7 @@
hx-delete="{% url 'threshold_irc_network_del' key %}"
hx-target="#networks"
hx-swap="outerHTML"
class="button is-small is-danger">
class="button is-danger is-small">
<span class="icon" data-tooltip="Delete">
<i class="fa-solid fa-xmark" aria-hidden="true"></i>
</span>

10
core/templates/manage/threshold/irc/overview/overview.html
View File

@ -37,6 +37,15 @@
hx-swap="outerHTML">
</div>
<div
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'threshold_irc_overview_alerts' %}"
hx-trigger="load"
hx-target="#alerts"
hx-swap="outerHTML">
</div>
<div class="columns">
<div class="column">
<div class="box">
@ -62,7 +71,6 @@
<div class="column">
<div class="box">
<div id="alerts">
Alerts here
</div>
</div>
</div>

121
core/templates/modals/context.html
View File

@ -1,121 +0,0 @@
{% extends 'mixins/wm/modal.html' %}
{% load index %}
{% load static %}
{% block scripts %}
<script>
document.addEventListener("restore-modal-scroll", function(event) {
var modalContent = document.getElementsByClassName("modal-content")[0];
var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
var scrollpos = localStorage.getItem('scrollpos_modal_content');
if (scrollpos == 'BOTTOM') {
modalContent.scrollTop = maxScroll;
} else if (scrollpos) {
modalContent.scrollTop = scrollpos;
};
});
document.addEventListener("htmx:beforeSwap", function(event) {
var modalContent = document.getElementsByClassName("modal-content")[0];
var scrollpos = modalContent.scrollTop;
if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
localStorage.setItem('scrollpos_modal_content', 'BOTTOM');
} else {
localStorage.setItem('scrollpos_modal_content', scrollpos);
}
});
</script>
{% endblock %}
{% block styles %}
<style>
#tab-content-{{ unique }} div {
display: none;
}
#tab-content-{{ unique }} div.is-active {
display: block;
}
</style>
{% endblock %}
{% block modal_content %}
{% include 'mixins/partials/notify.html' %}
<div class="tabs is-toggle is-fullwidth is-info" id="tabs-{{ unique }}">
<ul>
<li class="is-active" data-tab="1">
<a>
<span class="icon is-small"><i class="fa-solid fa-message-arrow-down"></i></span>
<span>Scrollback</span>
</a>
</li>
<li data-tab="2">
<a>
<span class="icon is-small"><i class="fa-solid fa-messages"></i></span>
<span>Context</span>
</a>
</li>
<li data-tab="3">
<a>
<span class="icon is-small"><i class="fa-solid fa-message"></i></span>
<span>Message</span>
</a>
</li>
<li data-tab="4">
<a>
<span class="icon is-small"><i class="fa-solid fa-asterisk"></i></span>
<span>Info</span>
</a>
</li>
</ul>
</div>
<div id="tab-content-{{ unique }}">
<div class="is-active" data-content="1">
<h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{% if num is not None %}{{ num }}{% endif %}</h4>
{% include 'partials/context_table.html' %}
{% if user.is_superuser and source == 'irc' %}
<form method="PUT">
<article class="field has-addons">
<article class="control is-expanded has-icons-left">
<input id="context-input" name="msg" class="input" type="text" placeholder="Type your message here">
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</article>
<article class="control">
<article class="field">
<button
id="search"
class="button is-info is-fullwidth"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-put="{% url 'threshold_irc_msg' net num %}"
hx-vals='{"channel": "{{ channel }}", "nick": "{{ nick }}"}'
hx-trigger="click"
hx-target="#context-input"
hx-swap="outerHTML">
Send
</button>
</article>
</article>
</article>
</form>
{% endif %}
</div>
<div data-content="2">
<h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
Context
</div>
<div data-content="3">
<h4 class="subtitle is-4">Message details</h4>
Message deetails
</div>
<div data-content="4">
<h4 class="subtitle is-4">Information about {{ channel }} on {{ net }}{{ num }}</h4>
info
</div>
</div>
<script>initTabs("{{ unique }}");</script>
{% endblock %}

111
core/templates/modals/drilldown.html
View File

@ -1,5 +1,108 @@
{% extends 'mixins/wm/modal.html' %}
{% load index %}
{% load static %}
{% block modal_content %}
{% include 'window-content/drilldown.html' %}
{% endblock %}
<script src="{% static 'modal.js' %}"></script>
<link rel ="stylesheet" href="{% static 'tabs.css' %}">
<script src="{% static 'tabs.js' %}"></script>
<div class="modal is-active is-clipped">
<div class="modal-background"></div>
<div class="modal-content">
<div class="box">
<div class="tabs is-toggle is-fullwidth is-info" id="tabs">
<ul>
<li class="is-active" data-tab="1">
<a>
<span class="icon is-small"><i class="fa-solid fa-user"></i></span>
<span>Channels</span>
</a>
</li>
<li data-tab="2">
<a>
<span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
<span>Users</span>
</a>
</li>
<li data-tab="3">
<a>
<span class="icon is-small"><i class="fa-solid fa-people"></i></span>
<span>Intersection</span>
</a>
</li>
<li data-tab="4">
<a>
<span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
<span>Intersection</span>
</a>
</li>
</ul>
</div>
<div id="tab-content">
<div class="is-active" data-content="1">
<h4 class="subtitle is-4">Channels for {{ nick }} on {{ net }}</h4>
{% for channel in chans %}
<a class="panel-block is-active">
<span class="panel-icon">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{{ channel }}
{% if nick in num_chans %}
<span class="tag">
{{ num_users|index:channel }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
<div data-content="2">
<h4 class="subtitle is-4">Users on {{ channel }} for {{ net }}</h4>
{% for user in users %}
<a class="panel-block is-active">
<span class="panel-icon">
<i class="fa-solid fa-user" aria-hidden="true"></i>
</span>
{{ user }}
{% if channel in num_users %}
<span class="tag">
{{ num_chans|index:user }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
<div data-content="3">
<h4 class="subtitle is-4">Users sharing channels with {{ nick }} on {{ net }}</h4>
{% for user in inter_users %}
<a class="panel-block is-active">
<span class="panel-icon">
<i class="fa-solid fa-user" aria-hidden="true"></i>
</span>
{{ user }}
{% if channel in num_users %}
<span class="tag">
{{ num_chans|index:user }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
<div data-content="4">
<h4 class="subtitle is-4">Channels sharing users with {{ channel }} on {{ net }}</h4>
{% for channel in inter_chans %}
<a class="panel-block is-active">
<span class="panel-icon">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{{ channel }}
{% if nick in num_chans %}
<span class="tag">
{{ num_users|index:channel }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
</div>
</div>
<button class="modal-close is-large" aria-label="close"></button>
</div>
</div>

1
core/templates/partials/context-input.html
View File

@ -1 +0,0 @@
<input id="context-input" name="msg" class="input is-{{ class }}" type="text" placeholder="Type your message here">

177
core/templates/partials/context_table.html
View File

@ -1,177 +0,0 @@
<article class="table-container" id="modal-context-table">
<table class="table is-fullwidth">
<thead>
<th></th>
<th></th>
<th></th>
</thead>
<tbody>
{% for item in object_list %}
{% if item.type == 'control' %}
<tr>
<td></td>
<td>
<span class="icon has-text-grey" data-tooltip="Hidden">
<i class="fa-solid fa-file-slash"></i>
</span>
</td>
<td>
<p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
</td>
</tr>
{% else %}
<tr>
<td>{{ item.time }}</td>
<td>
{% if item.type != 'znc' and item.type != 'self' and query is not True %}
<article class="nowrap-parent">
<article class="nowrap-child">
{% if item.type == 'msg' %}
<span class="icon" data-tooltip="Message">
<i class="fa-solid fa-message"></i>
</span>
{% elif item.type == 'join' %}
<span class="icon" data-tooltip="Join">
<i class="fa-solid fa-person-to-portal"></i>
</span>
{% elif item.type == 'part' %}
<span class="icon" data-tooltip="Part">
<i class="fa-solid fa-person-from-portal"></i>
</span>
{% elif item.type == 'quit' %}
<span class="icon" data-tooltip="Quit">
<i class="fa-solid fa-circle-xmark"></i>
</span>
{% elif item.type == 'kick' %}
<span class="icon" data-tooltip="Kick">
<i class="fa-solid fa-user-slash"></i>
</span>
{% elif item.type == 'nick' %}
<span class="icon" data-tooltip="Nick">
<i class="fa-solid fa-signature"></i>
</span>
{% elif item.type == 'mode' %}
<span class="icon" data-tooltip="Mode">
<i class="fa-solid fa-gear"></i>
</span>
{% elif item.type == 'action' %}
<span class="icon" data-tooltip="Action">
<i class="fa-solid fa-exclamation"></i>
</span>
{% elif item.type == 'notice' %}
<span class="icon" data-tooltip="Notice">
<i class="fa-solid fa-message-code"></i>
</span>
{% elif item.type == 'conn' %}
<span class="icon" data-tooltip="Connection">
<i class="fa-solid fa-cloud-exclamation"></i>
</span>
{% elif item.type == 'znc' %}
<span class="icon" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
{% elif item.type == 'query' %}
<span class="icon" data-tooltip="Query">
<i class="fa-solid fa-message"></i>
</span>
{% elif item.type == 'highlight' %}
<span class="icon" data-tooltip="Highlight">
<i class="fa-solid fa-exclamation"></i>
</span>
{% elif item.type == 'who' %}
<span class="icon" data-tooltip="Who">
<i class="fa-solid fa-passport"></i>
</span>
{% elif item.type == 'topic' %}
<span class="icon" data-tooltip="Topic">
<i class="fa-solid fa-sign"></i>
</span>
{% else %}
{{ item.type }}
{% endif %}
{% if item.online is True %}
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
<i class="fa-solid fa-circle"></i>
</span>
{% elif item.online is False %}
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
<i class="fa-solid fa-circle"></i>
</span>
{% else %}
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
<i class="fa-solid fa-circle"></i>
</span>
{% endif %}
{% if item.src == 'irc' %}
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"
class="has-text-black">
<span class="icon" data-tooltip="Open drilldown modal">
<i class="fa-solid fa-album"></i>
</span>
</a>
{% endif %}
</article>
<a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
{{ item.nick }}
</a>
{% if item.num_chans != '—' %}
<article class="nowrap-child">
<span class="tag">
{{ item.num_chans }}
</span>
</article>
{% endif %}
</article>
{% endif %}
{% if item.type == 'self' %}
<span class="icon has-text-primary" data-tooltip="You">
<i class="fa-solid fa-message-check"></i>
</span>
{% elif item.type == 'znc' %}
<span class="icon has-text-info" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
{% elif query %}
<span class="icon has-text-info" data-tooltip="Auth">
<i class="fa-solid fa-passport"></i>
</span>
{% endif %}
</td>
<td class="wrap">{{ item.msg }}</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
{% if object_list %}
<div
class="modal-refresh"
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context_table' %}"
hx-vals='{"net": "{{ net }}",
"num": "{{ num }}",
"source": "{{ source }}",
"channel": "{{ channel }}",
"time": "{{ time }}",
"date": "{{ date }}",
"index": "{{ index }}",
"type": "{{ type }}",
"mtype": "{{ mtype }}",
"nick": "{{ nick }}",
"dedup": "{{ params.dedup }}"}'
hx-target="#modal-context-table"
hx-trigger="every 5s">
</div>
{% endif %}
</article>
<script>
var modal_event = new Event('restore-modal-scroll');
document.dispatchEvent(modal_event);
</script>

5
core/templates/partials/notify-alt.html
View File

@ -1,5 +0,0 @@
{% if message is not None %}
<main class="notification is-{{ class }}" hx-ext="remove-me" remove-me="3s">
{{ message }}
</main>
{% endif %}

10
core/templates/partials/product-list.html
View File

@ -1,9 +1,8 @@
{% load static %}
{% load cache %}
{% load cachalot cache %}
{% get_last_invalidation 'core.Plan' as last %}
{% cache 600 objects_plans request.user.id plans last %}
{% for plan in plans %}
<div class="box">
<article class="media">
<div class="media-left">
@ -45,4 +44,5 @@
</article>
</div>
{% endfor %}
{% endcache %}

34
core/templates/partials/results_load.html
View File

@ -1,34 +0,0 @@
{% extends 'mixins/wm/widget.html' %}
{% load static %}
{% block heading %}
Results
{% endblock %}
{% block panel_content %}
{% include 'mixins/partials/notify.html' %}
{% if cache is not None %}
<span class="icon has-tooltip-bottom" data-tooltip="Cached">
<i class="fa-solid fa-database"></i>
</span>
{% endif %}
fetched {{ table.data|length }}
{% if params.rule is None %} hits {% else %} rule hits for {{ params.rule }}{% endif %}
in {{ took }}ms
{% if exemption is not None %}
<span class="icon has-tooltip-bottom" data-tooltip="God mode">
<i class="fa-solid fa-book-bible"></i>
</span>
{% else %}
{% if redacted is not None %}
<span class="icon has-tooltip-bottom" data-tooltip="{{ redacted }} redacted">
<i class="fa-solid fa-mask"></i>
</span>
{% endif %}
{% endif %}
{% include 'partials/results_table.html' %}
{% include 'partials/sentiment_chart.html' %}
{% endblock %}

536
core/templates/partials/results_table.html
View File

@ -1,536 +0,0 @@
{% load django_tables2 %}
{% load django_tables2_bulma_template %}
{% load static %}
{% load joinsep %}
{% load urlsafe %}
{% load pretty %}
{% load splitstr %}
{% load cache %}
{% cache 3600 results_table_full request.user.id table %}
{% block table-wrapper %}
<script src="{% static 'js/column-shifter.js' %}"></script>
<div id="drilldown-table" class="column-shifter-container" style="position:relative; z-index:1;">
{% block table %}
<div class="nowrap-parent">
<div class="nowrap-child">
<div class="dropdown" id="dropdown">
<div class="dropdown-trigger">
<button id="dropdown-trigger" class="button dropdown-toggle" aria-haspopup="true" aria-controls="dropdown-menu">
<span>Show/hide fields</span>
<span class="icon is-small">
<i class="fas fa-angle-down" aria-hidden="true"></i>
</span>
</button>
</div>
<div class="dropdown-menu" id="dropdown-menu" role="menu">
<div class="dropdown-content" style="position:absolute; z-index:2;">
{% for column in table.columns %}
{% if column.name in show %}
<a class="btn-shift-column dropdown-item"
data-td-class="{{ column.name }}"
data-state="on"
{% if not forloop.last %} style="border-bottom:1px solid #ccc;" {%endif %}
data-table-class-container="drilldown-table">
<span class="check icon" data-tooltip="Visible" style="display:none;">
<i class="fa-solid fa-check"></i>
</span>
<span class="uncheck icon" data-tooltip="Hidden" style="display:none;">
<i class="fa-solid fa-xmark"></i>
</span>
{{ column.header }}
</a>
{% endif %}
{% endfor %}
</div>
</div>
</div>
</div>
<div class="nowrap-child">
<span id="loader" class="button is-light has-text-link is-loading">Static</span>
</div>
</div>
<script>
var dropdown_button = document.getElementById("dropdown-trigger");
var dropdown = document.getElementById("dropdown");
dropdown_button.addEventListener('click', function(e) {
// elements[i].preventDefault();
dropdown.classList.toggle('is-active');
});
</script>
<div id="table-container" style="display:none;">
<table {% render_attrs table.attrs class="table drilldown-results-table is-fullwidth" %}>
{% block table.thead %}
{% if table.show_header %}
<thead {% render_attrs table.attrs.thead class="" %}>
{% block table.thead.row %}
<tr>
{% for column in table.columns %}
{% if column.name in show %}
{% block table.thead.th %}
<th class="orderable {{ column.name }}">
<div class="nowrap-parent">
{% if column.orderable %}
<div class="nowrap-child">
{% if column.is_ordered %}
{% is_descending column.order_by as descending %}
{% if descending %}
<span class="icon" aria-hidden="true">{% block table.desc_icon %}<i class="fa-solid fa-sort-down"></i>{% endblock table.desc_icon %}</span>
{% else %}
<span class="icon" aria-hidden="true">{% block table.asc_icon %}<i class="fa-solid fa-sort-up"></i>{% endblock table.asc_icon %}</span>
{% endif %}
{% else %}
<span class="icon" aria-hidden="true">{% block table.orderable_icon %}<i class="fa-solid fa-sort"></i>{% endblock table.orderable_icon %}</span>
{% endif %}
</div>
<div class="nowrap-child">
<a
hx-get="search/partial/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="#drilldown-table"
hx-swap="outerHTML"
hx-indicator="#spinner"
style="cursor: pointer;">
{{ column.header }}
</a>
</div>
{% else %}
<div class="nowrap-child">
{{ column.header }}
</div>
{% endif %}
</div>
</th>
{% endblock table.thead.th %}
{% endif %}
{% endfor %}
</tr>
{% endblock table.thead.row %}
</thead>
{% endif %}
{% endblock table.thead %}
{% block table.tbody %}
<tbody {{ table.attrs.tbody.as_html }}>
{% for row in table.paginated_rows %}
{% block table.tbody.row %}
{% if row.cells.type == 'control' %}
<tr>
<td></td>
<td>
<span class="icon has-text-grey" data-tooltip="Hidden">
<i class="fa-solid fa-file-slash"></i>
</span>
</td>
<td>
<p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
</td>
</tr>
{% else %}
<tr class="
{% if row.cells.exemption == True %}has-background-grey-lighter
{% elif cell == 'join' %}has-background-success-light
{% elif cell == 'quit' %}has-background-danger-light
{% elif cell == 'kick' %}has-background-danger-light
{% elif cell == 'part' %}has-background-warning-light
{% elif cell == 'mode' %}has-background-info-light
{% endif %}">
{% for column, cell in row.items %}
{% if column.name in show %}
{% block table.tbody.td %}
{% if cell == '—' %}
<td class="{{ column.name }}">
<span class="icon">
<i class="fa-solid fa-file-slash"></i>
</span>
</td>
{% elif column.name == 'src' %}
<td class="{{ column.name }}">
<a
class="has-text-grey"
onclick="populateSearch('src', '{{ cell|escapejs }}')">
{% if row.cells.src == 'irc' %}
<span class="icon" data-tooltip="IRC">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{% elif row.cells.src == 'dis' %}
<span class="icon" data-tooltip="Discord">
<i class="fa-brands fa-discord" aria-hidden="true"></i>
</span>
{% elif row.cells.src == '4ch' %}
<span class="icon" data-tooltip="4chan">
<i class="fa-solid fa-leaf" aria-hidden="true"></i>
</span>
{% endif %}
</a>
</td>
{% elif column.name == 'ts' %}
<td class="{{ column.name }}">
<p>{{ row.cells.date }}</p>
<p>{{ row.cells.time }}</p>
</td>
{% elif column.name == 'match_ts' %}
<td class="{{ column.name }}">
{% with match_ts=cell|splitstr:'T' %}
<p>{{ match_ts.0 }}</p>
<p>{{ match_ts.1 }}</p>
{% endwith %}
</td>
{% elif column.name == 'type' or column.name == 'mtype' %}
<td class="{{ column.name }}">
<a
class="has-text-grey"
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
{% if cell == 'msg' %}
<span class="icon" data-tooltip="Message">
<i class="fa-solid fa-message"></i>
</span>
{% elif cell == 'join' %}
<span class="icon" data-tooltip="Join">
<i class="fa-solid fa-person-to-portal"></i>
</span>
{% elif cell == 'part' %}
<span class="icon" data-tooltip="Part">
<i class="fa-solid fa-person-from-portal"></i>
</span>
{% elif cell == 'quit' %}
<span class="icon" data-tooltip="Quit">
<i class="fa-solid fa-circle-xmark"></i>
</span>
{% elif cell == 'kick' %}
<span class="icon" data-tooltip="Kick">
<i class="fa-solid fa-user-slash"></i>
</span>
{% elif cell == 'nick' %}
<span class="icon" data-tooltip="Nick">
<i class="fa-solid fa-signature"></i>
</span>
{% elif cell == 'mode' %}
<span class="icon" data-tooltip="Mode">
<i class="fa-solid fa-gear"></i>
</span>
{% elif cell == 'action' %}
<span class="icon" data-tooltip="Action">
<i class="fa-solid fa-exclamation"></i>
</span>
{% elif cell == 'notice' %}
<span class="icon" data-tooltip="Notice">
<i class="fa-solid fa-message-code"></i>
</span>
{% elif cell == 'conn' %}
<span class="icon" data-tooltip="Connection">
<i class="fa-solid fa-cloud-exclamation"></i>
</span>
{% elif cell == 'znc' %}
<span class="icon" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
{% elif cell == 'query' %}
<span class="icon" data-tooltip="Query">
<i class="fa-solid fa-message"></i>
</span>
{% elif cell == 'highlight' %}
<span class="icon" data-tooltip="Highlight">
<i class="fa-solid fa-exclamation"></i>
</span>
{% elif cell == 'who' %}
<span class="icon" data-tooltip="Who">
<i class="fa-solid fa-passport"></i>
</span>
{% elif cell == 'topic' %}
<span class="icon" data-tooltip="Topic">
<i class="fa-solid fa-sign"></i>
</span>
{% else %}
{{ cell }}
{% endif %}
</a>
</td>
{% elif column.name == 'msg' %}
<td class="{{ column.name }} wrap">
<a
class="has-text-grey is-underlined"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "{{ row.cells.net|escapejs }}",
"num": "{{ row.cells.num|escapejs }}",
"source": "{{ row.cells.src|escapejs }}",
"channel": "{{ row.cells.channel|escapejs }}",
"time": "{{ row.cells.time|escapejs }}",
"date": "{{ row.cells.date|escapejs }}",
"index": "{% if row.cells.index != '—' %}{{row.cells.index}}{% else %}{{ params.index }}{% endif %}",
"type": "{{ row.cells.type }}",
"mtype": "{{ row.cells.mtype }}",
"nick": "{{ row.cells.nick|escapejs }}",
"dedup": "{{ params.dedup }}"}'
hx-target="#modals-here"
hx-trigger="click"
href="/?modal=context&net={{row.cells.net|escapejs}}&num={{row.cells.num|escapejs}}&source={{row.cells.src|escapejs}}&channel={{row.cells.channel|urlsafe}}&time={{row.cells.time|escapejs}}&date={{row.cells.date|escapejs}}&index={{params.index}}&type={{row.cells.type}}&mtype={{row.cells.mtype}}&nick={{row.cells.mtype|escapejs}}">
{{ row.cells.msg }}
</a>
</td>
{% elif column.name == 'nick' %}
<td class="{{ column.name }}">
<div class="nowrap-parent">
<div class="nowrap-child">
{% if row.cells.online is True %}
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
<i class="fa-solid fa-circle"></i>
</span>
{% elif row.cells.online is False %}
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
<i class="fa-solid fa-circle"></i>
</span>
{% else %}
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
<i class="fa-solid fa-circle"></i>
</span>
{% endif %}
</div>
<a class="nowrap-child has-text-grey" onclick="populateSearch('nick', '{{ cell|escapejs }}')">
{{ cell }}
</a>
<div class="nowrap-child">
{% if row.cells.src == 'irc' %}
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' %}"
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"
class="has-text-black">
<span class="icon" data-tooltip="Open drilldown modal">
<i class="fa-solid fa-album"></i>
</span>
</a>
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' type='window' %}"
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
hx-target="#windows-here"
hx-swap="afterend"
hx-trigger="click"
class="has-text-black">
<span class="icon" data-tooltip="Open drilldown window">
<i class="fa-solid fa-album"></i>
</span>
</a>
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' type='widget' %}"
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
hx-target="#widgets-here"
hx-trigger="click"
class="has-text-black">
<span class="icon" data-tooltip="Open drilldown widget">
<i class="fa-solid fa-album"></i>
</span>
</a>
{% endif %}
</div>
{% if row.cells.num_chans != '—' %}
<div class="nowrap-child">
<span class="tag">
{{ row.cells.num_chans }}
</span>
</div>
{% endif %}
</div>
</td>
{% elif column.name == 'channel' %}
<td class="{{ column.name }}">
{% if cell != '—' %}
<div class="nowrap-parent">
<a
class="nowrap-child has-text-grey"
onclick="populateSearch('channel', '{{ cell|escapejs }}')">
{{ cell }}
</a>
{% if row.cells.num_users != '—' %}
<div class="nowrap-child">
<span class="tag">
{{ row.cells.num_users }}
</span>
</div>
{% endif %}
</div>
{% else %}
{{ cell }}
{% endif %}
</td>
{% elif cell is True or cell is False %}
<td class="{{ column.name }}">
{% if cell is True %}
<span class="icon has-text-success">
<i class="fa-solid fa-check"></i>
</span>
{% else %}
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
{% endif %}
</td>
{% elif column.name == "tokens" %}
<td class="{{ column.name }}">
<div class="tags">
{% for word in cell %}
<a
class="tag"
onclick="populateSearch('{{ column.name }}', '{{ word }}')">
{{ word }}
</a>
{% endfor %}
</div>
</td>
{% elif column.name == "meta" %}
<td class="{{ column.name }}">
<pre class="small-field" style="cursor: pointer;">{{ cell|pretty }}</pre>
</td>
{% elif 'id' in column.name and column.name != "ident" %}
<td class="{{ column.name }}">
<div class="buttons">
<div class="nowrap-parent">
<!-- <input class="input" type="text" value="{{ cell }}" style="width: 50px;" readonly> -->
<a
class="has-text-grey button nowrap-child"
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
<span class="icon" data-tooltip="Populate {{ cell }}">
<i class="fa-solid fa-arrow-left-long-to-line" aria-hidden="true"></i>
</span>
</a>
<a
class="has-text-grey button nowrap-child"
onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ cell|escapejs }}');">
<span class="icon" data-tooltip="Copy to clipboard">
<i class="fa-solid fa-copy" aria-hidden="true"></i>
</span>
</a>
</div>
</div>
</td>
{% else %}
<td class="{{ column.name }}">
<a
class="has-text-grey"
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
{{ cell }}
</a>
</td>
{% endif %}
{% endblock table.tbody.td %}
{% endif %}
{% endfor %}
</tr>
{% endif %}
{% endblock table.tbody.row %}
{% empty %}
{% if table.empty_text %}
{% block table.tbody.empty_text %}
<tr><td class="{{ column.name }}" colspan="{{ table.columns|length }}">{{ table.empty_text }}</td></tr>
{% endblock table.tbody.empty_text %}
{% endif %}
{% endfor %}
</tbody>
{% endblock table.tbody %}
{% block table.tfoot %}
{% if table.has_footer %}
<tfoot {{ table.attrs.tfoot.as_html }}>
{% block table.tfoot.row %}
<tr>
{% for column in table.columns %}
{% block table.tfoot.td %}
<td class="{{ column.name }}" {{ column.attrs.tf.as_html }}>{{ column.footer }}</td>
{% endblock table.tfoot.td %}
{% endfor %}
</tr>
{% endblock table.tfoot.row %}
</tfoot>
{% endif %}
{% endblock table.tfoot %}
</table>
</div>
{% endblock table %}
{% block pagination %}
{% if table.page and table.paginator.num_pages > 1 %}
<nav class="pagination is-justify-content-flex-end" role="navigation" aria-label="pagination">
{% block pagination.previous %}
<a
class="pagination-previous is-flex-grow-0 {% if not table.page.has_previous %}is-hidden-mobile{% endif %}"
{% if table.page.has_previous %}
hx-get="search/partial/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="#drilldown-table"
hx-swap="outerHTML"
hx-indicator="#spinner"
{% else %}
href="#"
disabled
{% endif %}
style="order:1;">
{% block pagination.previous.text %}
<span aria-hidden="true">&laquo;</span>
{% endblock pagination.previous.text %}
</a>
{% endblock pagination.previous %}
{% block pagination.next %}
<a
class="pagination-next is-flex-grow-0 {% if not table.page.has_next %}is-hidden-mobile{% endif %}"
{% if table.page.has_next %}
hx-get="search/partial/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="#drilldown-table"
hx-swap="outerHTML"
hx-indicator="#spinner"
{% else %}
href="#"
disabled
{% endif %}
style="order:3;"
>
{% block pagination.next.text %}
<span aria-hidden="true">&raquo;</span>
{% endblock pagination.next.text %}
</a>
{% endblock pagination.next %}
{% if table.page.has_previous or table.page.has_next %}
{% block pagination.range %}
<ul class="pagination-list is-flex-grow-0" style="order:2;">
{% for p in table.page|table_page_range:table.paginator %}
<li>
<a
class="pagination-link {% if p == table.page.number %}is-current{% endif %}"
aria-label="Page {{ p }}" block
{% if p == table.page.number %}aria-current="page"{% endif %}
{% if p == table.page.number %}
href="#"
{% else %}
hx-get="search/partial/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="#drilldown-table"
hx-swap="outerHTML"
hx-indicator="#spinner"
{% endif %}
>
{% if p == '...' %}
<span class="pagination-ellipsis">&hellip;</span>
{% else %}
{{ p }}
{% endif %}
</a>
</li>
{% endfor %}
</ul>
{% endblock pagination.range %}
{% endif %}
</nav>
{% endif %}
{% endblock pagination %}
</div>
{% endblock table-wrapper %}
{% endcache %}

109
core/templates/partials/rule-list.html
View File

@ -1,109 +0,0 @@
{% load cache %}
{% load cachalot cache %}
{% get_last_invalidation 'core.NotificationRule' as last %}
{% include 'mixins/partials/notify.html' %}
{% cache 600 objects_rules request.user.id object_list last %}
<table
class="table is-fullwidth is-hoverable"
hx-target="#{{ context_object_name }}-table"
id="{{ context_object_name }}-table"
hx-swap="outerHTML"
hx-trigger="{{ context_object_name_singular }}Event from:body"
hx-get="{{ list_url }}">
<thead>
<th>id</th>
<th>user</th>
<th>name</th>
<th>interval</th>
<th>window</th>
<th>priority</th>
<th>topic</th>
<th>enabled</th>
<th>ingest</th>
<th>data length</th>
<th>match</th>
<th>actions</th>
</thead>
{% for item in object_list %}
<tr>
<td><a href="/?query=*&source=all&rule={{ item.id }}">{{ item.id }}</a></td>
<td>{{ item.user }}</td>
<td>{{ item.name }}</td>
<td>{{ item.interval }}s</td>
<td>{{ item.window|default_if_none:"—" }}</td>
<td>{{ item.priority }}</td>
<td>{{ item.topic|default_if_none:"—" }}</td>
<td>
{% if item.enabled %}
<span class="icon">
<i class="fa-solid fa-check"></i>
</span>
{% else %}
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
{% endif %}
</td>
<td>
{% if item.ingest %}
<span class="icon">
<i class="fa-solid fa-check"></i>
</span>
{% else %}
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
{% endif %}
</td>
<td>{{ item.data|length }}</td>
<td>{{ item.matches }}</td>
<td>
<div class="buttons">
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-get="{% url 'rule_update' type=type pk=item.id %}"
hx-trigger="click"
hx-target="#{{ type }}s-here"
hx-swap="innerHTML"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-pencil"></i>
</span>
</span>
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-delete="{% url 'rule_delete' type=type pk=item.id %}"
hx-trigger="click"
hx-target="#modals-here"
hx-swap="innerHTML"
hx-confirm="Are you sure you wish to delete {{ item.name }}?"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-xmark"></i>
</span>
</span>
</button>
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'rule_clear' type=type pk=item.id %}"
hx-trigger="click"
hx-target="#modals-here"
hx-swap="innerHTML"
hx-confirm="Are you sure you wish to clear matches for {{ item.name }}?"
class="button">
<span class="icon-text">
<span class="icon">
<i class="fa-solid fa-arrow-rotate-right"></i>
</span>
</span>
</button>
</div>
</td>
</tr>
{% endfor %}
</table>
{% endcache %}

10
core/templates/partials/sentiment_chart.html
View File

@ -1,10 +0,0 @@
{% load static %}
<div style="display: none" id="jsonData" data-json="{{ data }}">
</div>
{% if params.index != 'int' and params.index != 'meta' %}
<div id="sentiment-container" {% if params.show_sentiment is None %} class="is-hidden" {% endif %}>
<canvas id="sentiment-chart"></canvas>
</div>
<script src="{% static 'chart.js' %}"></script>
{% endif %}

378
core/templates/ui/drilldown/drilldown.html Normal file
View File

@ -0,0 +1,378 @@
{% extends "base.html" %}
{% load static %}
{% load joinsep %}
{% block content %}
<script src="{% static 'js/chart.js' %}"></script>
<script>
function setupTags() {
var inputTags = document.getElementById('tags');
new BulmaTagsInput(inputTags);
inputTags.BulmaTagsInput().on('before.add', function(item) {
if (item.includes(": ")) {
var spl = item.split(": ");
} else {
var spl = item.split(":");
}
var field = spl[0];
try {
var value = JSON.parse(spl[1]);
} catch {
var value = spl[1];
}
populateSearch(field, value);
return `${field}: ${value}`;
});
inputTags.BulmaTagsInput().on('after.remove', function(item) {
var spl = item.split(": ");
var field = spl[0];
try {
var value = JSON.parse(spl[1]);
} catch {
var value = spl[1].trim();
}
populateSearch(field, value);
});
}
function populateSearch(field, value) {
var queryElement = document.getElementById('query');
var present = true;
if (present == true) {
var combinations = [`${field}: "${value}"`,
`${field}: "${value}"`,
`${field}: ${value}`,
`${field}:${value}`,
`${field}:"${value}"`];
var toAppend = ` AND ${field}: "${value}"`;
// var toRemove = `${field}: "${value}"`;
// var tagText = `${field}: ${value}`;
} else {
var combinations = [`NOT ${field}: "${value}"`,
`NOT ${field}: "${value}"`,
`NOT ${field}: ${value}`,
`NOT ${field}:${value}`,
`NOT ${field}:"${value}"`];
// var toAppend = ` AND NOT ${field}: "${value}"`;
// var toRemove = `NOT ${field}: "${value}"`;
}
var contains = combinations.some(elem => queryElement.value.includes(elem));
if (!contains) {
queryElement.value+=toAppend;
} else {
for (var index in combinations) {
combination = combinations[index];
queryElement.value = queryElement.value.replaceAll("AND "+combination, "");
queryElement.value = queryElement.value.replaceAll(combination, "");
}
// queryElement.value = queryElement.value.replaceAll(toAppend, "");
// queryElement.value = queryElement.value.replaceAll(toRemove, "");
}
// if (!queryElement.value.includes(toAppend) && !queryElement.value.includes(toRemove)) {
// queryElement.value+=toAppend;
// } else {
// queryElement.value = queryElement.value.replaceAll(toAppend, "");
// queryElement.value = queryElement.value.replaceAll(toRemove, "");
// }
if (field == "src") {
document.getElementById("source").selectedIndex = 0;
}
if (queryElement.value.startsWith(" AND ")) {
queryElement.value = queryElement.value.replace(" AND ", "");
}
if (queryElement.value.startsWith("AND ")) {
queryElement.value = queryElement.value.replace("AND ", "");
}
htmx.trigger("#search", "click");
}
</script>
<div>
<form method="POST" hx-post="{% url 'home' %}"
hx-trigger="change"
hx-target="#results"
hx-swap="innerHTML"
hx-indicator="#spinner">
{% csrf_token %}
<div class="columns">
<div class="column">
<div class="field has-addons">
<div class="control is-expanded has-icons-left">
<input
hx-post="{% url 'home' %}"
hx-trigger="keyup changed delay:200ms"
hx-target="#results"
hx-swap="innerHTML" id="query" name="query" value="{{ params.query }}" class="input" type="text" placeholder="msg: science AND nick: BillNye AND channel: #science">
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</div>
<div class="control">
<div class="field">
<button
id="search"
class="button is-info is-fullwidth"
hx-post="{% url 'home' %}"
hx-trigger="click"
hx-target="#results"
hx-swap="innerHTML">
Search
</button>
</div>
</div>
</div>
</div>
<div class="column is-3">
<div class="nowrap-parent">
<div
data-script="on click toggle .is-hidden on #options"
class="button is-light has-text-link is-right nowrap-child">
Options
</div>
<div class="nowrap-child">
<span id="spinner" class="button is-light has-text-link is-loading htmx-indicator">Static</span>
</div>
</div>
</div>
</div>
<div id="options" class="box is-hidden">
<div class="columns is-multiline">
<div class="column is-narrow">
<div class="field has-addons">
<div class="control has-icons-left">
<span class="select">
<select name="size">
{% for size in sizes %}
{% if size == params.size %}
<option selected value="{{ size }}">{{ size }}</option>
{% else %}
<option value="{{ size }}">{{ size }}</option>
{% endif %}
{% endfor %}
</select>
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</span>
</div>
<p class="control">
<a class="button is-static">
results
</a>
</p>
</div>
</div>
<div class="column is-narrow">
<div class="field has-addons">
<div class="control has-icons-left">
<span class="select">
<select id="source" name="source">
{% if params.source == 'irc' %}
<option selected value="irc">IRC</option>
{% else %}
<option value="irc">IRC</option>
{% endif %}
{% if params.source == 'dis' %}
<option selected value="dis">Discord</option>
{% else %}
<option value="dis">Discord</option>
{% endif %}
{% if params.source == None %}
<option selected value="all">All</option>
{% elif params.source == 'all' %}
<option selected value="all">All</option>
{% else %}
<option value="all">All</option>
{% endif %}
</select>
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</span>
</div>
<p class="control">
<a class="button is-static">
source
</a>
</p>
</div>
</div>
<div class="column is-narrow">
<div id="sentiment">
<div class="field has-addons">
<div class="control">
<input
{% if params.check_sentiment != "on" %}
disabled="undefined"
{% endif %}
name="sentiment" id="sliderWithValue" class="slider has-output-tooltip is-fullwidth" min="-1" max="1"
{% if params.sentiment == None %}
value="0"
{% else %}
value="{{ params.sentiment }}"
{% endif %}
step="0.05" type="range">
<output for="sliderWithValue" class="slider-output">
{% if params.sentiment == None %}
0
{% else %}
{{ params.sentiment }}
{% endif %}
</output>
<script>bulmaSlider.attach();</script>
</div>
<p class="control">
<a class="button is-static">
sentiment
</a>
</p>
</div>
<div class="control">
<label class="radio button has-text-link">
<input type="radio"
value="below"
{% if params.sentiment_method == 'below' %}
checked
{% endif %}
name="sentiment_method">
<span class="icon" data-tooltip="Below">
<i class="fa-solid fa-face-frown"></i>
</span>
</label>
<label class="radio button has-text-link is-hidden">
<input type="radio"
value="exact"
{% if params.sentiment_method == 'exact' %}
checked
{% endif %}
name="sentiment_method">
<span class="icon" data-tooltip="Exact">
<i class="fa-solid fa-face-smile"></i>
</span>
</label>
<label class="radio button has-text-link">
<input type="radio"
value="above"
{% if params.sentiment_method == 'above' %}
checked
{% endif %}
name="sentiment_method">
<span class="icon" data-tooltip="Above">
<i class="fa-solid fa-face-smile"></i>
</span>
</label>
<label class="radio button has-text-link">
<input type="radio"
value="nonzero"
{% if params.sentiment_method == 'nonzero' %}
checked
{% endif %}
name="sentiment_method">
<span class="icon" data-tooltip="Nonzero">
<i class="fa-solid fa-face-meh-blank"></i>
</span>
</label>
</div>
</div>
<label class="checkbox">
<input type="checkbox"
name="check_sentiment"
{% if params.check_sentiment == "on" %}
checked
{% endif %}
data-script="on click toggle @disabled on #sliderWithValue then toggle @disabled on #sentiment">
Check sentiment
</label>
</div>
<div class="column is-narrow">
<div id="date">
<div class="field">
<div class="control">
<input type="date" name="dates" value="{{ params.date }}">
<script>
var options = {
"type": "datetime",
"isRange": true,
"color": "info",
"validateLabel": "Save",
"dateFormat": "yyyy-MM-dd",
"startDate": "{{ params.from_date|escapejs }}",
"startTime": "{{ params.from_time|escapejs }}",
"endDate": "{{ params.to_date|escapejs }}",
"endTime": "{{ params.to_time|escapejs }}",
};
// Initialize all input of type date
var calendars = bulmaCalendar.attach('[type="date"]', options);
// Loop on each calendar initialized
for(var i = 0; i < calendars.length; i++) {
// Add listener to select event
calendars[i].on('save', date => {
htmx.trigger("#search", "click");
});
}
</script>
</div>
</div>
<div class="control">
<label class="radio button has-text-link">
<input type="radio" value="desc" name="sorting"
{% if params.sorting == None %}
checked
{% elif params.sorting == 'desc' %}
checked
{% endif %}
>
<span class="icon" data-tooltip="Sort descending">
<i class="fa-solid fa-sort-down"></i>
</span>
</label>
<label class="radio button">
<input type="radio" value="asc" name="sorting"
{% if params.sorting == 'asc' %}
checked
{% endif %}>
<span class="icon" data-tooltip="Sort ascending">
<i class="fa-solid fa-sort-up"></i>
</span>
</label>
<label class="radio button">
<input type="radio" value="none" name="sorting"
{% if params.sorting == 'none' %}
checked
{% endif %}>
<span class="icon" data-tooltip="No sort">
<i class="fa-solid fa-sort"></i>
</span>
</label>
</div>
</div>
</div>
</div>
</div>
<div class="is-hidden"></div>
</form>
</div>
<div class="block">
<input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">
<script>
setupTags();
</script>
</div>
<div class="block">
<div id="results">
{% if results %}
{% include 'ui/drilldown/results.html' %}
{% endif %}
</div>
</div>
<div id="modals-here">
</div>
{% endblock %}

212
core/templates/ui/drilldown/results.html Normal file
View File

@ -0,0 +1,212 @@
{% load static %}
{% load index %}
{% load joinsep %}
{% include 'partials/notify.html' %}
{% if results %}
<div style="display: none" id="jsonData" data-json="{{ data }}">
</div>
<div class="has-text-grey-light nowrap-parent">
<div class="nowrap-child block">
<i class="fa-solid fa-chart-mixed"></i>
</div>
<div class="nowrap-child">
<p>fetched {{ results|length }} of {{ card }} hits in {{ took }}ms</p>
</div>
{% if exemption is not None %}
<div class="nowrap-child">
<i class="fa-solid fa-book-bible"></i>
</div>
{% else %}
{% if redacted != 0 %}
<div class="nowrap-child">
<p>{{ redacted }} redacted</p>
</div>
{% endif %}
{% endif %}
</div>
<div class="box">
<div style="height: 30rem">
<canvas id="volume"></canvas>
</div>
<script src="{% static 'chart.js' %}"></script>
</div>
<div class="box">
<div class="table-container">
<table class="table is-striped is-hoverable is-fullwidth">
<thead>
<tr>
<th>src</th>
<th>type</th>
<th>ts</th>
<th>msg</th>
<th>host</th>
<th>nick</th>
<th>actions</th>
<th>channel</th>
<th>net</th>
</tr>
</thead>
<tbody>
{% for item in results %}
{# Workaround for curlylint #}
<tr class="{% if item.exemption == True %}
has-background-grey-lighter
{% elif item.type == 'join' %}
has-background-success-light
{% elif item.type == 'quit' %}
has-background-danger-light
{% elif item.type == 'kick' %}
has-background-danger-light
{% elif item.type == 'part' %}
has-background-warning-light
{% elif item.type == 'mode' %}
has-background-info-light
{% endif %}">
<td>
<a class="has-text-link is-underlined"
onclick="populateSearch('src', '{{ item.src|escapejs }}')">
{% if item.src == 'irc' %}
<span class="icon" data-tooltip="IRC">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{% elif item.src == 'dis' %}
<span class="icon" data-tooltip="Discord">
<i class="fa-brands fa-discord" aria-hidden="true"></i>
</span>
{% endif %}
</a>
</td>
<td>
<a class="has-text-link is-underlined"
onclick="populateSearch('type', '{{ item.type|escapejs }}')">
{% if item.type == 'msg' %}
<span class="icon" data-tooltip="Message">
<i class="fa-solid fa-message"></i>
</span>
{% elif item.type == 'join' %}
<span class="icon" data-tooltip="Join">
<i class="fa-solid fa-person-to-portal"></i>
</span>
{% elif item.type == 'part' %}
<span class="icon" data-tooltip="Part">
<i class="fa-solid fa-person-from-portal"></i>
</span>
{% elif item.type == 'quit' %}
<span class="icon" data-tooltip="Quit">
<i class="fa-solid fa-circle-xmark"></i>
</span>
{% elif item.type == 'kick' %}
<span class="icon" data-tooltip="Kick">
<i class="fa-solid fa-user-slash"></i>
</span>
{% elif item.type == 'nick' %}
<span class="icon" data-tooltip="Nick">
<i class="fa-solid fa-signature"></i>
</span>
{% elif item.type == 'mode' %}
<span class="icon" data-tooltip="Mode">
<i class="fa-solid fa-gear"></i>
</span>
{% elif item.type == 'action' %}
<span class="icon" data-tooltip="Action">
<i class="fa-solid fa-exclamation"></i>
</span>
{% else %}
{{ item.type }}
{% endif %}
</a>
</td>
<td>
<p>{{ item.date }}</p>
<p>{{ item.time }}</p>
</td>
<td style="max-width: 10em" class="wrap">{{ item.msg }}</td>
<td>
<a class="has-text-link is-underlined"
onclick="populateSearch('host', '{{ item.host|escapejs }}')">
{{ item.host }}
</a>
</td>
<td>
<div class="nowrap-parent">
<div class="nowrap-child">
{% if item.online is True %}
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
<i class="fa-solid fa-circle"></i>
</span>
{% elif item.online is False %}
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
<i class="fa-solid fa-circle"></i>
</span>
{% else %}
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
<i class="fa-solid fa-circle"></i>
</span>
{% endif %}
</div>
<a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
{{ item.nick }}
</a>
{% if item.num_chans is not None %}
<div class="nowrap-child">
<span class="tag">
{{ item.num_chans }}
</span>
</div>
{% endif %}
</div>
</td>
<td>
{% if item.src == 'irc' %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small">
Information
</button>
{% endif %}
</td>
<td>
<div class="nowrap-parent">
<a class="nowrap-child has-text-link is-underlined"
onclick="populateSearch('channel', '{{ item.channel|escapejs }}')">
{{ item.channel }}
</a>
{% if item.num_users is not None %}
<div class="nowrap-child">
<span class="tag">
{{ item.num_users }}
</span>
</div>
{% endif %}
</div>
</td>
<td>
<a class="has-text-link is-underlined"
onclick="populateSearch('net', '{{ item.net|escapejs }}')">
{{ item.net }}
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endif %}
{# Update the tags in case the user changed the query #}
{# Check for focus and refocus #}
<script>
var inputTags = document.getElementsByClassName('tags-input');
var inputBox = document.querySelector("[placeholder='Add query']");
var isFocused = (document.activeElement === inputBox);
inputTags[0].outerHTML = '<input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">';
setupTags();
var inputBox = document.querySelector("[placeholder='Add query']");
if (isFocused) {
inputBox.focus();
}
</script>

32
core/templates/ui/drilldown/table_results.html Normal file
View File

@ -0,0 +1,32 @@
{% load static %}
{% load index %}
{% include 'partials/notify.html' %}
{% if table %}
<div style="display: none" id="jsonData" data-json="{{ data }}">
</div>
<div class="has-text-grey-light nowrap-parent">
<div class="nowrap-child block">
<i class="fa-solid fa-chart-mixed"></i>
</div>
<div class="nowrap-child">
<p>fetched {{ results|length }} of {{ card }} hits in {{ took }}ms</p>
</div>
{% if exemption is not None %}
<div class="nowrap-child">
<i class="fa-solid fa-book-bible"></i>
</div>
{% else %}
{% if redacted != 0 %}
<div class="nowrap-child">
<p>{{ redacted }} redacted</p>
</div>
{% endif %}
{% endif %}
</div>
<div class="box">
<div class="table-container">
{% include 'ui/drilldown/table_results_partial.html' %}
</div>
</div>
{% endif %}

75
core/templates/ui/drilldown/table_results_partial.html Normal file
View File

@ -0,0 +1,75 @@
{% extends 'django-tables2/bulma.html' %}
{% load django_tables2 %}
{% load i18n %}
{% block table.thead %}
{% if table.show_header %}
<thead {{ table.attrs.thead.as_html }}>
<tr>
{% for column in table.columns %}
<th
{{ column.attrs.th.as_html }}
hx-post="{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="div.table-container"
hx-swap="outerHTML"
hx-indicator=".progress"
style="cursor: pointer;">
{{ column.header }}
</th>
{% endfor %}
</tr>
</thead>
{% endif %}
{% endblock table.thead %}
{# Pagination block overrides #}
{% block pagination.previous %}
<li class="previous page-item">
<div
hx-post="{% querystring table.prefixed_page_field=table.page.previous_page_number %}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="div.table-container"
hx-swap="outerHTML"
hx-indicator=".progress"
class="page-link">
<span aria-hidden="true">&laquo;</span>
{% trans 'previous' %}
</div>
</li>
{% endblock pagination.previous %}
{% block pagination.range %}
{% for p in table.page|table_page_range:table.paginator %}
<li class="page-item{% if table.page.number == p %} active{% endif %}">
<div
class="page-link"
{% if p != '...' %}hx-post="{% querystring table.prefixed_page_field=p %}"{% endif %}
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="div.table-container"
hx-swap="outerHTML"
hx-indicator=".progress">
{{ p }}
</div>
</li>
{% endfor %}
{% endblock pagination.range %}
{% block pagination.next %}
<li class="next page-item">
<div
hx-post="{% querystring table.prefixed_page_field=table.page.next_page_number %}"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-trigger="click"
hx-target="div.table-container"
hx-swap="outerHTML"
hx-indicator=".progress"
class="page-link">
{% trans 'next' %}
<span aria-hidden="true">&raquo;</span>
</div>
</li>
{% endblock pagination.next %}

7
core/templates/ui/insights/info.html
View File

@ -4,7 +4,7 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'chans_insights' index=index %}"
hx-post="{% url 'chans_insights' %}"
hx-trigger="load"
hx-target="#channels"
hx-swap="outerHTML">
@ -13,13 +13,12 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'nicks_insights' index=index %}"
hx-post="{% url 'nicks_insights' %}"
hx-trigger="load"
hx-target="#nicks"
hx-swap="outerHTML">
</div>
<div id="info">
{% include 'mixins/partials/notify.html' %}
{% if item is not None %}
<div class="content" style="max-height: 30em; overflow: auto;">
<div class="table-container">
@ -81,7 +80,7 @@
{% if item.src == 'irc' %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_insights' index=index %}"
hx-post="{% url 'modal_insights' %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"

39
core/templates/ui/insights/insights.html
View File

@ -1,8 +1,39 @@
{% extends "base.html" %}
{% load static %}
{% block content %}
{% include 'mixins/partials/notify.html' %}
<script src="{% static 'tabs.js' %}"></script>
<script>
// tabbed browsing for the modal
function initTabs() {
TABS.forEach((tab) => {
tab.addEventListener('click', (e) => {
let selected = tab.getAttribute('data-tab');
updateActiveTab(tab);
updateActiveContent(selected);
})
})
}
function updateActiveTab(selected) {
TABS.forEach((tab) => {
if (tab && tab.classList.contains(ACTIVE_CLASS)) {
tab.classList.remove(ACTIVE_CLASS);
}
});
selected.classList.add(ACTIVE_CLASS);
}
function updateActiveContent(selected) {
CONTENT.forEach((item) => {
if (item && item.classList.contains(ACTIVE_CLASS)) {
item.classList.remove(ACTIVE_CLASS);
}
let data = item.getAttribute('data-content');
if (data === selected) {
item.classList.add(ACTIVE_CLASS);
}
});
}
</script>
<style>
.icon { border-bottom: 0px !important;}
</style>
@ -15,7 +46,7 @@
{% csrf_token %}
<div class="field has-addons">
<div class="control is-expanded has-icons-left">
<input id="query_full" name="query" class="input" type="text" placeholder="nickname">
<input id="query" name="query" class="input" type="text" placeholder="nickname">
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
@ -23,7 +54,7 @@
<div class="control">
<button
class="button is-info is-fullwidth"
hx-post="{% url 'search_insights' index=index %}"
hx-post="{% url 'search_insights' %}"
hx-trigger="click"
hx-target="#info"
hx-swap="outerHTML">

2
core/templates/ui/insights/nicks.html
View File

@ -3,7 +3,7 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}'
hx-post="{% url 'meta_insights' index=index %}"
hx-post="{% url 'meta_insights' %}"
hx-trigger="load"
hx-target="#meta"
hx-swap="outerHTML">

19
core/templates/widgets/drilldown.html
View File

@ -1,19 +0,0 @@
{% extends 'mixins/wm/widget.html' %}
{% block widget_options %}
gs-w="5" gs-h="15"
{% endblock %}
{% block heading %}
Drilldown
{% endblock %}
{% block panel_content %}
{% include 'window-content/drilldown.html' %}
{% endblock %}
{% block custom_script_end %}
initTabs("{{ unique }}");
{% endblock %}

Some files were not shown because too many files have changed in this diff Show More