Compare commits
121 Commits
@ -0,0 +1,28 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM python:3
|
||||
ARG OPERATION
|
||||
|
||||
RUN useradd -d /code pathogen
|
||||
RUN mkdir -p /code
|
||||
RUN chown -R pathogen:pathogen /code
|
||||
|
||||
RUN mkdir -p /conf/static
|
||||
RUN chown -R pathogen:pathogen /conf
|
||||
|
||||
RUN mkdir /venv
|
||||
RUN chown pathogen:pathogen /venv
|
||||
|
||||
USER pathogen
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
WORKDIR /code
|
||||
COPY requirements.txt /code/
|
||||
RUN python -m venv /venv
|
||||
RUN . /venv/bin/activate && pip install -r requirements.txt
|
||||
|
||||
# CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini
|
||||
|
||||
CMD if [ "$OPERATION" = "uwsgi" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi
|
||||
|
||||
# CMD . /venv/bin/activate && uvicorn --reload --reload-include *.html --workers 2 --uds /var/run/socks/app.sock app.asgi:application
|
||||
# CMD . /venv/bin/activate && gunicorn -b 0.0.0.0:8000 --reload app.asgi:application -k uvicorn.workers.UvicornWorker
|
||||
@ -0,0 +1,20 @@
|
||||
run:
|
||||
docker-compose --env-file=stack.env up -d
|
||||
|
||||
build:
|
||||
docker-compose --env-file=stack.env build
|
||||
|
||||
stop:
|
||||
docker-compose --env-file=stack.env down
|
||||
|
||||
log:
|
||||
docker-compose --env-file=stack.env logs -f
|
||||
|
||||
migrate:
|
||||
docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py migrate"
|
||||
|
||||
makemigrations:
|
||||
docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py makemigrations"
|
||||
|
||||
auth:
|
||||
docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py createsuperuser"
|
||||
@ -0,0 +1,692 @@
|
||||
# from copy import deepcopy
|
||||
# from datetime import datetime, timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import AsyncElasticsearch, Elasticsearch
|
||||
from elasticsearch.exceptions import NotFoundError, RequestError
|
||||
|
||||
from core.db import StorageBackend, add_defaults
|
||||
|
||||
# from json import dumps
|
||||
# pp = lambda x: print(dumps(x, indent=2))
|
||||
from core.db.processing import parse_results
|
||||
from core.lib.parsing import (
|
||||
QueryError,
|
||||
parse_date_time,
|
||||
parse_index,
|
||||
parse_rule,
|
||||
parse_sentiment,
|
||||
parse_size,
|
||||
parse_sort,
|
||||
parse_source,
|
||||
)
|
||||
|
||||
# These are sometimes numeric, sometimes strings.
|
||||
# If they are seen to be numeric first, ES will erroneously
|
||||
# index them as "long" and then subsequently fail to index messages
|
||||
# with strings in the field.
|
||||
keyword_fields = ["nick_id", "user_id", "net_id"]
|
||||
|
||||
mapping = {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"ts": {"type": "date", "format": "epoch_second"},
|
||||
"match_ts": {"type": "date", "format": "iso8601"},
|
||||
"file_tim": {"type": "date", "format": "epoch_millis"},
|
||||
"rule_id": {"type": "keyword"},
|
||||
}
|
||||
}
|
||||
}
|
||||
for field in keyword_fields:
|
||||
mapping["mappings"]["properties"][field] = {"type": "text"}
|
||||
|
||||
|
||||
class ElasticsearchBackend(StorageBackend):
|
||||
def __init__(self):
|
||||
super().__init__("elasticsearch")
|
||||
self.client = None
|
||||
self.async_client = None
|
||||
|
||||
def initialise(self, **kwargs):
|
||||
"""
|
||||
Inititialise the Elasticsearch API endpoint.
|
||||
"""
|
||||
auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
|
||||
client = Elasticsearch(
|
||||
settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
|
||||
)
|
||||
self.client = client
|
||||
|
||||
async def async_initialise(self, **kwargs):
|
||||
"""
|
||||
Inititialise the Elasticsearch API endpoint in async mode.
|
||||
"""
|
||||
global mapping
|
||||
auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
|
||||
client = AsyncElasticsearch(
|
||||
settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
|
||||
)
|
||||
self.async_client = client
|
||||
|
||||
# Create the rule storage indices
|
||||
if await client.indices.exists(index=settings.INDEX_RULE_STORAGE):
|
||||
await client.indices.put_mapping(
|
||||
index=settings.INDEX_RULE_STORAGE,
|
||||
properties=mapping["mappings"]["properties"],
|
||||
)
|
||||
else:
|
||||
await client.indices.create(
|
||||
index=settings.INDEX_RULE_STORAGE, mappings=mapping["mappings"]
|
||||
)
|
||||
|
||||
def delete_rule_entries(self, rule_id):
|
||||
"""
|
||||
Delete all entries for a given rule.
|
||||
:param rule_id: The rule ID to delete.
|
||||
"""
|
||||
if self.client is None:
|
||||
self.initialise()
|
||||
search_query = self.construct_query(None, None, blank=True)
|
||||
search_query["query"]["bool"]["must"].append(
|
||||
{"match_phrase": {"rule_id": rule_id}}
|
||||
)
|
||||
return self.client.delete_by_query(
|
||||
index=settings.INDEX_RULE_STORAGE, body=search_query
|
||||
)
|
||||
|
||||
def construct_context_query(
|
||||
self, index, net, channel, src, num, size, type=None, nicks=None
|
||||
):
|
||||
# Get the initial query
|
||||
query = self.construct_query(None, size, blank=True)
|
||||
|
||||
extra_must = []
|
||||
extra_should = []
|
||||
extra_should2 = []
|
||||
if num:
|
||||
extra_must.append({"match_phrase": {"num": num}})
|
||||
if net:
|
||||
extra_must.append({"match_phrase": {"net": net}})
|
||||
if channel:
|
||||
extra_must.append({"match": {"channel": channel}})
|
||||
if nicks:
|
||||
for nick in nicks:
|
||||
extra_should2.append({"match": {"nick": nick}})
|
||||
|
||||
types = ["msg", "notice", "action", "kick", "topic", "mode"]
|
||||
fields = [
|
||||
"nick",
|
||||
"ident",
|
||||
"host",
|
||||
"channel",
|
||||
"ts",
|
||||
"msg",
|
||||
"type",
|
||||
"net",
|
||||
"src",
|
||||
"tokens",
|
||||
]
|
||||
query["fields"] = fields
|
||||
|
||||
if index == "internal":
|
||||
fields.append("mtype")
|
||||
if channel == "*status" or type == "znc":
|
||||
if {"match": {"channel": channel}} in extra_must:
|
||||
extra_must.remove({"match": {"channel": channel}})
|
||||
extra_should2 = []
|
||||
# Type is one of msg or notice
|
||||
# extra_should.append({"match": {"mtype": "msg"}})
|
||||
# extra_should.append({"match": {"mtype": "notice"}})
|
||||
extra_should.append({"match": {"type": "znc"}})
|
||||
extra_should.append({"match": {"type": "self"}})
|
||||
|
||||
extra_should2.append({"match": {"type": "znc"}})
|
||||
extra_should2.append({"match": {"nick": channel}})
|
||||
elif type == "auth":
|
||||
if {"match": {"channel": channel}} in extra_must:
|
||||
extra_must.remove({"match": {"channel": channel}})
|
||||
extra_should2 = []
|
||||
extra_should2.append({"match": {"nick": channel}})
|
||||
# extra_should2.append({"match": {"mtype": "msg"}})
|
||||
# extra_should2.append({"match": {"mtype": "notice"}})
|
||||
|
||||
extra_should.append({"match": {"type": "query"}})
|
||||
extra_should2.append({"match": {"type": "self"}})
|
||||
extra_should.append({"match": {"nick": channel}})
|
||||
else:
|
||||
for ctype in types:
|
||||
extra_should.append({"match": {"mtype": ctype}})
|
||||
else:
|
||||
for ctype in types:
|
||||
extra_should.append({"match": {"type": ctype}})
|
||||
# query = {
|
||||
# "index": index,
|
||||
# "limit": size,
|
||||
# "query": {
|
||||
# "bool": {
|
||||
# "must": [
|
||||
# # {"equals": {"src": src}},
|
||||
# # {
|
||||
# # "bool": {
|
||||
# # "should": [*extra_should],
|
||||
# # }
|
||||
# # },
|
||||
# # {
|
||||
# # "bool": {
|
||||
# # "should": [*extra_should2],
|
||||
# # }
|
||||
# # },
|
||||
# *extra_must,
|
||||
# ]
|
||||
# }
|
||||
# },
|
||||
# "fields": fields,
|
||||
# # "_source": False,
|
||||
# }
|
||||
if extra_must:
|
||||
for x in extra_must:
|
||||
query["query"]["bool"]["must"].append(x)
|
||||
if extra_should:
|
||||
query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
|
||||
if extra_should2:
|
||||
query["query"]["bool"]["must"].append(
|
||||
{"bool": {"should": [*extra_should2]}}
|
||||
)
|
||||
return query
|
||||
|
||||
def construct_query(self, query, size=None, blank=False, **kwargs):
|
||||
"""
|
||||
Accept some query parameters and construct an Elasticsearch query.
|
||||
"""
|
||||
query_base = {
|
||||
# "size": size,
|
||||
"query": {"bool": {"must": []}},
|
||||
}
|
||||
if size:
|
||||
query_base["size"] = size
|
||||
query_string = {
|
||||
"query_string": {
|
||||
"query": query,
|
||||
# "fields": fields,
|
||||
# "default_field": "msg",
|
||||
# "type": "best_fields",
|
||||
"fuzziness": "AUTO",
|
||||
"fuzzy_transpositions": True,
|
||||
"fuzzy_max_expansions": 50,
|
||||
"fuzzy_prefix_length": 0,
|
||||
# "minimum_should_match": 1,
|
||||
"default_operator": "and",
|
||||
"analyzer": "standard",
|
||||
"lenient": True,
|
||||
"boost": 1,
|
||||
"allow_leading_wildcard": True,
|
||||
# "enable_position_increments": False,
|
||||
"phrase_slop": 3,
|
||||
# "max_determinized_states": 10000,
|
||||
"quote_field_suffix": "",
|
||||
"quote_analyzer": "standard",
|
||||
"analyze_wildcard": False,
|
||||
"auto_generate_synonyms_phrase_query": True,
|
||||
}
|
||||
}
|
||||
if not blank:
|
||||
query_base["query"]["bool"]["must"].append(query_string)
|
||||
return query_base
|
||||
|
||||
def parse(self, response, **kwargs):
|
||||
parsed = parse_results(response, **kwargs)
|
||||
return parsed
|
||||
|
||||
def run_query(self, user, search_query, **kwargs):
|
||||
"""
|
||||
Low level helper to run an ES query.
|
||||
Accept a user to pass it to the filter, so we can
|
||||
avoid filtering for superusers.
|
||||
Accept fields and size, for the fields we want to match and the
|
||||
number of results to return.
|
||||
"""
|
||||
if self.client is None:
|
||||
self.initialise()
|
||||
index = kwargs.get("index")
|
||||
try:
|
||||
response = self.client.search(body=search_query, index=index)
|
||||
except RequestError as err:
|
||||
print("Elasticsearch error", err)
|
||||
return err
|
||||
except NotFoundError as err:
|
||||
print("Elasticsearch error", err)
|
||||
return err
|
||||
return response
|
||||
|
||||
async def async_run_query(self, user, search_query, **kwargs):
|
||||
"""
|
||||
Low level helper to run an ES query.
|
||||
Accept a user to pass it to the filter, so we can
|
||||
avoid filtering for superusers.
|
||||
Accept fields and size, for the fields we want to match and the
|
||||
number of results to return.
|
||||
"""
|
||||
if self.async_client is None:
|
||||
await self.async_initialise()
|
||||
index = kwargs.get("index")
|
||||
try:
|
||||
response = await self.async_client.search(body=search_query, index=index)
|
||||
except RequestError as err:
|
||||
print("Elasticsearch error", err)
|
||||
return err
|
||||
except NotFoundError as err:
|
||||
print("Elasticsearch error", err)
|
||||
return err
|
||||
return response
|
||||
|
||||
async def async_store_matches(self, matches):
|
||||
"""
|
||||
Store a list of matches in Elasticsearch.
|
||||
:param index: The index to store the matches in.
|
||||
:param matches: A list of matches to store.
|
||||
"""
|
||||
if self.async_client is None:
|
||||
await self.async_initialise()
|
||||
for match in matches:
|
||||
result = await self.async_client.index(
|
||||
index=settings.INDEX_RULE_STORAGE, body=match
|
||||
)
|
||||
if not result["result"] == "created":
|
||||
self.log.error(f"Indexing failed: {result}")
|
||||
self.log.debug(f"Indexed {len(matches)} messages in ES")
|
||||
|
||||
def store_matches(self, matches):
|
||||
"""
|
||||
Store a list of matches in Elasticsearch.
|
||||
:param index: The index to store the matches in.
|
||||
:param matches: A list of matches to store.
|
||||
"""
|
||||
if self.client is None:
|
||||
self.initialise()
|
||||
for match in matches:
|
||||
result = self.client.index(index=settings.INDEX_RULE_STORAGE, body=match)
|
||||
if not result["result"] == "created":
|
||||
self.log.error(f"Indexing failed: {result}")
|
||||
self.log.debug(f"Indexed {len(matches)} messages in ES")
|
||||
|
||||
def prepare_schedule_query(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query with reduced functionality.
|
||||
"""
|
||||
data = rule_object.parsed
|
||||
|
||||
if "tags" in data:
|
||||
tags = data["tags"]
|
||||
else:
|
||||
tags = []
|
||||
|
||||
if "query" in data:
|
||||
query = data["query"][0]
|
||||
data["query"] = query
|
||||
|
||||
add_bool = []
|
||||
add_top = []
|
||||
if "source" in data:
|
||||
total_count = len(data["source"])
|
||||
total_sources = len(settings.MAIN_SOURCES) + len(
|
||||
settings.SOURCES_RESTRICTED
|
||||
)
|
||||
if total_count != total_sources:
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for source_iter in data["source"]:
|
||||
add_top_tmp["bool"]["should"].append(
|
||||
{"match_phrase": {"src": source_iter}}
|
||||
)
|
||||
add_top.append(add_top_tmp)
|
||||
if "tokens" in data:
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for token in data["tokens"]:
|
||||
add_top_tmp["bool"]["should"].append(
|
||||
{"match_phrase": {"tokens": token}}
|
||||
)
|
||||
add_top.append(add_top_tmp)
|
||||
for field, values in data.items():
|
||||
if field not in ["source", "index", "tags", "query", "sentiment", "tokens"]:
|
||||
for value in values:
|
||||
add_top.append({"match": {field: value}})
|
||||
# Bypass the check for query and tags membership since we can search by msg, etc
|
||||
search_query = self.parse_query(
|
||||
data, tags, None, False, add_bool, bypass_check=True
|
||||
)
|
||||
if rule_object.window is not None:
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
"gte": f"now-{rule_object.window}",
|
||||
"lte": "now",
|
||||
}
|
||||
}
|
||||
}
|
||||
add_top.append(range_query)
|
||||
self.add_bool(search_query, add_bool)
|
||||
self.add_top(search_query, add_top)
|
||||
# if "sentiment" in data:
|
||||
search_query["aggs"] = {
|
||||
"avg_sentiment": {
|
||||
"avg": {"field": "sentiment"},
|
||||
}
|
||||
}
|
||||
|
||||
return search_query
|
||||
|
||||
def schedule_check_aggregations(self, rule_object, result_map):
|
||||
"""
|
||||
Check the results of a scheduled query for aggregations.
|
||||
"""
|
||||
if rule_object.aggs is None:
|
||||
return result_map
|
||||
for index, (meta, result) in result_map.items():
|
||||
# Default to true, if no aggs are found, we still want to match
|
||||
match = True
|
||||
for agg_name, (operator, number) in rule_object.aggs.items():
|
||||
if agg_name in meta["aggs"]:
|
||||
agg_value = meta["aggs"][agg_name]["value"]
|
||||
|
||||
# TODO: simplify this, match is default to True
|
||||
if operator == ">":
|
||||
if agg_value > number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
elif operator == "<":
|
||||
if agg_value < number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
elif operator == "=":
|
||||
if agg_value == number:
|
||||
match = True
|
||||
else:
|
||||
match = False
|
||||
else:
|
||||
match = False
|
||||
else:
|
||||
# No aggregation found, but it is required
|
||||
match = False
|
||||
result_map[index][0]["aggs"][agg_name]["match"] = match
|
||||
|
||||
return result_map
|
||||
|
||||
def schedule_query_results_test_sync(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query test with reduced functionality.
|
||||
Sync version for running from Django forms.
|
||||
Does not return results.
|
||||
"""
|
||||
data = rule_object.parsed
|
||||
|
||||
search_query = self.prepare_schedule_query(rule_object)
|
||||
for index in data["index"]:
|
||||
if "message" in search_query:
|
||||
self.log.error(f"Error parsing test query: {search_query['message']}")
|
||||
continue
|
||||
response = self.run_query(
|
||||
rule_object.user,
|
||||
search_query,
|
||||
index=index,
|
||||
)
|
||||
self.log.debug(f"Running scheduled test query on {index}: {search_query}")
|
||||
# self.log.debug(f"Response from scheduled query: {response}")
|
||||
if isinstance(response, Exception):
|
||||
error = response.info["error"]["root_cause"][0]["reason"]
|
||||
self.log.error(f"Error running test scheduled search: {error}")
|
||||
raise QueryError(error)
|
||||
|
||||
async def schedule_query_results(self, rule_object):
|
||||
"""
|
||||
Helper to run a scheduled query with reduced functionality and async.
|
||||
"""
|
||||
result_map = {}
|
||||
data = rule_object.parsed
|
||||
|
||||
search_query = self.prepare_schedule_query(rule_object)
|
||||
|
||||
for index in data["index"]:
|
||||
if "message" in search_query:
|
||||
self.log.error(f"Error parsing query: {search_query['message']}")
|
||||
continue
|
||||
response = await self.async_run_query(
|
||||
rule_object.user,
|
||||
search_query,
|
||||
index=index,
|
||||
)
|
||||
self.log.debug(f"Running scheduled query on {index}: {search_query}")
|
||||
# self.log.debug(f"Response from scheduled query: {response}")
|
||||
if isinstance(response, Exception):
|
||||
error = response.info["error"]["root_cause"][0]["reason"]
|
||||
self.log.error(f"Error running scheduled search: {error}")
|
||||
raise QueryError(error)
|
||||
if len(response["hits"]["hits"]) == 0:
|
||||
# No results, skip
|
||||
result_map[index] = ({}, [])
|
||||
continue
|
||||
meta, response = self.parse(response, meta=True)
|
||||
# print("Parsed response", response)
|
||||
if "message" in response:
|
||||
self.log.error(f"Error running scheduled search: {response['message']}")
|
||||
continue
|
||||
result_map[index] = (meta, response)
|
||||
|
||||
# Average aggregation check
|
||||
# Could probably do this in elasticsearch
|
||||
result_map = self.schedule_check_aggregations(rule_object, result_map)
|
||||
|
||||
return result_map
|
||||
|
||||
def query_results(
|
||||
self,
|
||||
request,
|
||||
query_params,
|
||||
size=None,
|
||||
annotate=True,
|
||||
custom_query=False,
|
||||
reverse=False,
|
||||
dedup=False,
|
||||
dedup_fields=None,
|
||||
tags=None,
|
||||
):
|
||||
add_bool = []
|
||||
add_top = []
|
||||
add_top_negative = []
|
||||
|
||||
add_defaults(query_params)
|
||||
|
||||
# Now, run the helpers for SIQTSRSS/ADR
|
||||
# S - Size
|
||||
# I - Index
|
||||
# Q - Query
|
||||
# T - Tags
|
||||
# S - Source
|
||||
# R - Ranges
|
||||
# S - Sort
|
||||
# S - Sentiment
|
||||
# A - Annotate
|
||||
# D - Dedup
|
||||
# R - Reverse
|
||||
|
||||
# S - Size
|
||||
if request.user.is_anonymous:
|
||||
sizes = settings.MAIN_SIZES_ANON
|
||||
else:
|
||||
sizes = settings.MAIN_SIZES
|
||||
if not size:
|
||||
size = parse_size(query_params, sizes)
|
||||
if isinstance(size, dict):
|
||||
return size
|
||||
|
||||
rule_object = parse_rule(request.user, query_params)
|
||||
if isinstance(rule_object, dict):
|
||||
return rule_object
|
||||
|
||||
if rule_object is not None:
|
||||
index = settings.INDEX_RULE_STORAGE
|
||||
add_bool.append({"rule_id": str(rule_object.id)})
|
||||
else:
|
||||
# I - Index
|
||||
index = parse_index(request.user, query_params)
|
||||
if isinstance(index, dict):
|
||||
return index
|
||||
|
||||
# Q/T - Query/Tags
|
||||
search_query = self.parse_query(
|
||||
query_params, tags, size, custom_query, add_bool
|
||||
)
|
||||
# Query should be a dict, so check if it contains message here
|
||||
if "message" in search_query:
|
||||
return search_query
|
||||
|
||||
# S - Sources
|
||||
sources = parse_source(request.user, query_params)
|
||||
if isinstance(sources, dict):
|
||||
return sources
|
||||
total_count = len(sources)
|
||||
# Total is -1 due to the "all" source
|
||||
total_sources = (
|
||||
len(settings.MAIN_SOURCES) - 1 + len(settings.SOURCES_RESTRICTED)
|
||||
)
|
||||
|
||||
# If the sources the user has access to are equal to all
|
||||
# possible sources, then we don't need to add the source
|
||||
# filter to the query.
|
||||
if total_count != total_sources:
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append(
|
||||
{"match_phrase": {"src": source_iter}}
|
||||
)
|
||||
if query_params["source"] != "all":
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
# R - Ranges
|
||||
# date_query = False
|
||||
from_ts, to_ts = parse_date_time(query_params)
|
||||
if from_ts:
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
"gt": from_ts,
|
||||
"lt": to_ts,
|
||||
}
|
||||
}
|
||||
}
|
||||
add_top.append(range_query)
|
||||
|
||||
# S - Sort
|
||||
sort = parse_sort(query_params)
|
||||
if isinstance(sort, dict):
|
||||
return sort
|
||||
|
||||
if rule_object is not None:
|
||||
field = "match_ts"
|
||||
else:
|
||||
field = "ts"
|
||||
if sort:
|
||||
# For Druid compatibility
|
||||
sort_map = {"ascending": "asc", "descending": "desc"}
|
||||
sorting = [
|
||||
{
|
||||
field: {
|
||||
"order": sort_map[sort],
|
||||
}
|
||||
}
|
||||
]
|
||||
search_query["sort"] = sorting
|
||||
|
||||
# S - Sentiment
|
||||
sentiment_r = parse_sentiment(query_params)
|
||||
if isinstance(sentiment_r, dict):
|
||||
return sentiment_r
|
||||
if sentiment_r:
|
||||
if rule_object is not None:
|
||||
sentiment_index = "meta.aggs.avg_sentiment.value"
|
||||
else:
|
||||
sentiment_index = "sentiment"
|
||||
sentiment_method, sentiment = sentiment_r
|
||||
range_query_compare = {"range": {sentiment_index: {}}}
|
||||
range_query_precise = {
|
||||
"match": {
|
||||
sentiment_index: None,
|
||||
}
|
||||
}
|
||||
if sentiment_method == "below":
|
||||
range_query_compare["range"][sentiment_index]["lt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "above":
|
||||
range_query_compare["range"][sentiment_index]["gt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "exact":
|
||||
range_query_precise["match"][sentiment_index] = sentiment
|
||||
add_top.append(range_query_precise)
|
||||
elif sentiment_method == "nonzero":
|
||||
range_query_precise["match"][sentiment_index] = 0
|
||||
add_top_negative.append(range_query_precise)
|
||||
|
||||
# Add in the additional information we already populated
|
||||
self.add_bool(search_query, add_bool)
|
||||
self.add_top(search_query, add_top)
|
||||
self.add_top(search_query, add_top_negative, negative=True)
|
||||
|
||||
response = self.query(
|
||||
request.user,
|
||||
search_query,
|
||||
index=index,
|
||||
)
|
||||
if "message" in response:
|
||||
return response
|
||||
|
||||
# A/D/R - Annotate/Dedup/Reverse
|
||||
response["object_list"] = self.process_results(
|
||||
response["object_list"],
|
||||
annotate=annotate,
|
||||
dedup=dedup,
|
||||
dedup_fields=dedup_fields,
|
||||
reverse=reverse,
|
||||
)
|
||||
|
||||
context = response
|
||||
return context
|
||||
|
||||
def query_single_result(self, request, query_params):
|
||||
context = self.query_results(request, query_params, size=100)
|
||||
|
||||
if not context:
|
||||
return {"message": "Failed to run query", "message_class": "danger"}
|
||||
if "message" in context:
|
||||
return context
|
||||
dedup_set = {item["nick"] for item in context["object_list"]}
|
||||
if dedup_set:
|
||||
context["item"] = context["object_list"][0]
|
||||
|
||||
return context
|
||||
|
||||
def add_bool(self, search_query, add_bool):
|
||||
"""
|
||||
Add the specified boolean matches to search query.
|
||||
"""
|
||||
if not add_bool:
|
||||
return
|
||||
for item in add_bool:
|
||||
search_query["query"]["bool"]["must"].append({"match_phrase": item})
|
||||
|
||||
def add_top(self, search_query, add_top, negative=False):
|
||||
"""
|
||||
Merge add_top with the base of the search_query.
|
||||
"""
|
||||
if not add_top:
|
||||
return
|
||||
if negative:
|
||||
for item in add_top:
|
||||
if "must_not" in search_query["query"]["bool"]:
|
||||
search_query["query"]["bool"]["must_not"].append(item)
|
||||
else:
|
||||
search_query["query"]["bool"]["must_not"] = [item]
|
||||
else:
|
||||
for item in add_top:
|
||||
if "query" not in search_query:
|
||||
search_query["query"] = {"bool": {"must": []}}
|
||||
search_query["query"]["bool"]["must"].append(item)
|
||||
@ -1,485 +0,0 @@
|
||||
# from copy import deepcopy
|
||||
# from datetime import datetime, timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from opensearchpy import OpenSearch
|
||||
from opensearchpy.exceptions import NotFoundError, RequestError
|
||||
|
||||
from core.db import StorageBackend
|
||||
|
||||
# from json import dumps
|
||||
# pp = lambda x: print(dumps(x, indent=2))
|
||||
from core.db.processing import annotate_results, parse_results
|
||||
from core.views.helpers import dedup_list
|
||||
|
||||
|
||||
class OpensearchBackend(StorageBackend):
|
||||
def __init__(self):
|
||||
super().__init__("Opensearch")
|
||||
|
||||
def initialise(self, **kwargs):
|
||||
"""
|
||||
Inititialise the OpenSearch API endpoint.
|
||||
"""
|
||||
auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
|
||||
client = OpenSearch(
|
||||
# fmt: off
|
||||
hosts=[{"host": settings.OPENSEARCH_URL,
|
||||
"port": settings.OPENSEARCH_PORT}],
|
||||
http_compress=False, # enables gzip compression for request bodies
|
||||
http_auth=auth,
|
||||
# client_cert = client_cert_path,
|
||||
# client_key = client_key_path,
|
||||
use_ssl=settings.OPENSEARCH_TLS,
|
||||
verify_certs=False,
|
||||
ssl_assert_hostname=False,
|
||||
ssl_show_warn=False,
|
||||
# a_certs=ca_certs_path,
|
||||
)
|
||||
self.client = client
|
||||
|
||||
def construct_query(self, query, size, use_query_string=True, tokens=False):
|
||||
"""
|
||||
Accept some query parameters and construct an OpenSearch query.
|
||||
"""
|
||||
if not size:
|
||||
size = 5
|
||||
query_base = {
|
||||
"size": size,
|
||||
"query": {"bool": {"must": []}},
|
||||
}
|
||||
query_string = {
|
||||
"query_string": {
|
||||
"query": query,
|
||||
# "fields": fields,
|
||||
# "default_field": "msg",
|
||||
# "type": "best_fields",
|
||||
"fuzziness": "AUTO",
|
||||
"fuzzy_transpositions": True,
|
||||
"fuzzy_max_expansions": 50,
|
||||
"fuzzy_prefix_length": 0,
|
||||
# "minimum_should_match": 1,
|
||||
"default_operator": "or",
|
||||
"analyzer": "standard",
|
||||
"lenient": True,
|
||||
"boost": 1,
|
||||
"allow_leading_wildcard": True,
|
||||
# "enable_position_increments": False,
|
||||
"phrase_slop": 3,
|
||||
# "max_determinized_states": 10000,
|
||||
"quote_field_suffix": "",
|
||||
"quote_analyzer": "standard",
|
||||
"analyze_wildcard": False,
|
||||
"auto_generate_synonyms_phrase_query": True,
|
||||
}
|
||||
}
|
||||
query_tokens = {
|
||||
"simple_query_string": {
|
||||
# "tokens": query,
|
||||
"query": query,
|
||||
"fields": ["tokens"],
|
||||
"flags": "ALL",
|
||||
"fuzzy_transpositions": True,
|
||||
"fuzzy_max_expansions": 50,
|
||||
"fuzzy_prefix_length": 0,
|
||||
"default_operator": "and",
|
||||
"analyzer": "standard",
|
||||
"lenient": True,
|
||||
"boost": 1,
|
||||
"quote_field_suffix": "",
|
||||
"analyze_wildcard": False,
|
||||
"auto_generate_synonyms_phrase_query": False,
|
||||
}
|
||||
}
|
||||
if tokens:
|
||||
query_base["query"]["bool"]["must"].append(query_tokens)
|
||||
# query["query"]["bool"]["must"].append(query_string)
|
||||
# query["query"]["bool"]["must"][0]["query_string"]["fields"] = ["tokens"]
|
||||
elif use_query_string:
|
||||
query_base["query"]["bool"]["must"].append(query_string)
|
||||
return query_base
|
||||
|
||||
def run_query(self, client, user, query, custom_query=False, index=None, size=None):
|
||||
"""
|
||||
Low level helper to run an ES query.
|
||||
Accept a user to pass it to the filter, so we can
|
||||
avoid filtering for superusers.
|
||||
Accept fields and size, for the fields we want to match and the
|
||||
number of results to return.
|
||||
"""
|
||||
if not index:
|
||||
index = settings.INDEX_MAIN
|
||||
if custom_query:
|
||||
search_query = query
|
||||
else:
|
||||
search_query = self.construct_query(query, size)
|
||||
try:
|
||||
response = client.search(body=search_query, index=index)
|
||||
except RequestError as err:
|
||||
print("OpenSearch error", err)
|
||||
return err
|
||||
except NotFoundError as err:
|
||||
print("OpenSearch error", err)
|
||||
return err
|
||||
return response
|
||||
|
||||
def query_results(
|
||||
self,
|
||||
request,
|
||||
query_params,
|
||||
size=None,
|
||||
annotate=True,
|
||||
custom_query=False,
|
||||
reverse=False,
|
||||
dedup=False,
|
||||
dedup_fields=None,
|
||||
lookup_hashes=True,
|
||||
tags=None,
|
||||
):
|
||||
"""
|
||||
API helper to alter the OpenSearch return format into something
|
||||
a bit better to parse.
|
||||
Accept a HTTP request object. Run the query, and annotate the
|
||||
results with the other data we have.
|
||||
"""
|
||||
# is_anonymous = isinstance(request.user, AnonymousUser)
|
||||
query = None
|
||||
message = None
|
||||
message_class = None
|
||||
add_bool = []
|
||||
add_top = []
|
||||
add_top_negative = []
|
||||
sort = None
|
||||
query_created = False
|
||||
|
||||
# Lookup the hash values but don't disclose them to the user
|
||||
# denied = []
|
||||
# if lookup_hashes:
|
||||
# if settings.HASHING:
|
||||
# query_params = deepcopy(query_params)
|
||||
# denied_q = hash_lookup(request.user, query_params)
|
||||
# denied.extend(denied_q)
|
||||
# if tags:
|
||||
# denied_t = hash_lookup(request.user, tags, query_params)
|
||||
# denied.extend(denied_t)
|
||||
|
||||
# message = "Permission denied: "
|
||||
# for x in denied:
|
||||
# if isinstance(x, SearchDenied):
|
||||
# message += f"Search({x.key}: {x.value}) "
|
||||
# elif isinstance(x, LookupDenied):
|
||||
# message += f"Lookup({x.key}: {x.value}) "
|
||||
# if denied:
|
||||
# # message = [f"{i}" for i in message]
|
||||
# # message = "\n".join(message)
|
||||
# message_class = "danger"
|
||||
# return {"message": message, "class": message_class}
|
||||
|
||||
if request.user.is_anonymous:
|
||||
sizes = settings.MAIN_SIZES_ANON
|
||||
else:
|
||||
sizes = settings.MAIN_SIZES
|
||||
if not size:
|
||||
if "size" in query_params:
|
||||
size = query_params["size"]
|
||||
if size not in sizes:
|
||||
message = "Size is not permitted"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
else:
|
||||
size = 20
|
||||
source = None
|
||||
if "source" in query_params:
|
||||
source = query_params["source"]
|
||||
|
||||
if source in settings.SOURCES_RESTRICTED:
|
||||
if not request.user.has_perm("core.restricted_sources"):
|
||||
message = "Access denied"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
elif source not in settings.MAIN_SOURCES:
|
||||
message = "Invalid source"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
if source == "all":
|
||||
source = None # the next block will populate it
|
||||
|
||||
if source:
|
||||
sources = [source]
|
||||
else:
|
||||
sources = settings.MAIN_SOURCES
|
||||
if request.user.has_perm("core.restricted_sources"):
|
||||
for source_iter in settings.SOURCES_RESTRICTED:
|
||||
sources.append(source_iter)
|
||||
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
# date_query = False
|
||||
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
|
||||
query_params.keys()
|
||||
):
|
||||
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
|
||||
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
"gt": from_ts,
|
||||
"lt": to_ts,
|
||||
}
|
||||
}
|
||||
}
|
||||
add_top.append(range_query)
|
||||
|
||||
# if date_query:
|
||||
# if settings.DELAY_RESULTS:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if request.user.has_perm("core.bypass_delay"):
|
||||
# add_top.append(range_query)
|
||||
# else:
|
||||
# delay_as_ts = datetime.now() - timedelta(
|
||||
# days=settings.DELAY_DURATION
|
||||
# )
|
||||
# lt_as_ts = datetime.strptime(
|
||||
# range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
|
||||
# )
|
||||
# if lt_as_ts > delay_as_ts:
|
||||
# range_query["range"]["ts"][
|
||||
# "lt"
|
||||
# ] = f"now-{settings.DELAY_DURATION}d"
|
||||
# add_top.append(range_query)
|
||||
# else:
|
||||
# add_top.append(range_query)
|
||||
# else:
|
||||
# if settings.DELAY_RESULTS:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if not request.user.has_perm("core.bypass_delay"):
|
||||
# range_query = {
|
||||
# "range": {
|
||||
# "ts": {
|
||||
# # "gt": ,
|
||||
# "lt": f"now-{settings.DELAY_DURATION}d",
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# add_top.append(range_query)
|
||||
|
||||
if "sorting" in query_params:
|
||||
sorting = query_params["sorting"]
|
||||
if sorting not in ("asc", "desc", "none"):
|
||||
message = "Invalid sort"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if sorting in ("asc", "desc"):
|
||||
sort = [
|
||||
{
|
||||
"ts": {
|
||||
"order": sorting,
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
if "check_sentiment" in query_params:
|
||||
if "sentiment_method" not in query_params:
|
||||
message = "No sentiment method"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if "sentiment" in query_params:
|
||||
sentiment = query_params["sentiment"]
|
||||
try:
|
||||
sentiment = float(sentiment)
|
||||
except ValueError:
|
||||
message = "Sentiment is not a float"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
sentiment_method = query_params["sentiment_method"]
|
||||
range_query_compare = {"range": {"sentiment": {}}}
|
||||
range_query_precise = {
|
||||
"match": {
|
||||
"sentiment": None,
|
||||
}
|
||||
}
|
||||
if sentiment_method == "below":
|
||||
range_query_compare["range"]["sentiment"]["lt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "above":
|
||||
range_query_compare["range"]["sentiment"]["gt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "exact":
|
||||
range_query_precise["match"]["sentiment"] = sentiment
|
||||
add_top.append(range_query_precise)
|
||||
elif sentiment_method == "nonzero":
|
||||
range_query_precise["match"]["sentiment"] = 0
|
||||
add_top_negative.append(range_query_precise)
|
||||
|
||||
# Only one of query or query_full can be active at once
|
||||
# We prefer query because it's simpler
|
||||
if "query" in query_params:
|
||||
query = query_params["query"]
|
||||
search_query = self.construct_query(query, size, tokens=True)
|
||||
query_created = True
|
||||
elif "query_full" in query_params:
|
||||
query_full = query_params["query_full"]
|
||||
# if request.user.has_perm("core.query_search"):
|
||||
search_query = self.construct_query(query_full, size)
|
||||
query_created = True
|
||||
# else:
|
||||
# message = "You cannot search by query string"
|
||||
# message_class = "danger"
|
||||
# return {"message": message, "class": message_class}
|
||||
else:
|
||||
if custom_query:
|
||||
search_query = custom_query
|
||||
|
||||
if tags:
|
||||
# Get a blank search query
|
||||
if not query_created:
|
||||
search_query = self.construct_query(None, size, use_query_string=False)
|
||||
query_created = True
|
||||
for tagname, tagvalue in tags.items():
|
||||
add_bool.append({tagname: tagvalue})
|
||||
|
||||
required_any = ["query_full", "query", "tags"]
|
||||
if not any([field in query_params.keys() for field in required_any]):
|
||||
if not custom_query:
|
||||
message = "Empty query!"
|
||||
message_class = "warning"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
if add_bool:
|
||||
# if "bool" not in search_query["query"]:
|
||||
# search_query["query"]["bool"] = {}
|
||||
# if "must" not in search_query["query"]["bool"]:
|
||||
# search_query["query"]["bool"] = {"must": []}
|
||||
|
||||
for item in add_bool:
|
||||
search_query["query"]["bool"]["must"].append({"match_phrase": item})
|
||||
if add_top:
|
||||
for item in add_top:
|
||||
search_query["query"]["bool"]["must"].append(item)
|
||||
if add_top_negative:
|
||||
for item in add_top_negative:
|
||||
if "must_not" in search_query["query"]["bool"]:
|
||||
search_query["query"]["bool"]["must_not"].append(item)
|
||||
else:
|
||||
search_query["query"]["bool"]["must_not"] = [item]
|
||||
if sort:
|
||||
search_query["sort"] = sort
|
||||
|
||||
if "index" in query_params:
|
||||
index = query_params["index"]
|
||||
if index == "main":
|
||||
index = settings.INDEX_MAIN
|
||||
else:
|
||||
if not request.user.has_perm(f"core.index_{index}"):
|
||||
message = "Not permitted to search by this index"
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
if index == "meta":
|
||||
index = settings.INDEX_META
|
||||
elif index == "internal":
|
||||
index = settings.INDEX_INT
|
||||
else:
|
||||
message = "Index is not valid."
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
|
||||
else:
|
||||
index = settings.INDEX_MAIN
|
||||
|
||||
results = self.query(
|
||||
request.user, # passed through run_main_query to filter_blacklisted
|
||||
search_query,
|
||||
custom_query=True,
|
||||
index=index,
|
||||
size=size,
|
||||
)
|
||||
if not results:
|
||||
return False
|
||||
if isinstance(results, Exception):
|
||||
message = f"Error: {results.info['error']['root_cause'][0]['type']}"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if len(results["hits"]["hits"]) == 0:
|
||||
message = "No results."
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
results_parsed = parse_results(results)
|
||||
|
||||
if annotate:
|
||||
annotate_results(results_parsed)
|
||||
if "dedup" in query_params:
|
||||
if query_params["dedup"] == "on":
|
||||
dedup = True
|
||||
else:
|
||||
dedup = False
|
||||
else:
|
||||
dedup = False
|
||||
|
||||
if reverse:
|
||||
results_parsed = results_parsed[::-1]
|
||||
|
||||
if dedup:
|
||||
if not dedup_fields:
|
||||
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
|
||||
results_parsed = dedup_list(results_parsed, dedup_fields)
|
||||
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if settings.ENCRYPTION:
|
||||
# encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
|
||||
|
||||
# if settings.HASHING:
|
||||
# hash_list(request.user, results_parsed)
|
||||
|
||||
# if settings.OBFUSCATION:
|
||||
# obfuscate_list(request.user, results_parsed)
|
||||
|
||||
# if settings.RANDOMISATION:
|
||||
# randomise_list(request.user, results_parsed)
|
||||
|
||||
# process_list(results)
|
||||
|
||||
# IMPORTANT! - DO NOT PASS query_params to the user!
|
||||
context = {
|
||||
"object_list": results_parsed,
|
||||
"card": results["hits"]["total"]["value"],
|
||||
"took": results["took"],
|
||||
}
|
||||
if "redacted" in results:
|
||||
context["redacted"] = results["redacted"]
|
||||
if "exemption" in results:
|
||||
context["exemption"] = results["exemption"]
|
||||
if query:
|
||||
context["query"] = query
|
||||
# if settings.DELAY_RESULTS:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if not request.user.has_perm("core.bypass_delay"):
|
||||
# context["delay"] = settings.DELAY_DURATION
|
||||
# if settings.RANDOMISATION:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if not request.user.has_perm("core.bypass_randomisation"):
|
||||
# context["randomised"] = True
|
||||
return context
|
||||
|
||||
def query_single_result(self, request, query_params):
|
||||
context = self.query_results(request, query_params, size=100)
|
||||
|
||||
if not context:
|
||||
return {"message": "Failed to run query", "message_class": "danger"}
|
||||
if "message" in context:
|
||||
return context
|
||||
dedup_set = {item["nick"] for item in context["object_list"]}
|
||||
if dedup_set:
|
||||
context["item"] = context["object_list"][0]
|
||||
|
||||
return context
|
||||
@ -0,0 +1,107 @@
|
||||
import requests
|
||||
|
||||
from core.util import logs
|
||||
|
||||
NTFY_URL = "https://ntfy.sh"
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
# Actual function to send a message to a topic
|
||||
def ntfy_sendmsg(**kwargs):
|
||||
"""
|
||||
Send a message to a topic using NTFY.
|
||||
kwargs:
|
||||
msg: Message to send, must be specified
|
||||
notification_settings: Notification settings, must be specified
|
||||
url: URL to NTFY server, can be None to use default
|
||||
topic: Topic to send message to, must be specified
|
||||
priority: Priority of message, optional
|
||||
title: Title of message, optional
|
||||
tags: Tags to add to message, optional
|
||||
"""
|
||||
msg = kwargs.get("msg", None)
|
||||
notification_settings = kwargs.get("notification_settings")
|
||||
|
||||
title = kwargs.get("title", None)
|
||||
priority = notification_settings.get("priority", None)
|
||||
tags = kwargs.get("tags", None)
|
||||
url = notification_settings.get("url") or NTFY_URL
|
||||
topic = notification_settings.get("topic", None)
|
||||
|
||||
headers = {"Title": "Fisk"}
|
||||
if title:
|
||||
headers["Title"] = title
|
||||
if priority:
|
||||
headers["Priority"] = priority
|
||||
if tags:
|
||||
headers["Tags"] = tags
|
||||
try:
|
||||
requests.post(
|
||||
f"{url}/{topic}",
|
||||
data=msg,
|
||||
headers=headers,
|
||||
)
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
log.error(f"Error sending notification: {e}")
|
||||
|
||||
|
||||
def webhook_sendmsg(**kwargs):
|
||||
"""
|
||||
Send a message to a webhook.
|
||||
kwargs:
|
||||
msg: Message to send, must be specified
|
||||
notification_settings: Notification settings, must be specified
|
||||
url: URL to webhook, must be specified"""
|
||||
msg = kwargs.get("msg", None)
|
||||
notification_settings = kwargs.get("notification_settings")
|
||||
url = notification_settings.get("url")
|
||||
headers = {"Content-type": "application/json"}
|
||||
try:
|
||||
requests.post(
|
||||
f"{url}",
|
||||
headers=headers,
|
||||
data=msg,
|
||||
)
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
log.error(f"Error sending webhook: {e}")
|
||||
|
||||
|
||||
# Sendmsg helper to send a message to a user's notification settings
|
||||
def sendmsg(**kwargs):
|
||||
"""
|
||||
Send a message to a user's notification settings.
|
||||
Fetches the user's default notification settings if not specified.
|
||||
kwargs:
|
||||
user: User to send message to, must be specified
|
||||
notification_settings: Notification settings, optional
|
||||
service: Notification service to use
|
||||
|
||||
kwargs for both services:
|
||||
msg: Message to send, must be specified
|
||||
notification_settings: Notification settings, must be specified
|
||||
url: URL to NTFY server, can be None to use default
|
||||
|
||||
extra kwargs for ntfy:
|
||||
title: Title of message, optional
|
||||
tags: Tags to add to message, optional
|
||||
notification_settings: Notification settings, must be specified
|
||||
topic: Topic to send message to, must be specified
|
||||
priority: Priority of message, optional
|
||||
"""
|
||||
user = kwargs.get("user", None)
|
||||
notification_settings = kwargs.get(
|
||||
"notification_settings", user.get_notification_settings().__dict__
|
||||
)
|
||||
if not notification_settings:
|
||||
return
|
||||
|
||||
service = notification_settings.get("service")
|
||||
if service == "none":
|
||||
# Don't send anything
|
||||
return
|
||||
|
||||
if service == "ntfy":
|
||||
ntfy_sendmsg(**kwargs)
|
||||
elif service == "webhook":
|
||||
webhook_sendmsg(**kwargs)
|
||||
@ -0,0 +1,186 @@
|
||||
from datetime import datetime
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError
|
||||
|
||||
from core.models import NotificationRule
|
||||
|
||||
|
||||
class QueryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def parse_rule(user, query_params):
|
||||
"""
|
||||
Parse a rule query.
|
||||
"""
|
||||
if "rule" in query_params:
|
||||
try:
|
||||
rule_object = NotificationRule.objects.filter(id=query_params["rule"])
|
||||
except ValidationError:
|
||||
message = "Rule is not a valid UUID"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if not rule_object.exists():
|
||||
message = "Rule does not exist"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
rule_object = rule_object.first()
|
||||
if not rule_object.user == user:
|
||||
message = "Rule does not belong to you"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
return rule_object
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def parse_size(query_params, sizes):
|
||||
if "size" in query_params:
|
||||
size = query_params["size"]
|
||||
if size not in sizes:
|
||||
message = "Size is not permitted"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
size = int(size)
|
||||
else:
|
||||
size = 15
|
||||
|
||||
return size
|
||||
|
||||
|
||||
def parse_index(user, query_params, raise_error=False):
|
||||
if "index" in query_params:
|
||||
index = query_params["index"]
|
||||
if index == "main":
|
||||
index = settings.INDEX_MAIN
|
||||
else:
|
||||
if not user.has_perm(f"core.index_{index}"):
|
||||
message = f"Not permitted to search by this index: {index}"
|
||||
if raise_error:
|
||||
raise QueryError(message)
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
if index == "meta":
|
||||
index = settings.INDEX_META
|
||||
elif index == "internal":
|
||||
index = settings.INDEX_INT
|
||||
elif index == "restricted":
|
||||
if not user.has_perm("core.restricted_sources"):
|
||||
message = f"Not permitted to search by this index: {index}"
|
||||
if raise_error:
|
||||
raise QueryError(message)
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
index = settings.INDEX_RESTRICTED
|
||||
else:
|
||||
message = f"Index is not valid: {index}"
|
||||
if raise_error:
|
||||
raise QueryError(message)
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
else:
|
||||
index = settings.INDEX_MAIN
|
||||
return index
|
||||
|
||||
|
||||
def parse_source(user, query_params, raise_error=False):
|
||||
source = None
|
||||
if "source" in query_params:
|
||||
source = query_params["source"]
|
||||
|
||||
# Validate permissions for restricted sources
|
||||
if source in settings.SOURCES_RESTRICTED:
|
||||
if not user.has_perm("core.restricted_sources"):
|
||||
message = f"Access denied: {source}"
|
||||
if raise_error:
|
||||
raise QueryError(message)
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
# Check validity of source
|
||||
elif source not in settings.MAIN_SOURCES:
|
||||
message = f"Invalid source: {source}"
|
||||
if raise_error:
|
||||
raise QueryError(message)
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
if source == "all":
|
||||
source = None # the next block will populate it
|
||||
|
||||
if source:
|
||||
sources = [source]
|
||||
else:
|
||||
# Here we need to populate what "all" means for the user.
|
||||
# They may only have access to a subset of the sources.
|
||||
# We build a custom source list with ones they have access
|
||||
# to, and then remove "all" from the list.
|
||||
sources = list(settings.MAIN_SOURCES)
|
||||
if user.has_perm("core.restricted_sources"):
|
||||
# If the user can use restricted sources, add them in.
|
||||
for source_iter in settings.SOURCES_RESTRICTED:
|
||||
sources.append(source_iter)
|
||||
|
||||
# Get rid of "all", it's just a meta-source
|
||||
if "all" in sources:
|
||||
sources.remove("all")
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
def parse_sort(query_params):
|
||||
sort = None
|
||||
if "sorting" in query_params:
|
||||
sorting = query_params["sorting"]
|
||||
if sorting not in ("asc", "desc", "none"):
|
||||
message = "Invalid sort"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if sorting == "asc":
|
||||
sort = "ascending"
|
||||
elif sorting == "desc":
|
||||
sort = "descending"
|
||||
return sort
|
||||
|
||||
|
||||
def parse_date_time(query_params):
|
||||
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
|
||||
query_params.keys()
|
||||
):
|
||||
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
|
||||
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
|
||||
from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
|
||||
to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
|
||||
|
||||
return (from_ts, to_ts)
|
||||
return (None, None)
|
||||
|
||||
|
||||
def parse_sentiment(query_params):
|
||||
sentiment = None
|
||||
if "check_sentiment" in query_params:
|
||||
if "sentiment_method" not in query_params:
|
||||
message = "No sentiment method"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if "sentiment" in query_params:
|
||||
sentiment = query_params["sentiment"]
|
||||
try:
|
||||
sentiment = float(sentiment)
|
||||
except ValueError:
|
||||
message = "Sentiment is not a float"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
sentiment_method = query_params["sentiment_method"]
|
||||
|
||||
return (sentiment_method, sentiment)
|
||||
@ -0,0 +1,787 @@
|
||||
from yaml import dump, load
|
||||
from yaml.parser import ParserError
|
||||
from yaml.scanner import ScannerError
|
||||
|
||||
try:
|
||||
from yaml import CDumper as Dumper
|
||||
from yaml import CLoader as Loader
|
||||
except ImportError:
|
||||
from yaml import Loader, Dumper
|
||||
|
||||
import uuid
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
|
||||
import orjson
|
||||
from siphashc import siphash
|
||||
|
||||
from core.lib.notify import sendmsg
|
||||
from core.lib.parsing import parse_index, parse_source
|
||||
from core.util import logs
|
||||
|
||||
log = logs.get_logger("rules")
|
||||
|
||||
SECONDS_PER_UNIT = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
|
||||
|
||||
MAX_WINDOW = 2592000
|
||||
MAX_AMOUNT_NTFY = 10
|
||||
MAX_AMOUNT_WEBHOOK = 1000
|
||||
HIGH_FREQUENCY_MIN_SEC = 60
|
||||
|
||||
|
||||
class RuleParseError(Exception):
|
||||
def __init__(self, message, field):
|
||||
super().__init__(message)
|
||||
self.field = field
|
||||
|
||||
|
||||
def format_ntfy(**kwargs):
|
||||
"""
|
||||
Format a message for ntfy.
|
||||
If the message is a list, it will be joined with newlines.
|
||||
If the message is None, it will be replaced with an empty string.
|
||||
If specified, `matched` will be pretty-printed in the first line.
|
||||
kwargs:
|
||||
rule: The rule object, must be specified
|
||||
index: The index the rule matched on, can be None
|
||||
message: The message to send, can be None
|
||||
meta:
|
||||
matched: The matched fields, can be None
|
||||
total_hits: The total number of matches, optional
|
||||
"""
|
||||
rule = kwargs.get("rule")
|
||||
index = kwargs.get("index")
|
||||
message = kwargs.get("message")
|
||||
|
||||
meta = kwargs.get("meta", {})
|
||||
total_hits = meta.get("total_hits", 0)
|
||||
matched = meta.get("matched")
|
||||
|
||||
if message:
|
||||
# Dump the message in YAML for readability
|
||||
messages_formatted = ""
|
||||
if isinstance(message, list):
|
||||
for message_iter in message:
|
||||
messages_formatted += dump(
|
||||
message_iter, Dumper=Dumper, default_flow_style=False
|
||||
)
|
||||
messages_formatted += "\n"
|
||||
else:
|
||||
messages_formatted = dump(message, Dumper=Dumper, default_flow_style=False)
|
||||
else:
|
||||
messages_formatted = ""
|
||||
|
||||
if matched:
|
||||
matched = ", ".join([f"{k}: {v}" for k, v in matched.items()])
|
||||
else:
|
||||
matched = ""
|
||||
|
||||
notify_message = f"{rule.name} on {index}: {matched}\n{messages_formatted}"
|
||||
notify_message += f"\nTotal hits: {total_hits}"
|
||||
notify_message = notify_message.encode("utf-8", "replace")
|
||||
|
||||
return notify_message
|
||||
|
||||
|
||||
def format_webhook(**kwargs):
|
||||
"""
|
||||
Format a message for a webhook.
|
||||
Adds some metadata to the message that would normally be only in
|
||||
notification_settings.
|
||||
Dumps the message in JSON.
|
||||
kwargs:
|
||||
rule: The rule object, must be specified
|
||||
index: The index the rule matched on, can be None
|
||||
message: The message to send, can be None, but will be sent as None
|
||||
meta:
|
||||
matched: The matched fields, can be None, but will be sent as None
|
||||
total_hits: The total number of matches, optional
|
||||
notification_settings: The notification settings, must be specified
|
||||
priority: The priority of the message, optional
|
||||
topic: The topic of the message, optional
|
||||
"""
|
||||
# rule = kwargs.get("rule")
|
||||
# index = kwargs.get("index")
|
||||
message = kwargs.get("message")
|
||||
meta = kwargs.get("meta")
|
||||
|
||||
notification_settings = kwargs.get("notification_settings")
|
||||
notify_message = {
|
||||
"data": message,
|
||||
"meta": meta,
|
||||
}
|
||||
if "priority" in notification_settings:
|
||||
notify_message["priority"] = notification_settings["priority"]
|
||||
if "topic" in notification_settings:
|
||||
notify_message["topic"] = notification_settings["topic"]
|
||||
notify_message = orjson.dumps(notify_message)
|
||||
|
||||
return notify_message
|
||||
|
||||
|
||||
def rule_notify(rule, index, message, meta=None):
|
||||
"""
|
||||
Send a notification for a matching rule.
|
||||
Gets the notification settings for the rule.
|
||||
Runs the formatting helpers for the service.
|
||||
:param rule: The rule object, must be specified
|
||||
:param index: The index the rule matched on, can be None
|
||||
:param message: The message to send, can be None
|
||||
:param meta: dict of metadata, contains `aggs` key for the matched fields
|
||||
"""
|
||||
# If there is no message, don't say anything matched
|
||||
if message:
|
||||
word = "match"
|
||||
else:
|
||||
word = "no match"
|
||||
|
||||
title = f"Rule {rule.name} {word} on {index}"
|
||||
|
||||
# The user notification settings are merged in with this
|
||||
notification_settings = rule.get_notification_settings()
|
||||
if not notification_settings:
|
||||
# No/invalid notification settings, don't send anything
|
||||
return
|
||||
if notification_settings.get("service") == "none":
|
||||
# Don't send anything
|
||||
return
|
||||
|
||||
# double sigh
|
||||
message_copy = deepcopy(message)
|
||||
for index, _ in enumerate(message_copy):
|
||||
if "meta" in message_copy[index]:
|
||||
del message_copy[index]["meta"]
|
||||
|
||||
# Create a cast we can reuse for the formatting helpers and sendmsg
|
||||
cast = {
|
||||
"title": title,
|
||||
"user": rule.user,
|
||||
"rule": rule,
|
||||
"index": index,
|
||||
"message": message_copy,
|
||||
"notification_settings": notification_settings,
|
||||
}
|
||||
if meta:
|
||||
cast["meta"] = meta
|
||||
|
||||
if rule.service == "ntfy":
|
||||
cast["msg"] = format_ntfy(**cast)
|
||||
|
||||
elif rule.service == "webhook":
|
||||
cast["msg"] = format_webhook(**cast)
|
||||
|
||||
sendmsg(**cast)
|
||||
|
||||
|
||||
class NotificationRuleData(object):
|
||||
def __init__(self, user, cleaned_data, db):
|
||||
self.user = user
|
||||
self.object = None
|
||||
|
||||
# We are running live and have been passed a database object
|
||||
if not isinstance(cleaned_data, dict):
|
||||
self.object = cleaned_data
|
||||
cleaned_data = cleaned_data.__dict__
|
||||
|
||||
self.cleaned_data = cleaned_data
|
||||
self.db = db
|
||||
self.data = self.cleaned_data.get("data")
|
||||
self.window = self.cleaned_data.get("window")
|
||||
self.policy = self.cleaned_data.get("policy")
|
||||
self.parsed = None
|
||||
self.aggs = {}
|
||||
|
||||
self.validate_user_permissions()
|
||||
|
||||
self.parse_data()
|
||||
self.ensure_list()
|
||||
self.validate_permissions()
|
||||
self.validate_schedule_fields()
|
||||
self.validate_time_fields()
|
||||
if self.object is not None:
|
||||
self.populate_matched()
|
||||
|
||||
def clear_database_matches(self):
|
||||
"""
|
||||
Delete all matches for this rule.
|
||||
"""
|
||||
rule_id = str(self.object.id)
|
||||
self.db.delete_rule_entries(rule_id)
|
||||
|
||||
def populate_matched(self):
|
||||
"""
|
||||
On first creation, the match field is None. We need to populate it with
|
||||
a dictionary containing the index names as keys and False as values.
|
||||
"""
|
||||
if self.object.match is None:
|
||||
self.object.match = {}
|
||||
for index in self.parsed["index"]:
|
||||
if index not in self.object.match:
|
||||
self.object.match[index] = False
|
||||
self.object.save()
|
||||
|
||||
def format_matched(self, messages):
|
||||
matched = {}
|
||||
for message in messages:
|
||||
for field, value in self.parsed.items():
|
||||
if field == "msg":
|
||||
# Allow partial matches for msg
|
||||
for msg in value:
|
||||
if "msg" in message:
|
||||
if msg.lower() in message["msg"].lower():
|
||||
matched[field] = msg
|
||||
# Break out of the msg matching loop
|
||||
break
|
||||
# Continue to next field
|
||||
continue
|
||||
if field == "tokens":
|
||||
# Allow partial matches for tokens
|
||||
for token in value:
|
||||
if "tokens" in message:
|
||||
if token.lower() in [x.lower() for x in message["tokens"]]:
|
||||
matched[field] = token
|
||||
# Break out of the token matching loop
|
||||
break
|
||||
# Continue to next field
|
||||
continue
|
||||
if field in message and message[field] in value:
|
||||
# Do exact matches for all other fields
|
||||
matched[field] = message[field]
|
||||
return matched
|
||||
|
||||
def store_match(self, index, match):
|
||||
"""
|
||||
Store a match result.
|
||||
Accepts None for the index to set all indices.
|
||||
:param index: the index to store the match for, can be None
|
||||
:param match: the object that matched
|
||||
"""
|
||||
if match is not False:
|
||||
# Dump match to JSON while sorting the keys
|
||||
match_normalised = orjson.dumps(match, option=orjson.OPT_SORT_KEYS)
|
||||
match = siphash(self.db.hash_key, match_normalised)
|
||||
|
||||
if self.object.match is None:
|
||||
self.object.match = {}
|
||||
if not isinstance(self.object.match, dict):
|
||||
self.object.match = {}
|
||||
|
||||
if index is None:
|
||||
for index_iter in self.parsed["index"]:
|
||||
self.object.match[index_iter] = match
|
||||
else:
|
||||
self.object.match[index] = match
|
||||
self.object.save()
|
||||
log.debug(f"Stored match: {index} - {match}")
|
||||
|
||||
def get_match(self, index=None, match=None):
|
||||
"""
|
||||
Get a match result for an index.
|
||||
If the index is None, it will return True if any index has a match.
|
||||
:param index: the index to get the match for, can be None
|
||||
"""
|
||||
if self.object.match is None:
|
||||
self.object.match = {}
|
||||
self.object.save()
|
||||
return None
|
||||
if not isinstance(self.object.match, dict):
|
||||
return None
|
||||
|
||||
if index is None:
|
||||
# Check if we have any matches on all indices
|
||||
values = self.object.match.values()
|
||||
if not values:
|
||||
return None
|
||||
return any(values)
|
||||
|
||||
# Check if it's the same hash
|
||||
if match is not None:
|
||||
match_normalised = orjson.dumps(match, option=orjson.OPT_SORT_KEYS)
|
||||
match = siphash(self.db.hash_key, match_normalised)
|
||||
hash_matches = self.object.match.get(index) == match
|
||||
return hash_matches
|
||||
|
||||
returned_match = self.object.match.get(index, None)
|
||||
if type(returned_match) == int:
|
||||
# We are getting a hash from the database,
|
||||
# but we have nothing to check it against.
|
||||
# In this instance, we are checking if we got a match
|
||||
# at all last time. We can confidently say that since
|
||||
# we have a hash, we did.
|
||||
returned_match = True
|
||||
return returned_match
|
||||
|
||||
def format_aggs(self, aggs):
|
||||
"""
|
||||
Format aggregations for the query.
|
||||
We have self.aggs, which contains:
|
||||
{"avg_sentiment": (">", 0.5)}
|
||||
and aggs, which contains:
|
||||
{"avg_sentiment": {"value": 0.6}}
|
||||
It's matched already, we just need to format it like so:
|
||||
{"avg_sentiment": "0.06>0.5"}
|
||||
:param aggs: the aggregations to format
|
||||
:return: the formatted aggregations
|
||||
"""
|
||||
new_aggs = {}
|
||||
for agg_name, agg in aggs.items():
|
||||
if agg_name in self.aggs:
|
||||
op, value = self.aggs[agg_name]
|
||||
new_aggs[agg_name] = f"{agg['value']}{op}{value}"
|
||||
|
||||
return new_aggs
|
||||
|
||||
def reform_matches(self, index, matches, meta, mode):
|
||||
if not isinstance(matches, list):
|
||||
matches = [matches]
|
||||
matches_copy = matches.copy()
|
||||
match_ts = datetime.utcnow().isoformat()
|
||||
batch_id = uuid.uuid4()
|
||||
|
||||
# Filter empty fields in meta
|
||||
meta = {k: v for k, v in meta.items() if v}
|
||||
|
||||
for match_index, _ in enumerate(matches_copy):
|
||||
matches_copy[match_index]["index"] = index
|
||||
matches_copy[match_index]["rule_id"] = str(self.object.id)
|
||||
matches_copy[match_index]["meta"] = meta
|
||||
matches_copy[match_index]["match_ts"] = match_ts
|
||||
matches_copy[match_index]["mode"] = mode
|
||||
matches_copy[match_index]["batch_id"] = str(batch_id)
|
||||
return matches_copy
|
||||
|
||||
async def ingest_matches(self, index, matches, meta, mode):
|
||||
"""
|
||||
Store all matches for an index.
|
||||
:param index: the index to store the matches for
|
||||
:param matches: the matches to store
|
||||
"""
|
||||
# new_matches = self.reform_matches(index, matches, meta, mode)
|
||||
if self.object.ingest:
|
||||
await self.db.async_store_matches(matches)
|
||||
|
||||
def ingest_matches_sync(self, index, matches, meta, mode):
|
||||
"""
|
||||
Store all matches for an index.
|
||||
:param index: the index to store the matches for
|
||||
:param matches: the matches to store
|
||||
"""
|
||||
# new_matches = self.reform_matches(index, matches, meta, mode)
|
||||
if self.object.ingest:
|
||||
self.db.store_matches(matches)
|
||||
|
||||
async def rule_matched(self, index, message, meta, mode):
|
||||
"""
|
||||
A rule has matched.
|
||||
If the previous run did not match, send a notification after formatting
|
||||
the aggregations.
|
||||
:param index: the index the rule matched on
|
||||
:param message: the message object that matched
|
||||
:param aggs: the aggregations that matched
|
||||
"""
|
||||
current_match = self.get_match(index, message)
|
||||
log.debug(f"Rule matched: {index} - current match: {current_match}")
|
||||
|
||||
last_run_had_matches = current_match is True
|
||||
|
||||
if self.policy in ["change", "default"]:
|
||||
# Change or Default policy, notifying only on new results
|
||||
if last_run_had_matches:
|
||||
# Last run had matches, and this one did too
|
||||
# We don't need to notify
|
||||
return
|
||||
|
||||
elif self.policy == "always":
|
||||
# Only here for completeness, we notify below by default
|
||||
pass
|
||||
|
||||
# We hit the return above if we don't need to notify
|
||||
if "matched" not in meta:
|
||||
meta["matched"] = self.format_matched(message)
|
||||
if "aggs" in meta:
|
||||
aggs_formatted = self.format_aggs(meta["aggs"])
|
||||
if aggs_formatted:
|
||||
meta["matched_aggs"] = aggs_formatted
|
||||
|
||||
meta["is_match"] = True
|
||||
self.store_match(index, message)
|
||||
|
||||
message = self.reform_matches(index, message, meta, mode)
|
||||
rule_notify(self.object, index, message, meta)
|
||||
await self.ingest_matches(index, message, meta, mode)
|
||||
|
||||
def rule_matched_sync(self, index, message, meta, mode):
|
||||
"""
|
||||
A rule has matched.
|
||||
If the previous run did not match, send a notification after formatting
|
||||
the aggregations.
|
||||
:param index: the index the rule matched on
|
||||
:param message: the message object that matched
|
||||
:param aggs: the aggregations that matched
|
||||
"""
|
||||
current_match = self.get_match(index, message)
|
||||
log.debug(f"Rule matched: {index} - current match: {current_match}")
|
||||
|
||||
last_run_had_matches = current_match is True
|
||||
|
||||
if self.policy in ["change", "default"]:
|
||||
# Change or Default policy, notifying only on new results
|
||||
if last_run_had_matches:
|
||||
# Last run had matches, and this one did too
|
||||
# We don't need to notify
|
||||
return
|
||||
|
||||
elif self.policy == "always":
|
||||
# Only here for completeness, we notify below by default
|
||||
pass
|
||||
|
||||
# We hit the return above if we don't need to notify
|
||||
if "matched" not in meta:
|
||||
meta["matched"] = self.format_matched(message)
|
||||
if "aggs" in meta:
|
||||
aggs_formatted = self.format_aggs(meta["aggs"])
|
||||
if aggs_formatted:
|
||||
meta["matched_aggs"] = aggs_formatted
|
||||
|
||||
meta["is_match"] = True
|
||||
self.store_match(index, message)
|
||||
|
||||
message = self.reform_matches(index, message, meta, mode)
|
||||
rule_notify(self.object, index, message, meta)
|
||||
self.ingest_matches_sync(index, message, meta, mode)
|
||||
|
||||
# No async helper for this one as we only need it for schedules
|
||||
async def rule_no_match(self, index=None, message=None, mode=None):
|
||||
"""
|
||||
A rule has not matched.
|
||||
If the previous run did match, send a notification if configured to notify
|
||||
for empty matches.
|
||||
:param index: the index the rule did not match on, can be None
|
||||
|
||||
"""
|
||||
current_match = self.get_match(index)
|
||||
log.debug(
|
||||
f"Rule not matched: {index} - current match: {current_match}: {message}"
|
||||
)
|
||||
|
||||
last_run_had_matches = current_match is True
|
||||
initial = current_match is None
|
||||
|
||||
self.store_match(index, False)
|
||||
|
||||
if self.policy != "always":
|
||||
# We hit the return above if we don't need to notify
|
||||
if self.policy in ["change", "default"]:
|
||||
if not last_run_had_matches and not initial:
|
||||
# We don't need to notify if the last run didn't have matches
|
||||
return
|
||||
|
||||
if self.policy in ["always", "change"]:
|
||||
# Never notify for empty matches on default policy
|
||||
meta = {"msg": message, "is_match": False}
|
||||
matches = [{"msg": None}]
|
||||
message = self.reform_matches(index, matches, meta, mode)
|
||||
rule_notify(self.object, index, matches, meta)
|
||||
await self.ingest_matches(
|
||||
index=index,
|
||||
matches=matches,
|
||||
meta=meta,
|
||||
mode="schedule",
|
||||
)
|
||||
|
||||
async def run_schedule(self):
|
||||
"""
|
||||
Run the schedule query.
|
||||
Get the results from the database, and check if the rule has matched.
|
||||
Check if all of the required aggregations have matched.
|
||||
"""
|
||||
response = await self.db.schedule_query_results(self)
|
||||
if not response:
|
||||
# No results in the result_map
|
||||
await self.rule_no_match(
|
||||
message="No response from database", mode="schedule"
|
||||
)
|
||||
return
|
||||
for index, (meta, results) in response.items():
|
||||
if not results:
|
||||
# Falsy results, no matches
|
||||
await self.rule_no_match(
|
||||
index, message="No results for index", mode="schedule"
|
||||
)
|
||||
continue
|
||||
|
||||
# Add the match values of all aggregations to a list
|
||||
aggs_for_index = []
|
||||
for agg_name in self.aggs.keys():
|
||||
if agg_name in meta["aggs"]:
|
||||
if "match" in meta["aggs"][agg_name]:
|
||||
aggs_for_index.append(meta["aggs"][agg_name]["match"])
|
||||
|
||||
# All required aggs are present
|
||||
if len(aggs_for_index) == len(self.aggs.keys()):
|
||||
if all(aggs_for_index):
|
||||
# All aggs have matched
|
||||
await self.rule_matched(
|
||||
index, results[: self.object.amount], meta, mode="schedule"
|
||||
)
|
||||
continue
|
||||
# Default branch, since the happy path has a continue keyword
|
||||
await self.rule_no_match(
|
||||
index, message="Aggregation did not match", mode="schedule"
|
||||
)
|
||||
|
||||
def test_schedule(self):
|
||||
"""
|
||||
Test the schedule query to ensure it is valid.
|
||||
Raises an exception if the query is invalid.
|
||||
"""
|
||||
if self.db:
|
||||
self.db.schedule_query_results_test_sync(self)
|
||||
|
||||
def validate_schedule_fields(self):
|
||||
"""
|
||||
Ensure schedule fields are valid.
|
||||
index: can be a list, it will schedule one search per index.
|
||||
source: can be a list, it will be the filter for each search.
|
||||
tokens: can be list, it will ensure the message matches any token.
|
||||
msg: can be a list, it will ensure the message contains any msg.
|
||||
No other fields can be lists containing more than one item.
|
||||
:raises RuleParseError: if the fields are invalid
|
||||
"""
|
||||
is_schedule = self.is_schedule
|
||||
|
||||
if is_schedule:
|
||||
allowed_list_fields = ["index", "source", "tokens", "msg"]
|
||||
for field, value in self.parsed.items():
|
||||
if field not in allowed_list_fields:
|
||||
if len(value) > 1:
|
||||
raise RuleParseError(
|
||||
(
|
||||
f"For scheduled rules, field {field} cannot contain "
|
||||
"more than one item"
|
||||
),
|
||||
"data",
|
||||
)
|
||||
if len(str(value[0])) == 0:
|
||||
raise RuleParseError(f"Field {field} cannot be empty", "data")
|
||||
if "sentiment" in self.parsed:
|
||||
sentiment = str(self.parsed["sentiment"][0])
|
||||
sentiment = sentiment.strip()
|
||||
if sentiment[0] not in [">", "<", "="]:
|
||||
raise RuleParseError(
|
||||
(
|
||||
"Sentiment field must be a comparison operator and then a "
|
||||
"float: >0.02"
|
||||
),
|
||||
"data",
|
||||
)
|
||||
operator = sentiment[0]
|
||||
number = sentiment[1:]
|
||||
|
||||
try:
|
||||
number = float(number)
|
||||
except ValueError:
|
||||
raise RuleParseError(
|
||||
(
|
||||
"Sentiment field must be a comparison operator and then a "
|
||||
"float: >0.02"
|
||||
),
|
||||
"data",
|
||||
)
|
||||
self.aggs["avg_sentiment"] = (operator, number)
|
||||
|
||||
else:
|
||||
if "query" in self.parsed:
|
||||
raise RuleParseError(
|
||||
"Field query cannot be used with on-demand rules", "data"
|
||||
)
|
||||
if "tags" in self.parsed:
|
||||
raise RuleParseError(
|
||||
"Field tags cannot be used with on-demand rules", "data"
|
||||
)
|
||||
if self.policy != "default":
|
||||
raise RuleParseError(
|
||||
(
|
||||
f"Cannot use {self.cleaned_data['policy']} policy with "
|
||||
"on-demand rules"
|
||||
),
|
||||
"policy",
|
||||
)
|
||||
|
||||
@property
|
||||
def is_schedule(self):
|
||||
"""
|
||||
Check if the rule is a schedule rule.
|
||||
:return: True if the rule is a schedule rule, False otherwise
|
||||
"""
|
||||
if "interval" in self.cleaned_data:
|
||||
if self.cleaned_data["interval"] != 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
def ensure_list(self):
|
||||
"""
|
||||
Ensure all values in the data field are lists.
|
||||
Convert all strings to lists with one item.
|
||||
"""
|
||||
for field, value in self.parsed.items():
|
||||
if not isinstance(value, list):
|
||||
self.parsed[field] = [value]
|
||||
|
||||
def validate_user_permissions(self):
|
||||
"""
|
||||
Ensure the user can use notification rules.
|
||||
:raises RuleParseError: if the user does not have permission
|
||||
"""
|
||||
if not self.user.has_perm("core.use_rules"):
|
||||
raise RuleParseError("User does not have permission to use rules", "data")
|
||||
|
||||
def validate_time_fields(self):
|
||||
"""
|
||||
Validate the interval and window fields.
|
||||
Prohibit window being specified with an ondemand interval.
|
||||
Prohibit window not being specified with a non-ondemand interval.
|
||||
Prohibit amount being specified with an on-demand interval.
|
||||
Prohibut amount not being specified with a non-ondemand interval.
|
||||
Validate window field.
|
||||
Validate window unit and enforce maximum.
|
||||
:raises RuleParseError: if the fields are invalid
|
||||
"""
|
||||
interval = self.cleaned_data.get("interval")
|
||||
window = self.cleaned_data.get("window")
|
||||
amount = self.cleaned_data.get("amount")
|
||||
service = self.cleaned_data.get("service")
|
||||
|
||||
on_demand = interval == 0
|
||||
|
||||
# Not on demand and interval is too low
|
||||
if not on_demand and interval <= HIGH_FREQUENCY_MIN_SEC:
|
||||
if not self.user.has_perm("core.rules_high_frequency"):
|
||||
raise RuleParseError(
|
||||
"User does not have permission to use high frequency rules", "data"
|
||||
)
|
||||
|
||||
if not on_demand:
|
||||
if not self.user.has_perm("core.rules_scheduled"):
|
||||
raise RuleParseError(
|
||||
"User does not have permission to use scheduled rules", "data"
|
||||
)
|
||||
|
||||
if on_demand and window is not None:
|
||||
# Interval is on demand and window is specified
|
||||
# We can't have a window with on-demand rules
|
||||
raise RuleParseError(
|
||||
"Window cannot be specified with on-demand interval", "window"
|
||||
)
|
||||
|
||||
if not on_demand and window is None:
|
||||
# Interval is not on demand and window is not specified
|
||||
# We can't have a non-on-demand interval without a window
|
||||
raise RuleParseError(
|
||||
"Window must be specified with non-on-demand interval", "window"
|
||||
)
|
||||
|
||||
if not on_demand and amount is None:
|
||||
# Interval is not on demand and amount is not specified
|
||||
# We can't have a non-on-demand interval without an amount
|
||||
raise RuleParseError(
|
||||
"Amount must be specified with non-on-demand interval", "amount"
|
||||
)
|
||||
if on_demand and amount is not None:
|
||||
# Interval is on demand and amount is specified
|
||||
# We can't have an amount with on-demand rules
|
||||
raise RuleParseError(
|
||||
"Amount cannot be specified with on-demand interval", "amount"
|
||||
)
|
||||
|
||||
if window is not None:
|
||||
window_number = window[:-1]
|
||||
if not window_number.isdigit():
|
||||
raise RuleParseError("Window prefix must be a number", "window")
|
||||
window_number = int(window_number)
|
||||
window_unit = window[-1]
|
||||
if window_unit not in SECONDS_PER_UNIT:
|
||||
raise RuleParseError(
|
||||
(
|
||||
"Window unit must be one of "
|
||||
f"{', '.join(SECONDS_PER_UNIT.keys())},"
|
||||
f" not '{window_unit}'"
|
||||
),
|
||||
"window",
|
||||
)
|
||||
window_seconds = window_number * SECONDS_PER_UNIT[window_unit]
|
||||
if window_seconds > MAX_WINDOW:
|
||||
raise RuleParseError(
|
||||
f"Window cannot be larger than {MAX_WINDOW} seconds (30 days)",
|
||||
"window",
|
||||
)
|
||||
|
||||
if amount is not None:
|
||||
if service == "ntfy":
|
||||
if amount > MAX_AMOUNT_NTFY:
|
||||
raise RuleParseError(
|
||||
f"Amount cannot be larger than {MAX_AMOUNT_NTFY} for ntfy",
|
||||
"amount",
|
||||
)
|
||||
else:
|
||||
if amount > MAX_AMOUNT_WEBHOOK:
|
||||
raise RuleParseError(
|
||||
(
|
||||
f"Amount cannot be larger than {MAX_AMOUNT_WEBHOOK} for "
|
||||
f"{service}"
|
||||
),
|
||||
"amount",
|
||||
)
|
||||
|
||||
def validate_permissions(self):
|
||||
"""
|
||||
Validate permissions for the source and index variables.
|
||||
Also set the default values for the user if not present.
|
||||
Stores the default or expanded values in the parsed field.
|
||||
:raises QueryError: if the user does not have permission to use the source
|
||||
"""
|
||||
if "index" in self.parsed:
|
||||
index = self.parsed["index"]
|
||||
if type(index) == list:
|
||||
for i in index:
|
||||
parse_index(self.user, {"index": i}, raise_error=True)
|
||||
# else:
|
||||
# db.parse_index(self.user, {"index": index}, raise_error=True)
|
||||
else:
|
||||
# Get the default value for the user if not present
|
||||
index = parse_index(self.user, {}, raise_error=True)
|
||||
self.parsed["index"] = [index]
|
||||
|
||||
if "source" in self.parsed:
|
||||
source = self.parsed["source"]
|
||||
if type(source) == list:
|
||||
for i in source:
|
||||
parse_source(self.user, {"source": i}, raise_error=True)
|
||||
# else:
|
||||
# parse_source(self.user, {"source": source}, raise_error=True)
|
||||
else:
|
||||
# Get the default value for the user if not present
|
||||
source = parse_source(self.user, {}, raise_error=True)
|
||||
self.parsed["source"] = source
|
||||
|
||||
def parse_data(self):
|
||||
"""
|
||||
Parse the data in the text field to YAML.
|
||||
:raises RuleParseError: if the data is invalid
|
||||
"""
|
||||
try:
|
||||
self.parsed = load(self.data, Loader=Loader)
|
||||
except (ScannerError, ParserError) as e:
|
||||
raise RuleParseError(f"Invalid YAML: {e}", "data")
|
||||
|
||||
def __str__(self):
|
||||
"""
|
||||
Get a YAML representation of the data field of the rule.
|
||||
"""
|
||||
return dump(self.parsed, Dumper=Dumper)
|
||||
|
||||
def get_data(self):
|
||||
"""
|
||||
Return the data field as a dictionary.
|
||||
"""
|
||||
return self.parsed
|
||||
@ -0,0 +1,107 @@
|
||||
import msgpack
|
||||
from django.core.management.base import BaseCommand
|
||||
from redis import StrictRedis
|
||||
|
||||
from core.db.storage import db
|
||||
from core.lib.rules import NotificationRuleData
|
||||
from core.models import NotificationRule
|
||||
from core.util import logs
|
||||
|
||||
log = logs.get_logger("processing")
|
||||
|
||||
|
||||
def process_rules(data):
|
||||
all_rules = NotificationRule.objects.filter(enabled=True, interval=0)
|
||||
|
||||
for index, index_messages in data.items():
|
||||
for message in index_messages:
|
||||
for rule in all_rules:
|
||||
# Quicker helper to get the data without spinning
|
||||
# up a NotificationRuleData object
|
||||
parsed_rule = rule.parse()
|
||||
matched = {}
|
||||
# Rule is invalid, this shouldn't happen
|
||||
if "index" not in parsed_rule:
|
||||
continue
|
||||
if "source" not in parsed_rule:
|
||||
continue
|
||||
rule_index = parsed_rule["index"]
|
||||
rule_source = parsed_rule["source"]
|
||||
# if not type(rule_index) == list:
|
||||
# rule_index = [rule_index]
|
||||
# if not type(rule_source) == list:
|
||||
# rule_source = [rule_source]
|
||||
if index not in rule_index:
|
||||
# We don't care about this index, go to the next one
|
||||
continue
|
||||
if message["src"] not in rule_source:
|
||||
# We don't care about this source, go to the next one
|
||||
continue
|
||||
|
||||
matched["index"] = index
|
||||
matched["source"] = message["src"]
|
||||
|
||||
rule_field_length = len(parsed_rule.keys())
|
||||
matched_field_number = 0
|
||||
for field, value in parsed_rule.items():
|
||||
# if not type(value) == list:
|
||||
# value = [value]
|
||||
if field == "src":
|
||||
# We already checked this
|
||||
continue
|
||||
if field == "tokens":
|
||||
# Check if tokens are in the rule
|
||||
# We only check if *at least one* token matches
|
||||
for token in value:
|
||||
if "tokens" in message:
|
||||
if token in message["tokens"]:
|
||||
matched_field_number += 1
|
||||
matched[field] = token
|
||||
# Break out of the token matching loop
|
||||
break
|
||||
# Continue to next field
|
||||
continue
|
||||
|
||||
if field == "msg":
|
||||
# Allow partial matches for msg
|
||||
for msg in value:
|
||||
if "msg" in message:
|
||||
if msg.lower() in message["msg"].lower():
|
||||
matched_field_number += 1
|
||||
matched[field] = msg
|
||||
# Break out of the msg matching loop
|
||||
break
|
||||
# Continue to next field
|
||||
continue
|
||||
if field in message and message[field] in value:
|
||||
# Do exact matches for all other fields
|
||||
matched_field_number += 1
|
||||
matched[field] = message[field]
|
||||
# Subtract 2, 1 for source and 1 for index
|
||||
if matched_field_number == rule_field_length - 2:
|
||||
meta = {"matched": matched, "total_hits": 1}
|
||||
|
||||
# Parse the rule, we saved some work above to avoid doing this,
|
||||
# but it makes delivering messages significantly easier as we can
|
||||
# use the same code as for scheduling.
|
||||
rule_data_object = NotificationRuleData(rule.user, rule, db=db)
|
||||
# rule_notify(rule, index, message, meta=meta)
|
||||
rule_data_object.rule_matched_sync(
|
||||
index, message, meta=meta, mode="ondemand"
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **options):
|
||||
r = StrictRedis(unix_socket_path="/var/run/socks/redis.sock", db=0)
|
||||
p = r.pubsub()
|
||||
p.psubscribe("messages")
|
||||
for message in p.listen():
|
||||
if message:
|
||||
if message["channel"] == b"messages":
|
||||
data = message["data"]
|
||||
try:
|
||||
unpacked = msgpack.unpackb(data, raw=False)
|
||||
except TypeError:
|
||||
continue
|
||||
process_rules(unpacked)
|
||||
@ -0,0 +1,54 @@
|
||||
import asyncio
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from core.db.storage import db
|
||||
from core.lib.parsing import QueryError
|
||||
from core.lib.rules import NotificationRuleData, RuleParseError
|
||||
from core.models import NotificationRule
|
||||
from core.util import logs
|
||||
|
||||
log = logs.get_logger("scheduling")
|
||||
|
||||
INTERVALS = [5, 60, 900, 1800, 3600, 14400, 86400]
|
||||
|
||||
|
||||
async def job(interval_seconds):
|
||||
"""
|
||||
Run all schedules matching the given interval.
|
||||
:param interval_seconds: The interval to run.
|
||||
"""
|
||||
matching_rules = await sync_to_async(list)(
|
||||
NotificationRule.objects.filter(enabled=True, interval=interval_seconds)
|
||||
)
|
||||
for rule in matching_rules:
|
||||
log.debug(f"Running rule {rule}")
|
||||
try:
|
||||
rule = NotificationRuleData(rule.user, rule, db=db)
|
||||
await rule.run_schedule()
|
||||
# results = await db.schedule_query_results(rule.user, rule)
|
||||
except QueryError as e:
|
||||
log.error(f"Error running rule {rule}: {e}")
|
||||
except RuleParseError as e:
|
||||
log.error(f"Error parsing rule {rule}: {e}")
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **options):
|
||||
"""
|
||||
Start the scheduling process.
|
||||
"""
|
||||
scheduler = AsyncIOScheduler()
|
||||
for interval in INTERVALS:
|
||||
log.debug(f"Scheduling {interval} second job")
|
||||
scheduler.add_job(job, "interval", seconds=interval, args=[interval])
|
||||
scheduler.start()
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop.run_forever()
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
log.info("Process terminating")
|
||||
finally:
|
||||
loop.close()
|
||||
@ -0,0 +1,17 @@
|
||||
# Generated by Django 4.1.3 on 2022-11-29 12:04
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0010_alter_perms_options'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelOptions(
|
||||
name='perms',
|
||||
options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,25 @@
|
||||
# Generated by Django 4.1.3 on 2023-01-12 15:12
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0011_alter_perms_options'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='NotificationRule',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('name', models.CharField(max_length=255)),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('data', models.TextField()),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,24 @@
|
||||
# Generated by Django 4.1.3 on 2023-01-12 15:25
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0012_notificationrule'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='NotificationSettings',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('ntfy_topic', models.CharField(blank=True, max_length=255, null=True)),
|
||||
('ntfy_url', models.CharField(blank=True, max_length=255, null=True)),
|
||||
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,18 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-12 18:06
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0013_notificationsettings'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='priority',
|
||||
field=models.IntegerField(choices=[(1, 'min'), (2, 'low'), (3, 'default'), (4, 'high'), (5, 'max')], default=1),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,18 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-12 18:14
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0014_notificationrule_priority'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='topic',
|
||||
field=models.CharField(blank=True, max_length=255, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,23 @@
|
||||
# Generated by Django 4.1.3 on 2023-01-14 14:33
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0015_notificationrule_topic'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='interval',
|
||||
field=models.CharField(choices=[('ondemand', 'On demand'), ('minute', 'Every minute'), ('15m', 'Every 15 minutes'), ('30m', 'Every 30 minutes'), ('hour', 'Every hour'), ('4h', 'Every 4 hours'), ('day', 'Every day'), ('week', 'Every week'), ('month', 'Every month')], default='ondemand', max_length=255),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='window',
|
||||
field=models.CharField(blank=True, max_length=255, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,18 @@
|
||||
# Generated by Django 4.1.3 on 2023-01-14 14:54
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0016_notificationrule_interval_notificationrule_window'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='interval',
|
||||
field=models.IntegerField(choices=[(0, 'On demand'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=0),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,27 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-15 00:58
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0017_alter_notificationrule_interval'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelOptions(
|
||||
name='perms',
|
||||
options={'permissions': (('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('use_insights', 'Can use the Insights page'), ('use_rules', 'Can use the Rules page'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='match',
|
||||
field=models.BooleanField(default=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='interval',
|
||||
field=models.IntegerField(choices=[(0, 'On demand'), (5, 'Every 5 seconds'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=0),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,18 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-15 01:52
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0018_alter_perms_options_notificationrule_match_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='match',
|
||||
field=models.JSONField(blank=True, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,42 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-15 18:14
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0019_alter_notificationrule_match'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name='notificationsettings',
|
||||
old_name='ntfy_topic',
|
||||
new_name='topic',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='notificationsettings',
|
||||
name='ntfy_url',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='service',
|
||||
field=models.CharField(blank=True, choices=[('ntfy', 'NTFY'), ('wehbook', 'Custom webhook')], max_length=255, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='url',
|
||||
field=models.CharField(blank=True, max_length=1024, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='notificationsettings',
|
||||
name='service',
|
||||
field=models.CharField(blank=True, choices=[('ntfy', 'NTFY'), ('wehbook', 'Custom webhook')], max_length=255, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='notificationsettings',
|
||||
name='url',
|
||||
field=models.CharField(blank=True, max_length=1024, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,28 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-15 20:45
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0020_rename_ntfy_topic_notificationsettings_topic_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='amount',
|
||||
field=models.IntegerField(blank=True, default=1, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='service',
|
||||
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook')], default='ntfy', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationsettings',
|
||||
name='service',
|
||||
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook')], default='ntfy', max_length=255),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,23 @@
|
||||
# Generated by Django 4.1.5 on 2023-01-15 23:34
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0021_notificationrule_amount_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='send_empty',
|
||||
field=models.BooleanField(default=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='amount',
|
||||
field=models.PositiveIntegerField(blank=True, default=1, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,17 @@
|
||||
# Generated by Django 4.1.5 on 2023-02-02 19:07
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0022_notificationrule_send_empty_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelOptions(
|
||||
name='perms',
|
||||
options={'permissions': (('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('use_insights', 'Can use the Insights page'), ('use_rules', 'Can use the Rules page'), ('rules_scheduled', 'Can use the scheduled rules'), ('rules_high_frequency', 'Can use the high frequency rules'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,20 @@
|
||||
# Generated by Django 4.1.5 on 2023-02-02 19:08
|
||||
|
||||
import uuid
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0023_alter_perms_options'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='id',
|
||||
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,20 @@
|
||||
# Generated by Django 4.1.5 on 2023-02-02 19:35
|
||||
|
||||
import uuid
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0024_alter_notificationrule_id'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='id',
|
||||
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,28 @@
|
||||
# Generated by Django 4.1.5 on 2023-02-09 14:38
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0025_alter_notificationrule_id'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='notificationrule',
|
||||
name='policy',
|
||||
field=models.CharField(choices=[('default', 'Only trigger for matched events'), ('change', 'Trigger only if no results found when they were last run'), ('always', 'Always trigger regardless of whether results are found')], default='default', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='service',
|
||||
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook'), ('none', 'Disabled')], default='ntfy', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationsettings',
|
||||
name='service',
|
||||
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook'), ('none', 'Disabled')], default='ntfy', max_length=255),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,28 @@
|
||||
# Generated by Django 4.1.6 on 2023-02-13 10:18
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0026_notificationrule_policy_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='policy',
|
||||
field=models.CharField(choices=[('default', 'Default: Trigger only when there were no results last time'), ('change', 'Change: Default + trigger when there are no results (if there were before)'), ('always', 'Always: Trigger on every run (not recommended for low intervals)')], default='default', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='topic',
|
||||
field=models.CharField(blank=True, max_length=2048, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationsettings',
|
||||
name='topic',
|
||||
field=models.CharField(blank=True, max_length=2048, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,33 @@
|
||||
# Generated by Django 4.1.6 on 2023-02-13 21:03
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0027_alter_notificationrule_policy_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name='notificationrule',
|
||||
old_name='send_empty',
|
||||
new_name='ingest',
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='interval',
|
||||
field=models.IntegerField(choices=[(0, 'On demand'), (5, 'Every 5 seconds'), (60, 'Every minute'), (900, 'Every 15 minutes'), (1800, 'Every 30 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=60),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='service',
|
||||
field=models.CharField(choices=[('ntfy', 'NTFY'), ('webhook', 'Custom webhook'), ('none', 'Disabled')], default='webhook', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='notificationrule',
|
||||
name='window',
|
||||
field=models.CharField(blank=True, default='30d', max_length=255, null=True),
|
||||
),
|
||||
]
|
||||
@ -1,48 +1,152 @@
|
||||
{% extends "base.html" %}
|
||||
{% extends 'base.html' %}
|
||||
{% load static %}
|
||||
{% load joinsep %}
|
||||
{% block outer_content %}
|
||||
{% if params.modal == 'context' %}
|
||||
<div
|
||||
style="display: none;"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_context' %}"
|
||||
hx-vals='{"net": "{{ params.net|escapejs }}",
|
||||
"num": "{{ params.num|escapejs }}",
|
||||
"source": "{{ params.source|escapejs }}",
|
||||
"channel": "{{ params.channel|escapejs }}",
|
||||
"time": "{{ params.time|escapejs }}",
|
||||
"date": "{{ params.date|escapejs }}",
|
||||
"index": "{{ params.index }}",
|
||||
"type": "{{ params.type|escapejs }}",
|
||||
"mtype": "{{ params.mtype|escapejs }}",
|
||||
"nick": "{{ params.nick|escapejs }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="load">
|
||||
</div>
|
||||
{% endif %}
|
||||
<script src="{% static 'js/chart.js' %}"></script>
|
||||
<script src="{% static 'tabs.js' %}"></script>
|
||||
<script>
|
||||
function setupTags() {
|
||||
var inputTags = document.getElementById('tags');
|
||||
new BulmaTagsInput(inputTags);
|
||||
|
||||
{% block content %}
|
||||
<div class="block">
|
||||
{% for block in blocks %}
|
||||
{% if block.title is not None %}
|
||||
<h1 class="title">{{ block.title }}</h1>
|
||||
{% endif %}
|
||||
<div class="box">
|
||||
<div class="columns">
|
||||
{% if block.column1 is not None %}
|
||||
<div class="column">
|
||||
{{ block.column1 }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if block.column2 is not None %}
|
||||
<div class="column">
|
||||
{{ block.column2 }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if block.column3 is not None %}
|
||||
<div class="column">
|
||||
{{ block.column3 }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="columns">
|
||||
{% if block.image1 is not None %}
|
||||
<div class="column">
|
||||
<img src="{% static block.image1 %}">
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if block.image2 is not None %}
|
||||
<div class="column">
|
||||
<img src="{% static block.image2 %}">
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if block.image3 is not None %}
|
||||
<div class="column">
|
||||
<img src="{% static block.image3 %}">
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
inputTags.BulmaTagsInput().on('before.add', function(item) {
|
||||
if (item.includes(": ")) {
|
||||
var spl = item.split(": ");
|
||||
} else {
|
||||
var spl = item.split(":");
|
||||
}
|
||||
var field = spl[0];
|
||||
try {
|
||||
var value = JSON.parse(spl[1]);
|
||||
} catch {
|
||||
var value = spl[1];
|
||||
}
|
||||
return `${field}: ${value}`;
|
||||
});
|
||||
inputTags.BulmaTagsInput().on('after.remove', function(item) {
|
||||
var spl = item.split(": ");
|
||||
var field = spl[0];
|
||||
var value = spl[1].trim();
|
||||
});
|
||||
}
|
||||
function populateSearch(field, value) {
|
||||
var inputTags = document.getElementById('tags');
|
||||
inputTags.BulmaTagsInput().add(field+": "+value);
|
||||
//htmx.trigger("#search", "click");
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="grid-stack" id="grid-stack-main">
|
||||
<div class="grid-stack-item" gs-w="7" gs-h="10" gs-y="0" gs-x="1">
|
||||
<div class="grid-stack-item-content">
|
||||
<nav class="panel">
|
||||
<p class="panel-heading" style="padding: .2em; line-height: .5em;">
|
||||
<i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
|
||||
Search
|
||||
</p>
|
||||
<article class="panel-block is-active">
|
||||
{% include 'window-content/search.html' %}
|
||||
</article>
|
||||
</nav>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var grid = GridStack.init({
|
||||
cellHeight: 20,
|
||||
cellWidth: 50,
|
||||
cellHeightUnit: 'px',
|
||||
auto: true,
|
||||
float: true,
|
||||
draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
|
||||
removable: false,
|
||||
animate: true,
|
||||
});
|
||||
// GridStack.init();
|
||||
setupTags();
|
||||
|
||||
// a widget is ready to be loaded
|
||||
document.addEventListener('load-widget', function(event) {
|
||||
let container = htmx.find('#widget');
|
||||
// get the scripts, they won't be run on the new element so we need to eval them
|
||||
var scripts = htmx.findAll(container, "script");
|
||||
let widgetelement = container.firstElementChild.cloneNode(true);
|
||||
var new_id = widgetelement.id;
|
||||
|
||||
// check if there's an existing element like the one we want to swap
|
||||
let grid_element = htmx.find('#grid-stack-main');
|
||||
let existing_widget = htmx.find(grid_element, "#"+new_id);
|
||||
|
||||
// get the size and position attributes
|
||||
if (existing_widget) {
|
||||
let attrs = existing_widget.getAttributeNames();
|
||||
for (let i = 0, len = attrs.length; i < len; i++) {
|
||||
if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
|
||||
widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clear the queue element
|
||||
container.outerHTML = "";
|
||||
|
||||
// temporary workaround, other widgets can be duplicated, but not results
|
||||
if (widgetelement.id == 'widget-results') {
|
||||
grid.removeWidget("widget-results");
|
||||
}
|
||||
|
||||
grid.addWidget(widgetelement);
|
||||
// re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
|
||||
htmx.process(widgetelement);
|
||||
|
||||
// update size when the widget is loaded
|
||||
document.addEventListener('load-widget-results', function(evt) {
|
||||
var added_widget = htmx.find(grid_element, '#widget-results');
|
||||
var itemContent = htmx.find(added_widget, ".control");
|
||||
var scrollheight = itemContent.scrollHeight+80;
|
||||
var verticalmargin = 0;
|
||||
var cellheight = grid.opts.cellHeight;
|
||||
var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
|
||||
var opts = {
|
||||
h: height,
|
||||
}
|
||||
grid.update(
|
||||
added_widget,
|
||||
opts
|
||||
);
|
||||
});
|
||||
|
||||
// run the JS scripts inside the added element again
|
||||
// for instance, this will fix the dropdown
|
||||
for (var i = 0; i < scripts.length; i++) {
|
||||
eval(scripts[i].innerHTML);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
{% block widgets %}
|
||||
{% if table or message is not None %}
|
||||
{% include 'partials/results_load.html' %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@ -1,4 +1,4 @@
|
||||
{% extends 'wm/modal.html' %}
|
||||
{% extends 'mixins/wm/modal.html' %}
|
||||
|
||||
{% block modal_content %}
|
||||
{% include 'window-content/drilldown.html' %}
|
||||
|
||||
@ -1,48 +1,48 @@
|
||||
{% load static %}
|
||||
|
||||
{% for plan in plans %}
|
||||
|
||||
|
||||
<div class="box">
|
||||
<article class="media">
|
||||
<div class="media-left">
|
||||
<figure class="image is-64x64">
|
||||
<img src="{% static plan.image %}" alt="Image">
|
||||
</figure>
|
||||
</div>
|
||||
<div class="media-content">
|
||||
<div class="content">
|
||||
<p>
|
||||
<strong>{{ plan.name }}</strong> <small>£{{ plan.cost }}</small>
|
||||
{% if plan in user_plans %}
|
||||
<i class="fas fa-check" aria-hidden="true"></i>
|
||||
{% endif %}
|
||||
<br>
|
||||
{{ plan.description }}
|
||||
</p>
|
||||
{% load cache %}
|
||||
{% load cachalot cache %}
|
||||
{% get_last_invalidation 'core.Plan' as last %}
|
||||
{% cache 600 objects_plans request.user.id plans last %}
|
||||
{% for plan in plans %}
|
||||
<div class="box">
|
||||
<article class="media">
|
||||
<div class="media-left">
|
||||
<figure class="image is-64x64">
|
||||
<img src="{% static plan.image %}" alt="Image">
|
||||
</figure>
|
||||
</div>
|
||||
<nav class="level is-mobile">
|
||||
<div class="level-left">
|
||||
{% if plan not in user_plans %}
|
||||
<a class="level-item" href="/order/{{ plan.name }}">
|
||||
<span class="icon is-small has-text-success">
|
||||
<i class="fas fa-plus" aria-hidden="true"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
|
||||
{% if plan in user_plans %}
|
||||
<a class="level-item" href="/cancel_subscription/{{ plan.name }}">
|
||||
<span class="icon is-small has-text-info">
|
||||
<i class="fas fa-cancel" aria-hidden="true"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
<div class="media-content">
|
||||
<div class="content">
|
||||
<p>
|
||||
<strong>{{ plan.name }}</strong> <small>£{{ plan.cost }}</small>
|
||||
{% if plan in user_plans %}
|
||||
<i class="fas fa-check" aria-hidden="true"></i>
|
||||
{% endif %}
|
||||
<br>
|
||||
{{ plan.description }}
|
||||
</p>
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<nav class="level is-mobile">
|
||||
<div class="level-left">
|
||||
{% if plan not in user_plans %}
|
||||
<a class="level-item" href="/order/{{ plan.name }}">
|
||||
<span class="icon is-small has-text-success">
|
||||
<i class="fas fa-plus" aria-hidden="true"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
|
||||
{% if plan in user_plans %}
|
||||
<a class="level-item" href="/cancel_subscription/{{ plan.name }}">
|
||||
<span class="icon is-small has-text-info">
|
||||
<i class="fas fa-cancel" aria-hidden="true"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endcache %}
|
||||
@ -0,0 +1,536 @@
|
||||
{% load django_tables2 %}
|
||||
{% load django_tables2_bulma_template %}
|
||||
{% load static %}
|
||||
{% load joinsep %}
|
||||
{% load urlsafe %}
|
||||
{% load pretty %}
|
||||
{% load splitstr %}
|
||||
{% load cache %}
|
||||
|
||||
{% cache 3600 results_table_full request.user.id table %}
|
||||
{% block table-wrapper %}
|
||||
<script src="{% static 'js/column-shifter.js' %}"></script>
|
||||
<div id="drilldown-table" class="column-shifter-container" style="position:relative; z-index:1;">
|
||||
{% block table %}
|
||||
<div class="nowrap-parent">
|
||||
<div class="nowrap-child">
|
||||
<div class="dropdown" id="dropdown">
|
||||
<div class="dropdown-trigger">
|
||||
<button id="dropdown-trigger" class="button dropdown-toggle" aria-haspopup="true" aria-controls="dropdown-menu">
|
||||
<span>Show/hide fields</span>
|
||||
<span class="icon is-small">
|
||||
<i class="fas fa-angle-down" aria-hidden="true"></i>
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="dropdown-menu" id="dropdown-menu" role="menu">
|
||||
<div class="dropdown-content" style="position:absolute; z-index:2;">
|
||||
{% for column in table.columns %}
|
||||
{% if column.name in show %}
|
||||
<a class="btn-shift-column dropdown-item"
|
||||
data-td-class="{{ column.name }}"
|
||||
data-state="on"
|
||||
{% if not forloop.last %} style="border-bottom:1px solid #ccc;" {%endif %}
|
||||
data-table-class-container="drilldown-table">
|
||||
<span class="check icon" data-tooltip="Visible" style="display:none;">
|
||||
<i class="fa-solid fa-check"></i>
|
||||
</span>
|
||||
<span class="uncheck icon" data-tooltip="Hidden" style="display:none;">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
{{ column.header }}
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="nowrap-child">
|
||||
<span id="loader" class="button is-light has-text-link is-loading">Static</span>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
var dropdown_button = document.getElementById("dropdown-trigger");
|
||||
var dropdown = document.getElementById("dropdown");
|
||||
dropdown_button.addEventListener('click', function(e) {
|
||||
// elements[i].preventDefault();
|
||||
dropdown.classList.toggle('is-active');
|
||||
});
|
||||
|
||||
</script>
|
||||
<div id="table-container" style="display:none;">
|
||||
<table {% render_attrs table.attrs class="table drilldown-results-table is-fullwidth" %}>
|
||||
{% block table.thead %}
|
||||
{% if table.show_header %}
|
||||
<thead {% render_attrs table.attrs.thead class="" %}>
|
||||
{% block table.thead.row %}
|
||||
<tr>
|
||||
{% for column in table.columns %}
|
||||
{% if column.name in show %}
|
||||
{% block table.thead.th %}
|
||||
<th class="orderable {{ column.name }}">
|
||||
<div class="nowrap-parent">
|
||||
{% if column.orderable %}
|
||||
<div class="nowrap-child">
|
||||
{% if column.is_ordered %}
|
||||
{% is_descending column.order_by as descending %}
|
||||
{% if descending %}
|
||||
<span class="icon" aria-hidden="true">{% block table.desc_icon %}<i class="fa-solid fa-sort-down"></i>{% endblock table.desc_icon %}</span>
|
||||
{% else %}
|
||||
<span class="icon" aria-hidden="true">{% block table.asc_icon %}<i class="fa-solid fa-sort-up"></i>{% endblock table.asc_icon %}</span>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<span class="icon" aria-hidden="true">{% block table.orderable_icon %}<i class="fa-solid fa-sort"></i>{% endblock table.orderable_icon %}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="nowrap-child">
|
||||
<a
|
||||
hx-get="search/partial/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#drilldown-table"
|
||||
hx-swap="outerHTML"
|
||||
hx-indicator="#spinner"
|
||||
style="cursor: pointer;">
|
||||
{{ column.header }}
|
||||
</a>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="nowrap-child">
|
||||
{{ column.header }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</th>
|
||||
{% endblock table.thead.th %}
|
||||
{% endif %}
|
||||
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endblock table.thead.row %}
|
||||
</thead>
|
||||
{% endif %}
|
||||
{% endblock table.thead %}
|
||||
{% block table.tbody %}
|
||||
<tbody {{ table.attrs.tbody.as_html }}>
|
||||
{% for row in table.paginated_rows %}
|
||||
{% block table.tbody.row %}
|
||||
{% if row.cells.type == 'control' %}
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>
|
||||
<span class="icon has-text-grey" data-tooltip="Hidden">
|
||||
<i class="fa-solid fa-file-slash"></i>
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
<p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
|
||||
</td>
|
||||
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr class="
|
||||
{% if row.cells.exemption == True %}has-background-grey-lighter
|
||||
{% elif cell == 'join' %}has-background-success-light
|
||||
{% elif cell == 'quit' %}has-background-danger-light
|
||||
{% elif cell == 'kick' %}has-background-danger-light
|
||||
{% elif cell == 'part' %}has-background-warning-light
|
||||
{% elif cell == 'mode' %}has-background-info-light
|
||||
{% endif %}">
|
||||
{% for column, cell in row.items %}
|
||||
{% if column.name in show %}
|
||||
{% block table.tbody.td %}
|
||||
{% if cell == '—' %}
|
||||
<td class="{{ column.name }}">
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-file-slash"></i>
|
||||
</span>
|
||||
</td>
|
||||
{% elif column.name == 'src' %}
|
||||
<td class="{{ column.name }}">
|
||||
<a
|
||||
class="has-text-grey"
|
||||
onclick="populateSearch('src', '{{ cell|escapejs }}')">
|
||||
{% if row.cells.src == 'irc' %}
|
||||
<span class="icon" data-tooltip="IRC">
|
||||
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
|
||||
</span>
|
||||
{% elif row.cells.src == 'dis' %}
|
||||
<span class="icon" data-tooltip="Discord">
|
||||
<i class="fa-brands fa-discord" aria-hidden="true"></i>
|
||||
</span>
|
||||
{% elif row.cells.src == '4ch' %}
|
||||
<span class="icon" data-tooltip="4chan">
|
||||
<i class="fa-solid fa-leaf" aria-hidden="true"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
</td>
|
||||
{% elif column.name == 'ts' %}
|
||||
<td class="{{ column.name }}">
|
||||
<p>{{ row.cells.date }}</p>
|
||||
<p>{{ row.cells.time }}</p>
|
||||
</td>
|
||||
{% elif column.name == 'match_ts' %}
|
||||
<td class="{{ column.name }}">
|
||||
{% with match_ts=cell|splitstr:'T' %}
|
||||
<p>{{ match_ts.0 }}</p>
|
||||
<p>{{ match_ts.1 }}</p>
|
||||
{% endwith %}
|
||||
</td>
|
||||
{% elif column.name == 'type' or column.name == 'mtype' %}
|
||||
<td class="{{ column.name }}">
|
||||
<a
|
||||
class="has-text-grey"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
|
||||
{% if cell == 'msg' %}
|
||||
<span class="icon" data-tooltip="Message">
|
||||
<i class="fa-solid fa-message"></i>
|
||||
</span>
|
||||
{% elif cell == 'join' %}
|
||||
<span class="icon" data-tooltip="Join">
|
||||
<i class="fa-solid fa-person-to-portal"></i>
|
||||
</span>
|
||||
{% elif cell == 'part' %}
|
||||
<span class="icon" data-tooltip="Part">
|
||||
<i class="fa-solid fa-person-from-portal"></i>
|
||||
</span>
|
||||
{% elif cell == 'quit' %}
|
||||
<span class="icon" data-tooltip="Quit">
|
||||
<i class="fa-solid fa-circle-xmark"></i>
|
||||
</span>
|
||||
{% elif cell == 'kick' %}
|
||||
<span class="icon" data-tooltip="Kick">
|
||||
<i class="fa-solid fa-user-slash"></i>
|
||||
</span>
|
||||
{% elif cell == 'nick' %}
|
||||
<span class="icon" data-tooltip="Nick">
|
||||
<i class="fa-solid fa-signature"></i>
|
||||
</span>
|
||||
{% elif cell == 'mode' %}
|
||||
<span class="icon" data-tooltip="Mode">
|
||||
<i class="fa-solid fa-gear"></i>
|
||||
</span>
|
||||
{% elif cell == 'action' %}
|
||||
<span class="icon" data-tooltip="Action">
|
||||
<i class="fa-solid fa-exclamation"></i>
|
||||
</span>
|
||||
{% elif cell == 'notice' %}
|
||||
<span class="icon" data-tooltip="Notice">
|
||||
<i class="fa-solid fa-message-code"></i>
|
||||
</span>
|
||||
{% elif cell == 'conn' %}
|
||||
<span class="icon" data-tooltip="Connection">
|
||||
<i class="fa-solid fa-cloud-exclamation"></i>
|
||||
</span>
|
||||
{% elif cell == 'znc' %}
|
||||
<span class="icon" data-tooltip="ZNC">
|
||||
<i class="fa-brands fa-unity"></i>
|
||||
</span>
|
||||
{% elif cell == 'query' %}
|
||||
<span class="icon" data-tooltip="Query">
|
||||
<i class="fa-solid fa-message"></i>
|
||||
</span>
|
||||
{% elif cell == 'highlight' %}
|
||||
<span class="icon" data-tooltip="Highlight">
|
||||
<i class="fa-solid fa-exclamation"></i>
|
||||
</span>
|
||||
{% elif cell == 'who' %}
|
||||
<span class="icon" data-tooltip="Who">
|
||||
<i class="fa-solid fa-passport"></i>
|
||||
</span>
|
||||
{% elif cell == 'topic' %}
|
||||
<span class="icon" data-tooltip="Topic">
|
||||
<i class="fa-solid fa-sign"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
{{ cell }}
|
||||
{% endif %}
|
||||
</a>
|
||||
</td>
|
||||
{% elif column.name == 'msg' %}
|
||||
<td class="{{ column.name }} wrap">
|
||||
<a
|
||||
class="has-text-grey is-underlined"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_context' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net|escapejs }}",
|
||||
"num": "{{ row.cells.num|escapejs }}",
|
||||
"source": "{{ row.cells.src|escapejs }}",
|
||||
"channel": "{{ row.cells.channel|escapejs }}",
|
||||
"time": "{{ row.cells.time|escapejs }}",
|
||||
"date": "{{ row.cells.date|escapejs }}",
|
||||
"index": "{% if row.cells.index != '—' %}{{row.cells.index}}{% else %}{{ params.index }}{% endif %}",
|
||||
"type": "{{ row.cells.type }}",
|
||||
"mtype": "{{ row.cells.mtype }}",
|
||||
"nick": "{{ row.cells.nick|escapejs }}",
|
||||
"dedup": "{{ params.dedup }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="click"
|
||||
href="/?modal=context&net={{row.cells.net|escapejs}}&num={{row.cells.num|escapejs}}&source={{row.cells.src|escapejs}}&channel={{row.cells.channel|urlsafe}}&time={{row.cells.time|escapejs}}&date={{row.cells.date|escapejs}}&index={{params.index}}&type={{row.cells.type}}&mtype={{row.cells.mtype}}&nick={{row.cells.mtype|escapejs}}">
|
||||
{{ row.cells.msg }}
|
||||
</a>
|
||||
</td>
|
||||
{% elif column.name == 'nick' %}
|
||||
<td class="{{ column.name }}">
|
||||
<div class="nowrap-parent">
|
||||
<div class="nowrap-child">
|
||||
{% if row.cells.online is True %}
|
||||
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% elif row.cells.online is False %}
|
||||
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<a class="nowrap-child has-text-grey" onclick="populateSearch('nick', '{{ cell|escapejs }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
<div class="nowrap-child">
|
||||
{% if row.cells.src == 'irc' %}
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown modal">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' type='window' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
|
||||
hx-target="#windows-here"
|
||||
hx-swap="afterend"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown window">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' type='widget' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
|
||||
hx-target="#widgets-here"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown widget">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if row.cells.num_chans != '—' %}
|
||||
<div class="nowrap-child">
|
||||
<span class="tag">
|
||||
{{ row.cells.num_chans }}
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
{% elif column.name == 'channel' %}
|
||||
<td class="{{ column.name }}">
|
||||
{% if cell != '—' %}
|
||||
<div class="nowrap-parent">
|
||||
<a
|
||||
class="nowrap-child has-text-grey"
|
||||
onclick="populateSearch('channel', '{{ cell|escapejs }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
{% if row.cells.num_users != '—' %}
|
||||
<div class="nowrap-child">
|
||||
<span class="tag">
|
||||
{{ row.cells.num_users }}
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
{{ cell }}
|
||||
{% endif %}
|
||||
</td>
|
||||
{% elif cell is True or cell is False %}
|
||||
<td class="{{ column.name }}">
|
||||
{% if cell is True %}
|
||||
<span class="icon has-text-success">
|
||||
<i class="fa-solid fa-check"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
{% elif column.name == "tokens" %}
|
||||
<td class="{{ column.name }}">
|
||||
<div class="tags">
|
||||
{% for word in cell %}
|
||||
<a
|
||||
class="tag"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ word }}')">
|
||||
{{ word }}
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
{% elif column.name == "meta" %}
|
||||
<td class="{{ column.name }}">
|
||||
<pre class="small-field" style="cursor: pointer;">{{ cell|pretty }}</pre>
|
||||
</td>
|
||||
{% elif 'id' in column.name and column.name != "ident" %}
|
||||
<td class="{{ column.name }}">
|
||||
<div class="buttons">
|
||||
<div class="nowrap-parent">
|
||||
<!-- <input class="input" type="text" value="{{ cell }}" style="width: 50px;" readonly> -->
|
||||
<a
|
||||
class="has-text-grey button nowrap-child"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
|
||||
<span class="icon" data-tooltip="Populate {{ cell }}">
|
||||
<i class="fa-solid fa-arrow-left-long-to-line" aria-hidden="true"></i>
|
||||
</span>
|
||||
</a>
|
||||
<a
|
||||
class="has-text-grey button nowrap-child"
|
||||
onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ cell|escapejs }}');">
|
||||
<span class="icon" data-tooltip="Copy to clipboard">
|
||||
<i class="fa-solid fa-copy" aria-hidden="true"></i>
|
||||
</span>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
{% else %}
|
||||
<td class="{{ column.name }}">
|
||||
<a
|
||||
class="has-text-grey"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
</td>
|
||||
{% endif %}
|
||||
{% endblock table.tbody.td %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endblock table.tbody.row %}
|
||||
{% empty %}
|
||||
{% if table.empty_text %}
|
||||
{% block table.tbody.empty_text %}
|
||||
<tr><td class="{{ column.name }}" colspan="{{ table.columns|length }}">{{ table.empty_text }}</td></tr>
|
||||
{% endblock table.tbody.empty_text %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
{% endblock table.tbody %}
|
||||
{% block table.tfoot %}
|
||||
{% if table.has_footer %}
|
||||
<tfoot {{ table.attrs.tfoot.as_html }}>
|
||||
{% block table.tfoot.row %}
|
||||
<tr>
|
||||
{% for column in table.columns %}
|
||||
{% block table.tfoot.td %}
|
||||
<td class="{{ column.name }}" {{ column.attrs.tf.as_html }}>{{ column.footer }}</td>
|
||||
{% endblock table.tfoot.td %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endblock table.tfoot.row %}
|
||||
</tfoot>
|
||||
{% endif %}
|
||||
{% endblock table.tfoot %}
|
||||
</table>
|
||||
</div>
|
||||
{% endblock table %}
|
||||
{% block pagination %}
|
||||
{% if table.page and table.paginator.num_pages > 1 %}
|
||||
<nav class="pagination is-justify-content-flex-end" role="navigation" aria-label="pagination">
|
||||
{% block pagination.previous %}
|
||||
<a
|
||||
class="pagination-previous is-flex-grow-0 {% if not table.page.has_previous %}is-hidden-mobile{% endif %}"
|
||||
{% if table.page.has_previous %}
|
||||
hx-get="search/partial/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#drilldown-table"
|
||||
hx-swap="outerHTML"
|
||||
hx-indicator="#spinner"
|
||||
{% else %}
|
||||
href="#"
|
||||
disabled
|
||||
{% endif %}
|
||||
style="order:1;">
|
||||
{% block pagination.previous.text %}
|
||||
<span aria-hidden="true">«</span>
|
||||
{% endblock pagination.previous.text %}
|
||||
</a>
|
||||
{% endblock pagination.previous %}
|
||||
{% block pagination.next %}
|
||||
<a
|
||||
class="pagination-next is-flex-grow-0 {% if not table.page.has_next %}is-hidden-mobile{% endif %}"
|
||||
{% if table.page.has_next %}
|
||||
hx-get="search/partial/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#drilldown-table"
|
||||
hx-swap="outerHTML"
|
||||
hx-indicator="#spinner"
|
||||
{% else %}
|
||||
href="#"
|
||||
disabled
|
||||
{% endif %}
|
||||
style="order:3;"
|
||||
>
|
||||
{% block pagination.next.text %}
|
||||
<span aria-hidden="true">»</span>
|
||||
{% endblock pagination.next.text %}
|
||||
</a>
|
||||
{% endblock pagination.next %}
|
||||
{% if table.page.has_previous or table.page.has_next %}
|
||||
{% block pagination.range %}
|
||||
<ul class="pagination-list is-flex-grow-0" style="order:2;">
|
||||
{% for p in table.page|table_page_range:table.paginator %}
|
||||
<li>
|
||||
<a
|
||||
class="pagination-link {% if p == table.page.number %}is-current{% endif %}"
|
||||
aria-label="Page {{ p }}" block
|
||||
{% if p == table.page.number %}aria-current="page"{% endif %}
|
||||
{% if p == table.page.number %}
|
||||
href="#"
|
||||
{% else %}
|
||||
hx-get="search/partial/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#drilldown-table"
|
||||
hx-swap="outerHTML"
|
||||
hx-indicator="#spinner"
|
||||
{% endif %}
|
||||
>
|
||||
{% if p == '...' %}
|
||||
<span class="pagination-ellipsis">…</span>
|
||||
{% else %}
|
||||
{{ p }}
|
||||
{% endif %}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endblock pagination.range %}
|
||||
{% endif %}
|
||||
</nav>
|
||||
{% endif %}
|
||||
{% endblock pagination %}
|
||||
</div>
|
||||
{% endblock table-wrapper %}
|
||||
{% endcache %}
|
||||
@ -0,0 +1,109 @@
|
||||
{% load cache %}
|
||||
{% load cachalot cache %}
|
||||
{% get_last_invalidation 'core.NotificationRule' as last %}
|
||||
{% include 'mixins/partials/notify.html' %}
|
||||
{% cache 600 objects_rules request.user.id object_list last %}
|
||||
<table
|
||||
class="table is-fullwidth is-hoverable"
|
||||
hx-target="#{{ context_object_name }}-table"
|
||||
id="{{ context_object_name }}-table"
|
||||
hx-swap="outerHTML"
|
||||
hx-trigger="{{ context_object_name_singular }}Event from:body"
|
||||
hx-get="{{ list_url }}">
|
||||
<thead>
|
||||
<th>id</th>
|
||||
<th>user</th>
|
||||
<th>name</th>
|
||||
<th>interval</th>
|
||||
<th>window</th>
|
||||
<th>priority</th>
|
||||
<th>topic</th>
|
||||
<th>enabled</th>
|
||||
<th>ingest</th>
|
||||
<th>data length</th>
|
||||
<th>match</th>
|
||||
<th>actions</th>
|
||||
</thead>
|
||||
{% for item in object_list %}
|
||||
<tr>
|
||||
<td><a href="/?query=*&source=all&rule={{ item.id }}">{{ item.id }}</a></td>
|
||||
<td>{{ item.user }}</td>
|
||||
<td>{{ item.name }}</td>
|
||||
<td>{{ item.interval }}s</td>
|
||||
<td>{{ item.window|default_if_none:"—" }}</td>
|
||||
<td>{{ item.priority }}</td>
|
||||
<td>{{ item.topic|default_if_none:"—" }}</td>
|
||||
<td>
|
||||
{% if item.enabled %}
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-check"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if item.ingest %}
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-check"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ item.data|length }}</td>
|
||||
<td>{{ item.matches }}</td>
|
||||
<td>
|
||||
<div class="buttons">
|
||||
<button
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-get="{% url 'rule_update' type=type pk=item.id %}"
|
||||
hx-trigger="click"
|
||||
hx-target="#{{ type }}s-here"
|
||||
hx-swap="innerHTML"
|
||||
class="button">
|
||||
<span class="icon-text">
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-pencil"></i>
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
<button
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-delete="{% url 'rule_delete' type=type pk=item.id %}"
|
||||
hx-trigger="click"
|
||||
hx-target="#modals-here"
|
||||
hx-swap="innerHTML"
|
||||
hx-confirm="Are you sure you wish to delete {{ item.name }}?"
|
||||
class="button">
|
||||
<span class="icon-text">
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
<button
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'rule_clear' type=type pk=item.id %}"
|
||||
hx-trigger="click"
|
||||
hx-target="#modals-here"
|
||||
hx-swap="innerHTML"
|
||||
hx-confirm="Are you sure you wish to clear matches for {{ item.name }}?"
|
||||
class="button">
|
||||
<span class="icon-text">
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-arrow-rotate-right"></i>
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
</table>
|
||||
{% endcache %}
|
||||
@ -1,163 +0,0 @@
|
||||
{% extends "base.html" %}
|
||||
{% load static %}
|
||||
{% load joinsep %}
|
||||
{% block outer_content %}
|
||||
{% if params.modal == 'context' %}
|
||||
<div
|
||||
style="display: none;"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_context' %}"
|
||||
hx-vals='{"net": "{{ params.net|escapejs }}",
|
||||
"num": "{{ params.num|escapejs }}",
|
||||
"source": "{{ params.source|escapejs }}",
|
||||
"channel": "{{ params.channel|escapejs }}",
|
||||
"time": "{{ params.time|escapejs }}",
|
||||
"date": "{{ params.date|escapejs }}",
|
||||
"index": "{{ params.index }}",
|
||||
"type": "{{ params.type|escapejs }}",
|
||||
"mtype": "{{ params.mtype|escapejs }}",
|
||||
"nick": "{{ params.nick|escapejs }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="load">
|
||||
</div>
|
||||
{% endif %}
|
||||
<script src="{% static 'js/chart.js' %}"></script>
|
||||
<script src="{% static 'tabs.js' %}"></script>
|
||||
<script>
|
||||
function setupTags() {
|
||||
var inputTags = document.getElementById('tags');
|
||||
new BulmaTagsInput(inputTags);
|
||||
|
||||
inputTags.BulmaTagsInput().on('before.add', function(item) {
|
||||
if (item.includes(": ")) {
|
||||
var spl = item.split(": ");
|
||||
} else {
|
||||
var spl = item.split(":");
|
||||
}
|
||||
var field = spl[0];
|
||||
try {
|
||||
var value = JSON.parse(spl[1]);
|
||||
} catch {
|
||||
var value = spl[1];
|
||||
}
|
||||
return `${field}: ${value}`;
|
||||
});
|
||||
inputTags.BulmaTagsInput().on('after.remove', function(item) {
|
||||
var spl = item.split(": ");
|
||||
var field = spl[0];
|
||||
var value = spl[1].trim();
|
||||
});
|
||||
}
|
||||
function populateSearch(field, value) {
|
||||
var inputTags = document.getElementById('tags');
|
||||
inputTags.BulmaTagsInput().add(field+": "+value);
|
||||
//htmx.trigger("#search", "click");
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="grid-stack" id="grid-stack-main">
|
||||
<div class="grid-stack-item" gs-w="7" gs-h="10" gs-y="0" gs-x="1">
|
||||
<div class="grid-stack-item-content">
|
||||
<nav class="panel">
|
||||
<p class="panel-heading" style="padding: .2em; line-height: .5em;">
|
||||
<i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
|
||||
Search
|
||||
</p>
|
||||
<article class="panel-block is-active">
|
||||
{% include 'ui/drilldown/search_partial.html' %}
|
||||
</article>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var grid = GridStack.init({
|
||||
cellHeight: 20,
|
||||
cellWidth: 50,
|
||||
cellHeightUnit: 'px',
|
||||
auto: true,
|
||||
float: true,
|
||||
draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
|
||||
removable: false,
|
||||
animate: true,
|
||||
});
|
||||
// GridStack.init();
|
||||
setupTags();
|
||||
|
||||
// a widget is ready to be loaded
|
||||
document.addEventListener('load-widget', function(event) {
|
||||
let container = htmx.find('#drilldown-widget');
|
||||
// get the scripts, they won't be run on the new element so we need to eval them
|
||||
var scripts = htmx.findAll(container, "script");
|
||||
let widgetelement = container.firstElementChild.cloneNode(true);
|
||||
|
||||
// check if there's an existing element like the one we want to swap
|
||||
let grid_element = htmx.find('#grid-stack-main');
|
||||
let existing_widget = htmx.find(grid_element, '#drilldown-widget-results');
|
||||
|
||||
// get the size and position attributes
|
||||
if (existing_widget) {
|
||||
let attrs = existing_widget.getAttributeNames();
|
||||
for (let i = 0, len = attrs.length; i < len; i++) {
|
||||
if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
|
||||
widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clear the queue element
|
||||
container.outerHTML = "";
|
||||
|
||||
// temporary workaround, other widgets can be duplicated, but not results
|
||||
if (widgetelement.id == 'drilldown-widget-results') {
|
||||
grid.removeWidget("drilldown-widget-{{ unique }}");
|
||||
}
|
||||
|
||||
grid.addWidget(widgetelement);
|
||||
// re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
|
||||
htmx.process(widgetelement);
|
||||
|
||||
// update size when the widget is loaded
|
||||
document.addEventListener('load-widget-results', function(evt) {
|
||||
var added_widget = htmx.find(grid_element, '#drilldown-widget-results');
|
||||
console.log(added_widget);
|
||||
var itemContent = htmx.find(added_widget, ".control");
|
||||
console.log(itemContent);
|
||||
var scrollheight = itemContent.scrollHeight+80;
|
||||
var verticalmargin = 0;
|
||||
var cellheight = grid.opts.cellHeight;
|
||||
var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
|
||||
var opts = {
|
||||
h: height,
|
||||
}
|
||||
grid.update(
|
||||
added_widget,
|
||||
opts
|
||||
);
|
||||
});
|
||||
|
||||
// run the JS scripts inside the added element again
|
||||
// for instance, this will fix the dropdown
|
||||
for (var i = 0; i < scripts.length; i++) {
|
||||
eval(scripts[i].innerHTML);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<div id="modals-here">
|
||||
</div>
|
||||
<div id="items-here">
|
||||
</div>
|
||||
<div id="widgets-here" style="display: none;">
|
||||
</div>
|
||||
<div id="results" style="display: none;">
|
||||
{% if table %}
|
||||
{% include 'widgets/table_results.html' %}
|
||||
{% endif %}
|
||||
</div>
|
||||
<script>
|
||||
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
@ -1,508 +0,0 @@
|
||||
{% load django_tables2 %}
|
||||
{% load django_tables2_bulma_template %}
|
||||
{% load static %}
|
||||
{% load joinsep %}
|
||||
{% load urlsafe %}
|
||||
{% block table-wrapper %}
|
||||
<div id="drilldown-table" class="column-shifter-container" style="position:relative; z-index:1;">
|
||||
{% block table %}
|
||||
<div class="nowrap-parent">
|
||||
<div class="nowrap-child">
|
||||
<div class="dropdown" id="dropdown">
|
||||
<div class="dropdown-trigger">
|
||||
<button id="dropdown-trigger" class="button dropdown-toggle" aria-haspopup="true" aria-controls="dropdown-menu">
|
||||
<span>Show/hide fields</span>
|
||||
<span class="icon is-small">
|
||||
<i class="fas fa-angle-down" aria-hidden="true"></i>
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="dropdown-menu" id="dropdown-menu" role="menu">
|
||||
<div class="dropdown-content" style="position:absolute; z-index:2;">
|
||||
{% for column in table.columns %}
|
||||
{% if column.name in show %}
|
||||
<a class="btn-shift-column dropdown-item"
|
||||
data-td-class="{{ column.name }}"
|
||||
data-state="on"
|
||||
{% if not forloop.last %} style="border-bottom:1px solid #ccc;" {%endif %}
|
||||
data-table-class-container="drilldown-table">
|
||||
<span class="check icon" data-tooltip="Visible" style="display:none;">
|
||||
<i class="fa-solid fa-check"></i>
|
||||
</span>
|
||||
<span class="uncheck icon" data-tooltip="Hidden" style="display:none;">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
{{ column.header }}
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="nowrap-child">
|
||||
<span id="loader" class="button is-light has-text-link is-loading">Static</span>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
var dropdown_button = document.getElementById("dropdown-trigger");
|
||||
var dropdown = document.getElementById("dropdown");
|
||||
dropdown_button.addEventListener('click', function(e) {
|
||||
// elements[i].preventDefault();
|
||||
dropdown.classList.toggle('is-active');
|
||||
});
|
||||
|
||||
</script>
|
||||
<div id="table-container" style="display:none;">
|
||||
<table {% render_attrs table.attrs class="table drilldown-results-table is-fullwidth" %}>
|
||||
{% block table.thead %}
|
||||
{% if table.show_header %}
|
||||
<thead {% render_attrs table.attrs.thead class="" %}>
|
||||
{% block table.thead.row %}
|
||||
<tr>
|
||||
{% for column in table.columns %}
|
||||
{% if column.name in show %}
|
||||
{% block table.thead.th %}
|
||||
<th class="orderable {{ column.name }}">
|
||||
<div class="nowrap-parent">
|
||||
{% if column.orderable %}
|
||||
<div class="nowrap-child">
|
||||
{% if column.is_ordered %}
|
||||
{% is_descending column.order_by as descending %}
|
||||
{% if descending %}
|
||||
<span class="icon" aria-hidden="true">{% block table.desc_icon %}<i class="fa-solid fa-sort-down"></i>{% endblock table.desc_icon %}</span>
|
||||
{% else %}
|
||||
<span class="icon" aria-hidden="true">{% block table.asc_icon %}<i class="fa-solid fa-sort-up"></i>{% endblock table.asc_icon %}</span>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<span class="icon" aria-hidden="true">{% block table.orderable_icon %}<i class="fa-solid fa-sort"></i>{% endblock table.orderable_icon %}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="nowrap-child">
|
||||
<a
|
||||
hx-get="search/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#results"
|
||||
hx-swap="innerHTML"
|
||||
hx-indicator="#spinner"
|
||||
style="cursor: pointer;">
|
||||
{{ column.header }}
|
||||
</a>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="nowrap-child">
|
||||
{{ column.header }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</th>
|
||||
{% endblock table.thead.th %}
|
||||
{% endif %}
|
||||
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endblock table.thead.row %}
|
||||
</thead>
|
||||
{% endif %}
|
||||
{% endblock table.thead %}
|
||||
{% block table.tbody %}
|
||||
<tbody {{ table.attrs.tbody.as_html }}>
|
||||
{% for row in table.paginated_rows %}
|
||||
{% block table.tbody.row %}
|
||||
{% if row.cells.type == 'control' %}
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>
|
||||
<span class="icon has-text-grey" data-tooltip="Hidden">
|
||||
<i class="fa-solid fa-file-slash"></i>
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
<p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
|
||||
</td>
|
||||
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr class="
|
||||
{% if row.cells.exemption == True %}has-background-grey-lighter
|
||||
{% elif cell == 'join' %}has-background-success-light
|
||||
{% elif cell == 'quit' %}has-background-danger-light
|
||||
{% elif cell == 'kick' %}has-background-danger-light
|
||||
{% elif cell == 'part' %}has-background-warning-light
|
||||
{% elif cell == 'mode' %}has-background-info-light
|
||||
{% endif %}">
|
||||
{% for column, cell in row.items %}
|
||||
{% if column.name in show %}
|
||||
{% block table.tbody.td %}
|
||||
{% if cell == '—' %}
|
||||
<td class="{{ column.name }}">
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-file-slash"></i>
|
||||
</span>
|
||||
</td>
|
||||
{% elif column.name == 'tokens' %}
|
||||
<td class="{{ column.name }} wrap" style="max-width: 10em">
|
||||
{{ cell|joinsep:',' }}
|
||||
</td>
|
||||
{% elif column.name == 'src' %}
|
||||
<td class="{{ column.name }}">
|
||||
<a
|
||||
class="has-text-link is-underlined"
|
||||
onclick="populateSearch('src', '{{ cell|escapejs }}')">
|
||||
{% if row.cells.src == 'irc' %}
|
||||
<span class="icon" data-tooltip="IRC">
|
||||
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
|
||||
</span>
|
||||
{% elif row.cells.src == 'dis' %}
|
||||
<span class="icon" data-tooltip="Discord">
|
||||
<i class="fa-brands fa-discord" aria-hidden="true"></i>
|
||||
</span>
|
||||
{% elif row.cells.src == '4ch' %}
|
||||
<span class="icon" data-tooltip="4chan">
|
||||
<i class="fa-solid fa-leaf" aria-hidden="true"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
</td>
|
||||
{% elif column.name == 'ts' %}
|
||||
<td class="{{ column.name }}">
|
||||
<p>{{ row.cells.date }}</p>
|
||||
<p>{{ row.cells.time }}</p>
|
||||
</td>
|
||||
{% elif column.name == 'type' or column.name == 'mtype' %}
|
||||
<td class="{{ column.name }}">
|
||||
<a
|
||||
class="has-text-link is-underlined"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
|
||||
{% if cell == 'msg' %}
|
||||
<span class="icon" data-tooltip="Message">
|
||||
<i class="fa-solid fa-message"></i>
|
||||
</span>
|
||||
{% elif cell == 'join' %}
|
||||
<span class="icon" data-tooltip="Join">
|
||||
<i class="fa-solid fa-person-to-portal"></i>
|
||||
</span>
|
||||
{% elif cell == 'part' %}
|
||||
<span class="icon" data-tooltip="Part">
|
||||
<i class="fa-solid fa-person-from-portal"></i>
|
||||
</span>
|
||||
{% elif cell == 'quit' %}
|
||||
<span class="icon" data-tooltip="Quit">
|
||||
<i class="fa-solid fa-circle-xmark"></i>
|
||||
</span>
|
||||
{% elif cell == 'kick' %}
|
||||
<span class="icon" data-tooltip="Kick">
|
||||
<i class="fa-solid fa-user-slash"></i>
|
||||
</span>
|
||||
{% elif cell == 'nick' %}
|
||||
<span class="icon" data-tooltip="Nick">
|
||||
<i class="fa-solid fa-signature"></i>
|
||||
</span>
|
||||
{% elif cell == 'mode' %}
|
||||
<span class="icon" data-tooltip="Mode">
|
||||
<i class="fa-solid fa-gear"></i>
|
||||
</span>
|
||||
{% elif cell == 'action' %}
|
||||
<span class="icon" data-tooltip="Action">
|
||||
<i class="fa-solid fa-exclamation"></i>
|
||||
</span>
|
||||
{% elif cell == 'notice' %}
|
||||
<span class="icon" data-tooltip="Notice">
|
||||
<i class="fa-solid fa-message-code"></i>
|
||||
</span>
|
||||
{% elif cell == 'conn' %}
|
||||
<span class="icon" data-tooltip="Connection">
|
||||
<i class="fa-solid fa-cloud-exclamation"></i>
|
||||
</span>
|
||||
{% elif cell == 'znc' %}
|
||||
<span class="icon" data-tooltip="ZNC">
|
||||
<i class="fa-brands fa-unity"></i>
|
||||
</span>
|
||||
{% elif cell == 'query' %}
|
||||
<span class="icon" data-tooltip="Query">
|
||||
<i class="fa-solid fa-message"></i>
|
||||
</span>
|
||||
{% elif cell == 'highlight' %}
|
||||
<span class="icon" data-tooltip="Highlight">
|
||||
<i class="fa-solid fa-exclamation"></i>
|
||||
</span>
|
||||
{% elif cell == 'who' %}
|
||||
<span class="icon" data-tooltip="Who">
|
||||
<i class="fa-solid fa-passport"></i>
|
||||
</span>
|
||||
{% elif cell == 'topic' %}
|
||||
<span class="icon" data-tooltip="Topic">
|
||||
<i class="fa-solid fa-sign"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
{{ cell }}
|
||||
{% endif %}
|
||||
</a>
|
||||
</td>
|
||||
{% elif column.name == 'msg' %}
|
||||
<td class="{{ column.name }} wrap">
|
||||
<a
|
||||
class="has-text-grey is-underlined"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_context' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net|escapejs }}",
|
||||
"num": "{{ row.cells.num|escapejs }}",
|
||||
"source": "{{ row.cells.src|escapejs }}",
|
||||
"channel": "{{ row.cells.channel|escapejs }}",
|
||||
"time": "{{ row.cells.time|escapejs }}",
|
||||
"date": "{{ row.cells.date|escapejs }}",
|
||||
"index": "{{ params.index }}",
|
||||
"type": "{{ row.cells.type }}",
|
||||
"mtype": "{{ row.cells.mtype }}",
|
||||
"nick": "{{ row.cells.nick|escapejs }}",
|
||||
"dedup": "{{ params.dedup }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="click"
|
||||
href="/?modal=context&net={{row.cells.net|escapejs}}&num={{row.cells.num|escapejs}}&source={{row.cells.src|escapejs}}&channel={{row.cells.channel|urlsafe}}&time={{row.cells.time|escapejs}}&date={{row.cells.date|escapejs}}&index={{params.index}}&type={{row.cells.type}}&mtype={{row.cells.mtype}}&nick={{row.cells.mtype|escapejs}}">
|
||||
{{ row.cells.msg }}
|
||||
</a>
|
||||
</td>
|
||||
{% elif column.name == 'nick' %}
|
||||
<td class="{{ column.name }}">
|
||||
<div class="nowrap-parent">
|
||||
<div class="nowrap-child">
|
||||
{% if row.cells.online is True %}
|
||||
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% elif row.cells.online is False %}
|
||||
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ cell|escapejs }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
<div class="nowrap-child">
|
||||
{% if row.cells.src == 'irc' %}
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown modal">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' type='window' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
|
||||
hx-target="#items-here"
|
||||
hx-swap="afterend"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown window">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' type='widget' %}"
|
||||
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
|
||||
hx-target="#widgets-here"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown widget">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if row.cells.num_chans != '—' %}
|
||||
<div class="nowrap-child">
|
||||
<span class="tag">
|
||||
{{ row.cells.num_chans }}
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
{% elif column.name == 'channel' %}
|
||||
<td class="{{ column.name }}">
|
||||
{% if cell != '—' %}
|
||||
<div class="nowrap-parent">
|
||||
<a
|
||||
class="nowrap-child has-text-link is-underlined"
|
||||
onclick="populateSearch('channel', '{{ cell|escapejs }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
{% if row.cells.num_users != '—' %}
|
||||
<div class="nowrap-child">
|
||||
<span class="tag">
|
||||
{{ row.cells.num_users }}
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
{{ cell }}
|
||||
{% endif %}
|
||||
</td>
|
||||
{% elif cell is True or cell is False %}
|
||||
<td class="{{ column.name }}">
|
||||
{% if cell is True %}
|
||||
<span class="icon has-text-success">
|
||||
<i class="fa-solid fa-check"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
{% elif column.name|slice:":6" == "words_" %}
|
||||
<td class="{{ column.name }}">
|
||||
{% if cell.0.1|length == 0 %}
|
||||
<a
|
||||
class="tag is-info"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ cell }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
{% else %}
|
||||
<div class="tags">
|
||||
{% for word in cell %}
|
||||
<a
|
||||
class="tag is-info"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ word }}')">
|
||||
{{ word }}
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
{% else %}
|
||||
<td class="{{ column.name }}">
|
||||
<a
|
||||
class="has-text-link is-underlined"
|
||||
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
|
||||
{{ cell }}
|
||||
</a>
|
||||
</td>
|
||||
{% endif %}
|
||||
{% endblock table.tbody.td %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endblock table.tbody.row %}
|
||||
{% empty %}
|
||||
{% if table.empty_text %}
|
||||
{% block table.tbody.empty_text %}
|
||||
<tr><td class="{{ column.name }}" colspan="{{ table.columns|length }}">{{ table.empty_text }}</td></tr>
|
||||
{% endblock table.tbody.empty_text %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
{% endblock table.tbody %}
|
||||
{% block table.tfoot %}
|
||||
{% if table.has_footer %}
|
||||
<tfoot {{ table.attrs.tfoot.as_html }}>
|
||||
{% block table.tfoot.row %}
|
||||
<tr>
|
||||
{% for column in table.columns %}
|
||||
{% block table.tfoot.td %}
|
||||
<td class="{{ column.name }}" {{ column.attrs.tf.as_html }}>{{ column.footer }}</td>
|
||||
{% endblock table.tfoot.td %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endblock table.tfoot.row %}
|
||||
</tfoot>
|
||||
{% endif %}
|
||||
{% endblock table.tfoot %}
|
||||
</table>
|
||||
</div>
|
||||
{% endblock table %}
|
||||
{% block pagination %}
|
||||
{% if table.page and table.paginator.num_pages > 1 %}
|
||||
<nav class="pagination is-justify-content-flex-end" role="navigation" aria-label="pagination">
|
||||
{% block pagination.previous %}
|
||||
<a
|
||||
class="pagination-previous is-flex-grow-0 {% if not table.page.has_previous %}is-hidden-mobile{% endif %}"
|
||||
{% if table.page.has_previous %}
|
||||
hx-get="search/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#results"
|
||||
hx-swap="innerHTML"
|
||||
hx-indicator="#spinner"
|
||||
{% else %}
|
||||
href="#"
|
||||
disabled
|
||||
{% endif %}
|
||||
style="order:1;">
|
||||
{% block pagination.previous.text %}
|
||||
<span aria-hidden="true">«</span>
|
||||
{% endblock pagination.previous.text %}
|
||||
</a>
|
||||
{% endblock pagination.previous %}
|
||||
{% block pagination.next %}
|
||||
<a
|
||||
class="pagination-next is-flex-grow-0 {% if not table.page.has_next %}is-hidden-mobile{% endif %}"
|
||||
{% if table.page.has_next %}
|
||||
hx-get="search/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#results"
|
||||
hx-swap="innerHTML"
|
||||
hx-indicator="#spinner"
|
||||
{% else %}
|
||||
href="#"
|
||||
disabled
|
||||
{% endif %}
|
||||
style="order:3;"
|
||||
>
|
||||
{% block pagination.next.text %}
|
||||
<span aria-hidden="true">»</span>
|
||||
{% endblock pagination.next.text %}
|
||||
</a>
|
||||
{% endblock pagination.next %}
|
||||
{% if table.page.has_previous or table.page.has_next %}
|
||||
{% block pagination.range %}
|
||||
<ul class="pagination-list is-flex-grow-0" style="order:2;">
|
||||
{% for p in table.page|table_page_range:table.paginator %}
|
||||
<li>
|
||||
<a
|
||||
class="pagination-link {% if p == table.page.number %}is-current{% endif %}"
|
||||
aria-label="Page {{ p }}" block
|
||||
{% if p == table.page.number %}aria-current="page"{% endif %}
|
||||
{% if p == table.page.number %}
|
||||
href="#"
|
||||
{% else %}
|
||||
hx-get="search/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#results"
|
||||
hx-swap="innerHTML"
|
||||
hx-indicator="#spinner"
|
||||
{% endif %}
|
||||
>
|
||||
{% if p == '...' %}
|
||||
<span class="pagination-ellipsis">…</span>
|
||||
{% else %}
|
||||
{{ p }}
|
||||
{% endif %}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endblock pagination.range %}
|
||||
{% endif %}
|
||||
</nav>
|
||||
{% endif %}
|
||||
{% endblock pagination %}
|
||||
</div>
|
||||
{% endblock table-wrapper %}
|
||||
@ -1,122 +0,0 @@
|
||||
{% load index %}
|
||||
{% load static %}
|
||||
|
||||
<script src="{% static 'modal.js' %}"></script>
|
||||
<script>
|
||||
document.addEventListener("restore-modal-scroll", function(event) {
|
||||
var modalContent = document.getElementsByClassName("modal-content")[0];
|
||||
var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
|
||||
var scrollpos = localStorage.getItem('scrollpos_modal_content');
|
||||
if (scrollpos == 'BOTTOM') {
|
||||
modalContent.scrollTop = maxScroll;
|
||||
} else if (scrollpos) {
|
||||
modalContent.scrollTop = scrollpos;
|
||||
};
|
||||
});
|
||||
|
||||
document.addEventListener("htmx:beforeSwap", function(event) {
|
||||
var modalContent = document.getElementsByClassName("modal-content")[0];
|
||||
var scrollpos = modalContent.scrollTop;
|
||||
if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
|
||||
localStorage.setItem('scrollpos_modal_content', 'BOTTOM');
|
||||
} else {
|
||||
localStorage.setItem('scrollpos_modal_content', scrollpos);
|
||||
}
|
||||
|
||||
});
|
||||
</script>
|
||||
<style>
|
||||
#tab-content-{{ unique }} div {
|
||||
display: none;
|
||||
}
|
||||
|
||||
#tab-content-{{ unique }} div.is-active {
|
||||
display: block;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div id="modal" class="modal is-active is-clipped">
|
||||
<div class="modal-background"></div>
|
||||
<div class="modal-content">
|
||||
<div class="box">
|
||||
{% include 'partials/notify.html' %}
|
||||
<div class="tabs is-toggle is-fullwidth is-info" id="tabs-{{ unique }}">
|
||||
<ul>
|
||||
<li class="is-active" data-tab="1">
|
||||
<a>
|
||||
<span class="icon is-small"><i class="fa-solid fa-message-arrow-down"></i></span>
|
||||
<span>Scrollback</span>
|
||||
</a>
|
||||
</li>
|
||||
<li data-tab="2">
|
||||
<a>
|
||||
<span class="icon is-small"><i class="fa-solid fa-messages"></i></span>
|
||||
<span>Context</span>
|
||||
</a>
|
||||
</li>
|
||||
<li data-tab="3">
|
||||
<a>
|
||||
<span class="icon is-small"><i class="fa-solid fa-message"></i></span>
|
||||
<span>Message</span>
|
||||
</a>
|
||||
</li>
|
||||
<li data-tab="4">
|
||||
<a>
|
||||
<span class="icon is-small"><i class="fa-solid fa-asterisk"></i></span>
|
||||
<span>Info</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div id="tab-content-{{ unique }}">
|
||||
<div class="is-active" data-content="1">
|
||||
<h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }}</h4>
|
||||
{% include 'modals/context_table.html' %}
|
||||
{% if user.is_superuser and src == 'irc' %}
|
||||
<form method="PUT">
|
||||
<article class="field has-addons">
|
||||
<article class="control is-expanded has-icons-left">
|
||||
<input id="context-input" name="msg" class="input" type="text" placeholder="Type your message here">
|
||||
<span class="icon is-small is-left">
|
||||
<i class="fas fa-magnifying-glass"></i>
|
||||
</span>
|
||||
</article>
|
||||
<article class="control">
|
||||
<article class="field">
|
||||
<button
|
||||
id="search"
|
||||
class="button is-info is-fullwidth"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-put="{% url 'threshold_irc_msg' net num %}"
|
||||
hx-vals='{"channel": "{{ channel }}", "nick": "{{ nick }}"}'
|
||||
hx-trigger="click"
|
||||
hx-target="#context-input"
|
||||
hx-swap="outerHTML">
|
||||
Send
|
||||
</button>
|
||||
</article>
|
||||
</article>
|
||||
</article>
|
||||
</form>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
<div data-content="2">
|
||||
<h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
|
||||
Context
|
||||
</div>
|
||||
<div data-content="3">
|
||||
<h4 class="subtitle is-4">Message details</h4>
|
||||
Message deetails
|
||||
</div>
|
||||
<div data-content="4">
|
||||
<h4 class="subtitle is-4">Information about {{ channel }} on {{ net }}{{ num }}</h4>
|
||||
info
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>initTabs("{{ unique }}");</script>
|
||||
<button class="modal-close is-large" aria-label="close"></button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -1,177 +0,0 @@
|
||||
<article class="table-container" id="modal-context-table">
|
||||
<table class="table is-fullwidth">
|
||||
<thead>
|
||||
<th></th>
|
||||
<th></th>
|
||||
<th></th>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for item in object_list %}
|
||||
{% if item.type == 'control' %}
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>
|
||||
<span class="icon has-text-grey" data-tooltip="Hidden">
|
||||
<i class="fa-solid fa-file-slash"></i>
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
<p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr>
|
||||
<td>{{ item.time }}</td>
|
||||
<td>
|
||||
{% if item.type != 'znc' and item.type != 'self' and query is not True %}
|
||||
<article class="nowrap-parent">
|
||||
<article class="nowrap-child">
|
||||
{% if item.type == 'msg' %}
|
||||
<span class="icon" data-tooltip="Message">
|
||||
<i class="fa-solid fa-message"></i>
|
||||
</span>
|
||||
{% elif item.type == 'join' %}
|
||||
<span class="icon" data-tooltip="Join">
|
||||
<i class="fa-solid fa-person-to-portal"></i>
|
||||
</span>
|
||||
{% elif item.type == 'part' %}
|
||||
<span class="icon" data-tooltip="Part">
|
||||
<i class="fa-solid fa-person-from-portal"></i>
|
||||
</span>
|
||||
{% elif item.type == 'quit' %}
|
||||
<span class="icon" data-tooltip="Quit">
|
||||
<i class="fa-solid fa-circle-xmark"></i>
|
||||
</span>
|
||||
{% elif item.type == 'kick' %}
|
||||
<span class="icon" data-tooltip="Kick">
|
||||
<i class="fa-solid fa-user-slash"></i>
|
||||
</span>
|
||||
{% elif item.type == 'nick' %}
|
||||
<span class="icon" data-tooltip="Nick">
|
||||
<i class="fa-solid fa-signature"></i>
|
||||
</span>
|
||||
{% elif item.type == 'mode' %}
|
||||
<span class="icon" data-tooltip="Mode">
|
||||
<i class="fa-solid fa-gear"></i>
|
||||
</span>
|
||||
{% elif item.type == 'action' %}
|
||||
<span class="icon" data-tooltip="Action">
|
||||
<i class="fa-solid fa-exclamation"></i>
|
||||
</span>
|
||||
{% elif item.type == 'notice' %}
|
||||
<span class="icon" data-tooltip="Notice">
|
||||
<i class="fa-solid fa-message-code"></i>
|
||||
</span>
|
||||
{% elif item.type == 'conn' %}
|
||||
<span class="icon" data-tooltip="Connection">
|
||||
<i class="fa-solid fa-cloud-exclamation"></i>
|
||||
</span>
|
||||
{% elif item.type == 'znc' %}
|
||||
<span class="icon" data-tooltip="ZNC">
|
||||
<i class="fa-brands fa-unity"></i>
|
||||
</span>
|
||||
{% elif item.type == 'query' %}
|
||||
<span class="icon" data-tooltip="Query">
|
||||
<i class="fa-solid fa-message"></i>
|
||||
</span>
|
||||
{% elif item.type == 'highlight' %}
|
||||
<span class="icon" data-tooltip="Highlight">
|
||||
<i class="fa-solid fa-exclamation"></i>
|
||||
</span>
|
||||
{% elif item.type == 'who' %}
|
||||
<span class="icon" data-tooltip="Who">
|
||||
<i class="fa-solid fa-passport"></i>
|
||||
</span>
|
||||
{% elif item.type == 'topic' %}
|
||||
<span class="icon" data-tooltip="Topic">
|
||||
<i class="fa-solid fa-sign"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
{{ item.type }}
|
||||
{% endif %}
|
||||
{% if item.online is True %}
|
||||
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% elif item.online is False %}
|
||||
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
|
||||
<i class="fa-solid fa-circle"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
{% if item.src == 'irc' %}
|
||||
<a
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_drilldown' %}"
|
||||
hx-vals='{"net": "{{ item.net|escapejs }}", "nick": "{{ item.nick|escapejs }}", "channel": "{{ item.channel|escapejs }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="click"
|
||||
class="has-text-black">
|
||||
<span class="icon" data-tooltip="Open drilldown modal">
|
||||
<i class="fa-solid fa-album"></i>
|
||||
</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</article>
|
||||
<a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
|
||||
{{ item.nick }}
|
||||
</a>
|
||||
{% if item.num_chans != '—' %}
|
||||
<article class="nowrap-child">
|
||||
<span class="tag">
|
||||
{{ item.num_chans }}
|
||||
</span>
|
||||
</article>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endif %}
|
||||
{% if item.type == 'self' %}
|
||||
<span class="icon has-text-primary" data-tooltip="You">
|
||||
<i class="fa-solid fa-message-check"></i>
|
||||
</span>
|
||||
{% elif item.type == 'znc' %}
|
||||
<span class="icon has-text-info" data-tooltip="ZNC">
|
||||
<i class="fa-brands fa-unity"></i>
|
||||
</span>
|
||||
{% elif query %}
|
||||
<span class="icon has-text-info" data-tooltip="Auth">
|
||||
<i class="fa-solid fa-passport"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="wrap">{{ item.msg }}</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% if object_list %}
|
||||
<div
|
||||
class="modal-refresh"
|
||||
style="display: none;"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_context_table' %}"
|
||||
hx-vals='{"net": "{{ net }}",
|
||||
"num": "{{ num }}",
|
||||
"src": "{{ src }}",
|
||||
"channel": "{{ channel }}",
|
||||
"time": "{{ time }}",
|
||||
"date": "{{ date }}",
|
||||
"index": "{{ index }}",
|
||||
"type": "{{ type }}",
|
||||
"mtype": "{{ mtype }}",
|
||||
"nick": "{{ nick }}",
|
||||
"dedup": "{{ params.dedup }}"}'
|
||||
hx-target="#modal-context-table"
|
||||
hx-trigger="every 5s">
|
||||
</div>
|
||||
{% endif %}
|
||||
</article>
|
||||
|
||||
<script>
|
||||
var modal_event = new Event('restore-modal-scroll');
|
||||
document.dispatchEvent(modal_event);
|
||||
</script>
|
||||
@ -0,0 +1,27 @@
|
||||
{% load static %}
|
||||
|
||||
{% include 'mixins/partials/notify.html' %}
|
||||
{% if cache is not None %}
|
||||
<span class="icon has-tooltip-bottom" data-tooltip="Cached">
|
||||
<i class="fa-solid fa-database"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
|
||||
fetched {{ table.data|length }}
|
||||
{% if params.rule is None %} hits {% else %} rule hits for {{ params.rule }}{% endif %}
|
||||
in {{ took }}ms
|
||||
|
||||
{% if exemption is not None %}
|
||||
<span class="icon has-tooltip-bottom" data-tooltip="God mode">
|
||||
<i class="fa-solid fa-book-bible"></i>
|
||||
</span>
|
||||
{% else %}
|
||||
{% if redacted is not None %}
|
||||
<span class="icon has-tooltip-bottom" data-tooltip="{{ redacted }} redacted">
|
||||
<i class="fa-solid fa-mask"></i>
|
||||
</span>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% include 'partials/results_table.html' %}
|
||||
{% include 'partials/sentiment_chart.html' %}
|
||||
@ -1,8 +0,0 @@
|
||||
<magnet-block attract-distance="10" align-to="outer|center" class="floating-window">
|
||||
{% extends 'wm/panel.html' %}
|
||||
{% block heading %}
|
||||
{% endblock %}
|
||||
|
||||
{% block panel_content %}
|
||||
{% endblock %}
|
||||
</magnet-block>
|
||||
@ -1,19 +0,0 @@
|
||||
{% load static %}
|
||||
|
||||
<script src="{% static 'modal.js' %}"></script>
|
||||
{% block scripts %}
|
||||
{% endblock %}
|
||||
|
||||
{% block styles %}
|
||||
{% endblock %}
|
||||
|
||||
<div id="modal" class="modal is-active is-clipped">
|
||||
<div class="modal-background"></div>
|
||||
<div class="modal-content">
|
||||
<div class="box">
|
||||
{% block modal_content %}
|
||||
{% endblock %}
|
||||
<button class="modal-close is-large" aria-label="close"></button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -1,19 +0,0 @@
|
||||
|
||||
<nav class="panel">
|
||||
<p class="panel-heading" style="padding: .2em; line-height: .5em;">
|
||||
<i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
|
||||
{% block close_button %}
|
||||
<i
|
||||
class="fa-solid fa-xmark has-text-grey-light float-right"
|
||||
data-script="on click remove the closest <nav/>"></i>
|
||||
{% endblock %}
|
||||
{% block heading %}
|
||||
{% endblock %}
|
||||
</p>
|
||||
<article class="panel-block is-active">
|
||||
<div class="control">
|
||||
{% block panel_content %}
|
||||
{% endblock %}
|
||||
</div>
|
||||
</article>
|
||||
</nav>
|
||||
@ -1,37 +0,0 @@
|
||||
<div id="drilldown-widget">
|
||||
<div id="drilldown-widget-{{ unique }}" class="grid-stack-item" {% block widget_options %}{% endblock %}>
|
||||
<div class="grid-stack-item-content">
|
||||
|
||||
<nav class="panel">
|
||||
<p class="panel-heading" style="padding: .2em; line-height: .5em;">
|
||||
<i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
|
||||
{% block close_button %}
|
||||
<i
|
||||
class="fa-solid fa-xmark has-text-grey-light float-right"
|
||||
onclick='grid.removeWidget("drilldown-widget-{{ unique }}");'></i>
|
||||
{% endblock %}
|
||||
<i
|
||||
class="fa-solid fa-arrows-minimize has-text-grey-light float-right"
|
||||
onclick='grid.compact();'></i>
|
||||
{% block heading %}
|
||||
{% endblock %}
|
||||
</p>
|
||||
<article class="panel-block is-active">
|
||||
<div class="control">
|
||||
{% block panel_content %}
|
||||
{% endblock %}
|
||||
</div>
|
||||
</article>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
{% block custom_script %}
|
||||
{% endblock %}
|
||||
var widget_event = new Event('load-widget');
|
||||
document.dispatchEvent(widget_event);
|
||||
</script>
|
||||
{% block custom_end %}
|
||||
{% endblock %}
|
||||
@ -0,0 +1,15 @@
|
||||
import orjson
|
||||
from django import template
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
@register.filter
|
||||
def pretty(data):
|
||||
prettified = orjson.dumps(data, option=orjson.OPT_INDENT_2).decode("utf-8")
|
||||
if prettified.startswith("{"):
|
||||
prettified = prettified[1:]
|
||||
if prettified.endswith("}"):
|
||||
prettified = prettified[:-1]
|
||||
|
||||
return prettified
|
||||
@ -0,0 +1,8 @@
|
||||
from django import template
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
@register.filter
|
||||
def splitstr(value, arg):
|
||||
return value.split(arg)
|
||||
@ -1,324 +0,0 @@
|
||||
# import re
|
||||
# from base64 import b64encode
|
||||
# from random import randint
|
||||
|
||||
# from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
|
||||
# from cryptography.hazmat.primitives.ciphers.modes import ECB
|
||||
# from django.conf import settings
|
||||
# from siphashc import siphash
|
||||
# from sortedcontainers import SortedSet
|
||||
|
||||
# from core import r
|
||||
from django.conf import settings
|
||||
|
||||
|
||||
class SearchDenied:
|
||||
def __init__(self, key, value):
|
||||
self.key = key
|
||||
self.value = value
|
||||
|
||||
|
||||
class LookupDenied:
|
||||
def __init__(self, key, value):
|
||||
self.key = key
|
||||
self.value = value
|
||||
|
||||
|
||||
def remove_defaults(query_params):
|
||||
for field, value in list(query_params.items()):
|
||||
if field in settings.DRILLDOWN_DEFAULT_PARAMS:
|
||||
if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
|
||||
del query_params[field]
|
||||
|
||||
|
||||
def add_defaults(query_params):
|
||||
for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
|
||||
if field not in query_params:
|
||||
query_params[field] = value
|
||||
|
||||
|
||||
def dedup_list(data, check_keys):
|
||||
"""
|
||||
Remove duplicate dictionaries from list.
|
||||
"""
|
||||
seen = set()
|
||||
out = []
|
||||
|
||||
dup_count = 0
|
||||
for x in data:
|
||||
dedupeKey = tuple(x[k] for k in check_keys if k in x)
|
||||
if dedupeKey in seen:
|
||||
dup_count += 1
|
||||
continue
|
||||
if dup_count > 0:
|
||||
out.append({"type": "control", "hidden": dup_count})
|
||||
dup_count = 0
|
||||
out.append(x)
|
||||
seen.add(dedupeKey)
|
||||
if dup_count > 0:
|
||||
out.append({"type": "control", "hidden": dup_count})
|
||||
return out
|
||||
|
||||
|
||||
# from random import randint
|
||||
# from timeit import timeit
|
||||
# entries = 10000
|
||||
# a = [
|
||||
# {'ts': "sss", 'msg': randint(1, 2), str(randint(1, 2)): \
|
||||
# randint(1, 2)} for x in range(entries)
|
||||
# ]
|
||||
# kk = ["msg", "nick"]
|
||||
# call = lambda: dedup_list(a, kk)
|
||||
# #print(timeit(call, number=10))
|
||||
# print(dedup_list(a, kk))
|
||||
|
||||
# # sh-5.1$ python helpers.py
|
||||
# # 1.0805372429895215
|
||||
|
||||
|
||||
# def base36encode(number, alphabet="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
|
||||
# """Converts an integer to a base36 string."""
|
||||
# if not isinstance(number, (int)):
|
||||
# raise TypeError("number must be an integer")
|
||||
|
||||
# base36 = ""
|
||||
# sign = ""
|
||||
|
||||
# if number < 0:
|
||||
# sign = "-"
|
||||
# number = -number
|
||||
|
||||
# if 0 <= number < len(alphabet):
|
||||
# return sign + alphabet[number]
|
||||
|
||||
# while number != 0:
|
||||
# number, i = divmod(number, len(alphabet))
|
||||
# base36 = alphabet[i] + base36
|
||||
|
||||
# return sign + base36
|
||||
|
||||
|
||||
# def base36decode(number):
|
||||
# return int(number, 36)
|
||||
|
||||
|
||||
# def randomise_list(user, data):
|
||||
# """
|
||||
# Randomise data in a list of dictionaries.
|
||||
# """
|
||||
# if user.has_perm("core.bypass_randomisation"):
|
||||
# return
|
||||
# if isinstance(data, list):
|
||||
# for index, item in enumerate(data):
|
||||
# for key, value in item.items():
|
||||
# if key in settings.RANDOMISE_FIELDS:
|
||||
# if isinstance(value, int):
|
||||
# min_val = value - (value * settings.RANDOMISE_RATIO)
|
||||
# max_val = value + (value * settings.RANDOMISE_RATIO)
|
||||
# new_val = randint(int(min_val), int(max_val))
|
||||
# data[index][key] = new_val
|
||||
# elif isinstance(data, dict):
|
||||
# for key, value in data.items():
|
||||
# # if key in settings.RANDOMISE_FIELDS:
|
||||
# if isinstance(value, int):
|
||||
# min_val = value - (value * settings.RANDOMISE_RATIO)
|
||||
# max_val = value + (value * settings.RANDOMISE_RATIO)
|
||||
# new_val = randint(int(min_val), int(max_val))
|
||||
# data[key] = new_val
|
||||
|
||||
|
||||
# def obfuscate_list(user, data):
|
||||
# """
|
||||
# Obfuscate data in a list of dictionaries.
|
||||
# """
|
||||
# if user.has_perm("core.bypass_obfuscation"):
|
||||
# return
|
||||
# for index, item in enumerate(data):
|
||||
# for key, value in item.items():
|
||||
# # Obfuscate a ratio of the field
|
||||
# if key in settings.OBFUSCATE_FIELDS:
|
||||
# length = len(value) - 1
|
||||
# split = int(length * settings.OBFUSCATE_KEEP_RATIO)
|
||||
# first_part = value[:split]
|
||||
# second_part = value[split:]
|
||||
# second_len = len(second_part)
|
||||
# second_part = "*" * second_len
|
||||
# data[index][key] = first_part + second_part
|
||||
# # Obfuscate value based on fields
|
||||
# # Example: 2022-02-02 -> 2022-02-**
|
||||
# # 14:11:12 -> 14:11:**
|
||||
# elif key in settings.OBFUSCATE_FIELDS_SEP:
|
||||
# if "-" in value:
|
||||
# sep = "-"
|
||||
# value_spl = value.split("-")
|
||||
# hide_num = settings.OBFUSCATE_DASH_NUM
|
||||
# elif ":" in value:
|
||||
# sep = ":"
|
||||
# value_spl = value.split(":")
|
||||
# hide_num = settings.OBFUSCATE_COLON_NUM
|
||||
|
||||
# first_part = value_spl[:hide_num]
|
||||
# second_part = value_spl[hide_num:]
|
||||
# for index_x, x in enumerate(second_part):
|
||||
# x_len = len(x)
|
||||
# second_part[index_x] = "*" * x_len
|
||||
# result = sep.join([*first_part, *second_part])
|
||||
# data[index][key] = result
|
||||
# for key in settings.COMBINE_FIELDS:
|
||||
# for index, item in enumerate(data):
|
||||
# if key in item:
|
||||
# k1, k2 = settings.COMBINE_FIELDS[key]
|
||||
# if k1 in item and k2 in item:
|
||||
# data[index][key] = item[k1] + item[k2]
|
||||
|
||||
|
||||
# def hash_list(user, data, hash_keys=False):
|
||||
# """
|
||||
# Hash a list of dicts or a list with SipHash42.
|
||||
# """
|
||||
# if user.has_perm("core.bypass_hashing"):
|
||||
# return
|
||||
# cache = "cache.hash"
|
||||
# hash_table = {}
|
||||
# if isinstance(data, dict):
|
||||
# data_copy = [{x: data[x]} for x in data]
|
||||
# else:
|
||||
# data_copy = type(data)((data))
|
||||
# for index, item in enumerate(data_copy):
|
||||
# if "src" in item:
|
||||
# if item["src"] in settings.SAFE_SOURCES:
|
||||
# continue
|
||||
# if isinstance(item, dict):
|
||||
# for key, value in list(item.items()):
|
||||
# if (
|
||||
# key not in settings.WHITELIST_FIELDS
|
||||
# and key not in settings.NO_OBFUSCATE_PARAMS
|
||||
# ):
|
||||
# if isinstance(value, int):
|
||||
# value = str(value)
|
||||
# if isinstance(value, bool):
|
||||
# continue
|
||||
# if value is None:
|
||||
# continue
|
||||
# if hash_keys:
|
||||
# hashed = siphash(settings.HASHING_KEY, key)
|
||||
# else:
|
||||
# hashed = siphash(settings.HASHING_KEY, value)
|
||||
# encoded = base36encode(hashed)
|
||||
# if encoded not in hash_table:
|
||||
# if hash_keys:
|
||||
# hash_table[encoded] = key
|
||||
# else:
|
||||
# hash_table[encoded] = value
|
||||
# if hash_keys:
|
||||
# # Rename the dict key
|
||||
# data[encoded] = data.pop(key)
|
||||
# else:
|
||||
# data[index][key] = encoded
|
||||
# elif isinstance(item, str):
|
||||
# hashed = siphash(settings.HASHING_KEY, item)
|
||||
# encoded = base36encode(hashed)
|
||||
# if encoded not in hash_table:
|
||||
# hash_table[encoded] = item
|
||||
# data[index] = encoded
|
||||
# if hash_table:
|
||||
# r.hmset(cache, hash_table)
|
||||
|
||||
|
||||
# def hash_lookup(user, data_dict, supplementary_data=None):
|
||||
# cache = "cache.hash"
|
||||
# hash_list = SortedSet()
|
||||
# denied = []
|
||||
# for key, value in list(data_dict.items()):
|
||||
# if "source" in data_dict:
|
||||
# if data_dict["source"] in settings.SAFE_SOURCES:
|
||||
# continue
|
||||
# if "src" in data_dict:
|
||||
# if data_dict["src"] in settings.SAFE_SOURCES:
|
||||
# continue
|
||||
# if supplementary_data:
|
||||
# if "source" in supplementary_data:
|
||||
# if supplementary_data["source"] in settings.SAFE_SOURCES:
|
||||
# continue
|
||||
# if key in settings.SEARCH_FIELDS_DENY:
|
||||
# if not user.has_perm("core.bypass_hashing"):
|
||||
# data_dict[key] = SearchDenied(key=key, value=data_dict[key])
|
||||
# denied.append(data_dict[key])
|
||||
# if (
|
||||
# key not in settings.WHITELIST_FIELDS
|
||||
# and key not in settings.NO_OBFUSCATE_PARAMS
|
||||
# ):
|
||||
# if not value:
|
||||
# continue
|
||||
# # hashes = re.findall("\|([^\|]*)\|", value) # noqa
|
||||
# if isinstance(value, str):
|
||||
# hashes = re.findall("[A-Z0-9]{12,13}", value)
|
||||
# elif isinstance(value, dict):
|
||||
# hashes = []
|
||||
# for key, value in value.items():
|
||||
# if not value:
|
||||
# continue
|
||||
# hashes_iter = re.findall("[A-Z0-9]{12,13}", value)
|
||||
# for h in hashes_iter:
|
||||
# hashes.append(h)
|
||||
# if not hashes:
|
||||
# # Otherwise the user could inject plaintext search queries
|
||||
# if not user.has_perm("core.bypass_hashing"):
|
||||
# data_dict[key] = SearchDenied(key=key, value=data_dict[key])
|
||||
# denied.append(data_dict[key])
|
||||
# continue
|
||||
# else:
|
||||
# # There are hashes here but there shouldn't be!
|
||||
# if key in settings.TAG_SEARCH_DENY:
|
||||
# data_dict[key] = LookupDenied(key=key, value=data_dict[key])
|
||||
# denied.append(data_dict[key])
|
||||
# continue
|
||||
|
||||
# for hash in hashes:
|
||||
# hash_list.add(hash)
|
||||
|
||||
# if hash_list:
|
||||
# values = r.hmget(cache, *hash_list)
|
||||
# if not values:
|
||||
# return
|
||||
# for index, val in enumerate(values):
|
||||
# if val is None:
|
||||
# values[index] = b"ERR"
|
||||
# values = [x.decode() for x in values]
|
||||
# total = dict(zip(hash_list, values))
|
||||
# for key in data_dict.keys():
|
||||
# for hash in total:
|
||||
# if data_dict[key]:
|
||||
# if isinstance(data_dict[key], str):
|
||||
# if hash in data_dict[key]:
|
||||
# data_dict[key] = data_dict[key].replace(
|
||||
# f"{hash}", total[hash]
|
||||
# )
|
||||
# elif isinstance(data_dict[key], dict):
|
||||
# for k2, v2 in data_dict[key].items():
|
||||
# if hash in v2:
|
||||
# data_dict[key][k2] = v2.repl
|
||||
# ace(f"{hash}", total[hash])
|
||||
# return denied
|
||||
|
||||
|
||||
# def encrypt_list(user, data, secret):
|
||||
# if user.has_perm("core.bypass_encryption"):
|
||||
# return
|
||||
# cipher = Cipher(algorithms.AES(secret), ECB())
|
||||
# for index, item in enumerate(data):
|
||||
# for key, value in item.items():
|
||||
# if key not in settings.WHITELIST_FIELDS:
|
||||
# encryptor = cipher.encryptor()
|
||||
# if isinstance(value, int):
|
||||
# value = str(value)
|
||||
# if isinstance(value, bool):
|
||||
# continue
|
||||
# if value is None:
|
||||
# continue
|
||||
# decoded = value.encode("utf8", "replace")
|
||||
# length = 16 - (len(decoded) % 16)
|
||||
# decoded += bytes([length]) * length
|
||||
# ct = encryptor.update(decoded) + encryptor.finalize()
|
||||
# final_str = b64encode(ct)
|
||||
# data[index][key] = final_str.decode("utf-8", "replace")
|
||||
@ -0,0 +1,90 @@
|
||||
from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
|
||||
from django.shortcuts import render
|
||||
from mixins.views import ObjectCreate, ObjectDelete, ObjectList, ObjectUpdate
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from core.db.storage import db
|
||||
from core.forms import NotificationRuleForm, NotificationSettingsForm
|
||||
from core.lib.rules import NotificationRuleData
|
||||
from core.models import NotificationRule, NotificationSettings
|
||||
|
||||
|
||||
# Notifications - we create a new notification settings object if there isn't one
|
||||
# Hence, there is only an update view, not a create view.
|
||||
class NotificationsUpdate(LoginRequiredMixin, ObjectUpdate):
|
||||
model = NotificationSettings
|
||||
form_class = NotificationSettingsForm
|
||||
|
||||
page_title = "Update your notification settings"
|
||||
page_subtitle = (
|
||||
"At least the topic must be set if you want to receive notifications."
|
||||
)
|
||||
|
||||
submit_url_name = "notifications_update"
|
||||
submit_url_args = ["type"]
|
||||
|
||||
pk_required = False
|
||||
|
||||
hide_cancel = True
|
||||
|
||||
def get_object(self, **kwargs):
|
||||
notification_settings, _ = NotificationSettings.objects.get_or_create(
|
||||
user=self.request.user
|
||||
)
|
||||
return notification_settings
|
||||
|
||||
|
||||
class RuleList(LoginRequiredMixin, ObjectList):
|
||||
list_template = "partials/rule-list.html"
|
||||
model = NotificationRule
|
||||
page_title = "List of notification rules"
|
||||
|
||||
list_url_name = "rules"
|
||||
list_url_args = ["type"]
|
||||
|
||||
submit_url_name = "rule_create"
|
||||
|
||||
|
||||
class RuleCreate(LoginRequiredMixin, PermissionRequiredMixin, ObjectCreate):
|
||||
permission_required = "use_rules"
|
||||
model = NotificationRule
|
||||
form_class = NotificationRuleForm
|
||||
|
||||
submit_url_name = "rule_create"
|
||||
|
||||
|
||||
class RuleUpdate(LoginRequiredMixin, PermissionRequiredMixin, ObjectUpdate):
|
||||
permission_required = "use_rules"
|
||||
model = NotificationRule
|
||||
form_class = NotificationRuleForm
|
||||
|
||||
submit_url_name = "rule_update"
|
||||
|
||||
|
||||
class RuleDelete(LoginRequiredMixin, PermissionRequiredMixin, ObjectDelete):
|
||||
permission_required = "use_rules"
|
||||
model = NotificationRule
|
||||
|
||||
|
||||
class RuleClear(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
||||
permission_required = "use_rules"
|
||||
|
||||
def post(self, request, type, pk):
|
||||
template_name = "mixins/partials/notify.html"
|
||||
rule = NotificationRule.objects.get(pk=pk, user=request.user)
|
||||
if isinstance(rule.match, dict):
|
||||
for index in rule.match:
|
||||
rule.match[index] = None
|
||||
rule.save()
|
||||
|
||||
rule_data = NotificationRuleData(rule.user, rule, db=db)
|
||||
rule_data.clear_database_matches()
|
||||
|
||||
cleared_indices = ", ".join(rule.match)
|
||||
context = {
|
||||
"message": f"Cleared match status for indices: {cleared_indices}",
|
||||
"class": "success",
|
||||
}
|
||||
response = render(request, template_name, context)
|
||||
response["HX-Trigger"] = "notificationruleEvent"
|
||||
return response
|
||||
@ -1,67 +1,194 @@
|
||||
version: "2"
|
||||
|
||||
version: "2.2"
|
||||
|
||||
services:
|
||||
app:
|
||||
image: pathogen/neptune:latest
|
||||
build: ./docker
|
||||
container_name: neptune
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
OPERATION: ${OPERATION}
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
|
||||
ports:
|
||||
- "${NEPTUNE_PORT}:8000"
|
||||
- ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini
|
||||
- ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${APP_DATABASE_FILE}:/conf/db.sqlite3
|
||||
- neptune_static:${STATIC_ROOT}
|
||||
env_file:
|
||||
- stack.env
|
||||
volumes_from:
|
||||
- tmp
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_started
|
||||
collectstatic:
|
||||
condition: service_started
|
||||
networks:
|
||||
- default
|
||||
- pathogen
|
||||
- elastic
|
||||
|
||||
processing:
|
||||
image: pathogen/neptune:latest
|
||||
container_name: processing_neptune
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
OPERATION: ${OPERATION}
|
||||
command: sh -c '. /venv/bin/activate && python manage.py processing'
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini
|
||||
- ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${APP_DATABASE_FILE}:/conf/db.sqlite3
|
||||
- neptune_static:${STATIC_ROOT}
|
||||
env_file:
|
||||
- stack.env
|
||||
volumes_from:
|
||||
- tmp
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_started
|
||||
collectstatic:
|
||||
condition: service_started
|
||||
networks:
|
||||
- default
|
||||
- pathogen
|
||||
- elastic
|
||||
|
||||
scheduling:
|
||||
image: pathogen/neptune:latest
|
||||
container_name: scheduling_neptune
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
OPERATION: ${OPERATION}
|
||||
command: sh -c '. /venv/bin/activate && python manage.py scheduling'
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini
|
||||
- ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${APP_DATABASE_FILE}:/conf/db.sqlite3
|
||||
- neptune_static:${STATIC_ROOT}
|
||||
env_file:
|
||||
- .env
|
||||
- stack.env
|
||||
volumes_from:
|
||||
- tmp
|
||||
- tmp
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_started
|
||||
collectstatic:
|
||||
condition: service_started
|
||||
networks:
|
||||
- default
|
||||
- pathogen
|
||||
- elastic
|
||||
|
||||
migration:
|
||||
image: pathogen/neptune:latest
|
||||
container_name: migration_neptune
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
OPERATION: ${OPERATION}
|
||||
command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
|
||||
- ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${APP_DATABASE_FILE}:/conf/db.sqlite3
|
||||
- neptune_static:${STATIC_ROOT}
|
||||
volumes_from:
|
||||
- tmp
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
collectstatic:
|
||||
image: pathogen/neptune:latest
|
||||
container_name: collectstatic_neptune
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
OPERATION: ${OPERATION}
|
||||
command: sh -c '. /venv/bin/activate && python manage.py collectstatic --noinput'
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${APP_DATABASE_FILE}:/conf/db.sqlite3
|
||||
- neptune_static:${STATIC_ROOT}
|
||||
volumes_from:
|
||||
- tmp
|
||||
env_file:
|
||||
- stack.env
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
# pyroscope:
|
||||
# image: pyroscope/pyroscope
|
||||
# environment:
|
||||
# - PYROSCOPE_LOG_LEVEL=debug
|
||||
# ports:
|
||||
# - '4040:4040'
|
||||
# command:
|
||||
# - 'server'
|
||||
nginx:
|
||||
image: nginx:latest
|
||||
container_name: nginx_neptune
|
||||
ports:
|
||||
- ${APP_PORT}:9999
|
||||
ulimits:
|
||||
nproc: 65535
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${PORTAINER_GIT_DIR}/docker/nginx/conf.d/${OPERATION}.conf:/etc/nginx/conf.d/default.conf
|
||||
- neptune_static:${STATIC_ROOT}
|
||||
volumes_from:
|
||||
- tmp
|
||||
networks:
|
||||
- default
|
||||
- pathogen
|
||||
depends_on:
|
||||
app:
|
||||
condition: service_started
|
||||
|
||||
tmp:
|
||||
image: busybox
|
||||
command: chmod -R 777 /var/run/redis
|
||||
container_name: tmp_neptune
|
||||
command: chmod -R 777 /var/run/socks
|
||||
volumes:
|
||||
- /var/run/redis
|
||||
- /var/run/socks
|
||||
|
||||
redis:
|
||||
image: redis
|
||||
container_name: redis_neptune
|
||||
command: redis-server /etc/redis.conf
|
||||
ulimits:
|
||||
nproc: 65535
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
||||
volumes_from:
|
||||
- tmp
|
||||
healthcheck:
|
||||
test: "redis-cli -s /var/run/redis/redis.sock ping"
|
||||
test: "redis-cli -s /var/run/socks/redis.sock ping"
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
networks:
|
||||
- default
|
||||
- pathogen
|
||||
|
||||
networks:
|
||||
default:
|
||||
external:
|
||||
name: pathogen
|
||||
default:
|
||||
driver: bridge
|
||||
pathogen:
|
||||
external: true
|
||||
elastic:
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
neptune_static: {}
|
||||
@ -1,60 +0,0 @@
|
||||
version: "2"
|
||||
|
||||
services:
|
||||
app:
|
||||
image: pathogen/neptune:latest
|
||||
build: ./docker/prod
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${PORTAINER_GIT_DIR}/docker/prod/uwsgi.ini:/conf/uwsgi.ini
|
||||
- ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
|
||||
ports:
|
||||
- "${NEPTUNE_PORT}:8000" # uwsgi socket
|
||||
env_file:
|
||||
- ../stack.env
|
||||
volumes_from:
|
||||
- tmp
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_started
|
||||
|
||||
migration:
|
||||
image: pathogen/neptune:latest
|
||||
build: ./docker/prod
|
||||
command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}:/code
|
||||
- ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
|
||||
- ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
|
||||
volumes_from:
|
||||
- tmp
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
tmp:
|
||||
image: busybox
|
||||
command: chmod -R 777 /var/run/redis
|
||||
volumes:
|
||||
- /var/run/redis
|
||||
|
||||
redis:
|
||||
image: redis
|
||||
command: redis-server /etc/redis.conf
|
||||
volumes:
|
||||
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
||||
volumes_from:
|
||||
- tmp
|
||||
healthcheck:
|
||||
test: "redis-cli -s /var/run/redis/redis.sock ping"
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
networks:
|
||||
default:
|
||||
external:
|
||||
name: pathogen
|
||||
@ -0,0 +1,23 @@
|
||||
upstream django {
|
||||
#server app:8000;
|
||||
#server unix:///var/run/socks/app.sock;
|
||||
server app:8000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9999;
|
||||
|
||||
location = /favicon.ico { access_log off; log_not_found off; }
|
||||
|
||||
location /static/ {
|
||||
root /conf;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://django;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
upstream django {
|
||||
server app:8000;
|
||||
#server unix:///var/run/socks/app.sock;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9999;
|
||||
|
||||
location = /favicon.ico { access_log off; log_not_found off; }
|
||||
|
||||
location /static/ {
|
||||
root /conf;
|
||||
}
|
||||
|
||||
location / {
|
||||
include /etc/nginx/uwsgi_params; # the uwsgi_params file you installed
|
||||
uwsgi_pass django;
|
||||
uwsgi_param Host $host;
|
||||
uwsgi_param X-Real-IP $remote_addr;
|
||||
uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM python:3
|
||||
|
||||
RUN useradd -d /code pathogen
|
||||
RUN mkdir /code
|
||||
RUN chown pathogen:pathogen /code
|
||||
|
||||
RUN mkdir /conf
|
||||
RUN chown pathogen:pathogen /conf
|
||||
|
||||
RUN mkdir /venv
|
||||
RUN chown pathogen:pathogen /venv
|
||||
|
||||
USER pathogen
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
WORKDIR /code
|
||||
COPY requirements.prod.txt /code/
|
||||
RUN python -m venv /venv
|
||||
RUN . /venv/bin/activate && pip install -r requirements.prod.txt
|
||||
CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini
|
||||
@ -1,19 +0,0 @@
|
||||
wheel
|
||||
django
|
||||
django-crispy-forms
|
||||
crispy-bulma
|
||||
#opensearch-py
|
||||
stripe
|
||||
django-rest-framework
|
||||
numpy
|
||||
uwsgi
|
||||
django-tables2
|
||||
django-tables2-bulma-template
|
||||
django-htmx
|
||||
cryptography
|
||||
siphashc
|
||||
redis
|
||||
sortedcontainers
|
||||
django-debug-toolbar
|
||||
django-debug-toolbar-template-profiler
|
||||
orjson
|
||||
@ -1,2 +1,5 @@
|
||||
unixsocket /var/run/redis/redis.sock
|
||||
unixsocketperm 777
|
||||
unixsocket /var/run/socks/redis.sock
|
||||
unixsocketperm 777
|
||||
|
||||
# For Monolith PubSub
|
||||
port 6379
|
||||
@ -1,18 +0,0 @@
|
||||
wheel
|
||||
django
|
||||
django-crispy-forms
|
||||
crispy-bulma
|
||||
#opensearch-py
|
||||
stripe
|
||||
django-rest-framework
|
||||
numpy
|
||||
django-tables2
|
||||
django-tables2-bulma-template
|
||||
django-htmx
|
||||
cryptography
|
||||
siphashc
|
||||
redis
|
||||
sortedcontainers
|
||||
django-debug-toolbar
|
||||
django-debug-toolbar-template-profiler
|
||||
orjson
|
||||
@ -1,4 +1,6 @@
|
||||
NEPTUNE_PORT=5000
|
||||
PORTAINER_GIT_DIR=..
|
||||
NEPTUNE_LOCAL_SETTINGS=../app/local_settings.py
|
||||
NEPTUNE_DATABASE_FILE=../db.sqlite3
|
||||
APP_PORT=5000
|
||||
PORTAINER_GIT_DIR=.
|
||||
APP_LOCAL_SETTINGS=./app/local_settings.py
|
||||
APP_DATABASE_FILE=./db.sqlite3
|
||||
STATIC_ROOT=/conf/static
|
||||
OPERATION=dev
|
||||
Loading…
Reference in New Issue