Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement scrollback modal

This commit is contained in:
Mark Veidemanis 2022-08-09 07:20:30 +01:00
parent e335bdf722
commit 703f36751d
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
10 changed files with 338 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
core/lib/__init__.py Normal file
View File

31
core/lib/context.py Normal file
View File

@ -0,0 +1,31 @@
from core.lib.opensearch import client, run_main_query
from django.conf import settings
def construct_query(net, channel, src, num, size):
# Get the initial query
extra_must = []
if num:
extra_must.append({"match": {"num": num}})
types = ["msg", "notice", "action", "kick", "topic", "mode"]
query_should = [{"match": {"type": x}} for x in types]
query = {
"size": size,
"query": {
"bool": {
"must": [
{"match": {"net": net}},
{"match": {"channel": channel}},
{"match": {"src": src}},
{
"bool": {
"should": query_should,
}
},
*extra_must
]
}
},
"fields": ["nick", "ident", "host", "channel", "ts", "msg", "type", "net", "src"],
"_source": False,
}
return query

246
core/lib/opensearch.py
View File

@ -97,8 +97,14 @@ def filter_blacklisted(user, response):
# For every blacklisted type
for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
# Check this field we are matching exists
if blacklisted_type in item["_source"].keys():
content = item["_source"][blacklisted_type]
if "_source" in item.keys():
data_index = "_source"
elif "fields" in item.keys():
data_index = "fields"
else:
return False
if blacklisted_type in item[data_index].keys():
content = item[data_index][blacklisted_type]
# For every item in the blacklisted array for the type
for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
blacklisted_type
@ -109,7 +115,7 @@ def filter_blacklisted(user, response):
# Let the UI know something was redacted
if (
"exemption"
not in response["hits"]["hits"][index]["_source"]
not in response["hits"]["hits"][index][data_index]
):
response["redacted"] += 1
# Anonymous
@ -120,7 +126,7 @@ def filter_blacklisted(user, response):
if not user.is_superuser:
response["hits"]["hits"][index] = None
else:
response["hits"]["hits"][index]["_source"][
response["hits"]["hits"][index][data_index][
"exemption"
] = True
@ -128,6 +134,49 @@ def filter_blacklisted(user, response):
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
def construct_query(query, size):
"""
Accept some query parameters and construct an OpenSearch query.
"""
if not size:
size = 5
query = {
"size": size,
"query": {
"bool": {
"must": [
{
"query_string": {
"query": query,
# "fields": fields,
# "default_field": "msg",
# "type": "best_fields",
"fuzziness": "AUTO",
"fuzzy_transpositions": True,
"fuzzy_max_expansions": 50,
"fuzzy_prefix_length": 0,
# "minimum_should_match": 1,
"default_operator": "or",
"analyzer": "standard",
"lenient": True,
"boost": 1,
"allow_leading_wildcard": True,
# "enable_position_increments": False,
"phrase_slop": 3,
# "max_determinized_states": 10000,
"quote_field_suffix": "",
"quote_analyzer": "standard",
"analyze_wildcard": False,
"auto_generate_synonyms_phrase_query": True,
}
}
]
}
},
}
return query
def run_main_query(client, user, query, custom_query=False, index=None, size=None):
"""
Low level helper to run an ES query.
@ -153,12 +202,22 @@ def run_main_query(client, user, query, custom_query=False, index=None, size=Non
filter_blacklisted(user, response)
return response
def parse_results(results):
results_parsed = []
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
if "_source" in item.keys():
data_index = "_source"
elif "fields" in item.keys():
data_index = "fields"
else:
return False
element = item[data_index]
# Why are fields in lists...
if data_index == "fields":
element = {k: v[0] for k, v in element.items() if len(v)}
element["id"] = item["_id"]
# Split the timestamp into date and time
@ -173,11 +232,19 @@ def parse_results(results):
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
if "." in time:
time_spl = time.split(".")
if len(time_spl) == 2:
element["time"] = time.split(".")[0]
else:
element["time"] = time
else:
element["time"] = time
results_parsed.append(element)
return results_parsed
def query_results(request, query_params, size=None):
def query_results(request, query_params, size=None, annotate=True, custom_query=False):
"""
API helper to alter the OpenSearch return format into something
a bit better to parse.
@ -185,6 +252,7 @@ def query_results(request, query_params, size=None):
results with the other data we have.
"""
# is_anonymous = isinstance(request.user, AnonymousUser)
query = None
message = None
message_class = None
add_bool = []
@ -202,6 +270,8 @@ def query_results(request, query_params, size=None):
message = "Size is not permitted"
message_class = "danger"
return {"message": message, "class": message_class}
else:
size = 20
if "source" in query_params:
source = query_params["source"]
if source not in settings.OPENSEARCH_MAIN_SOURCES:
@ -275,72 +345,79 @@ def query_results(request, query_params, size=None):
if "query" in query_params:
query = query_params["query"]
search_query = construct_query(query, size)
if add_bool:
for item in add_bool:
search_query["query"]["bool"]["must"].append({"match": item})
if add_top:
for item in add_top:
search_query["query"]["bool"]["must"].append(item)
if add_top_negative:
for item in add_top_negative:
if "must_not" in search_query["query"]["bool"]:
search_query["query"]["bool"]["must_not"].append(item)
else:
search_query["query"]["bool"]["must_not"] = [item]
if sort:
search_query["sort"] = sort
if "index" in query_params:
if not request.user.is_superuser:
message = "How did you get here?"
message_class = "danger"
return {"message": message, "class": message_class}
else:
print("NO QUERY")
if custom_query:
print("CUSTOM")
search_query = custom_query
if add_bool:
for item in add_bool:
search_query["query"]["bool"]["must"].append({"match": item})
if add_top:
for item in add_top:
search_query["query"]["bool"]["must"].append(item)
if add_top_negative:
for item in add_top_negative:
if "must_not" in search_query["query"]["bool"]:
search_query["query"]["bool"]["must_not"].append(item)
else:
index = query_params["index"]
if index == "main":
index = settings.OPENSEARCH_INDEX_MAIN
elif index == "meta":
search_query["query"]["bool"]["must_not"] = [item]
if sort:
search_query["sort"] = sort
if "index" in query_params:
index = query_params["index"]
if index == "main":
index = settings.OPENSEARCH_INDEX_MAIN
else:
if request.user.is_superuser:
if index == "meta":
index = settings.OPENSEARCH_INDEX_META
elif index == "int":
if index == "int":
index = settings.OPENSEARCH_INDEX_INT
else:
message = "Index is not valid."
message_class = "danger"
return {"message": message, "class": message_class}
else:
index = settings.OPENSEARCH_INDEX_MAIN
results = run_main_query(
client,
request.user, # passed through run_main_query to filter_blacklisted
search_query,
custom_query=True,
index=index,
size=size,
)
if not results:
return False
if isinstance(results, Exception):
message = results.info["error"]["root_cause"][0]["reason"]
message_class = "danger"
return {"message": message, "class": message_class}
if len(results["hits"]["hits"]) == 0:
message = "No results."
message_class = "danger"
return {"message": message, "class": message_class}
results_parsed = parse_results(results)
else:
message = "Not permitted to search by this index"
message_class = "danger"
return {"message": message, "class": message_class}
else:
index = settings.OPENSEARCH_INDEX_MAIN
results = run_main_query(
client,
request.user, # passed through run_main_query to filter_blacklisted
search_query,
custom_query=True,
index=index,
size=size,
)
if not results:
return False
if isinstance(results, Exception):
message = results.info["error"]["root_cause"][0]["reason"]
message_class = "danger"
return {"message": message, "class": message_class}
if len(results["hits"]["hits"]) == 0:
message = "No results."
message_class = "danger"
return {"message": message, "class": message_class}
results_parsed = parse_results(results)
if annotate:
annotate_results(results_parsed)
context = {
"query": query,
"object_list": results_parsed,
"card": results["hits"]["total"]["value"],
"took": results["took"],
"redacted": results["redacted"],
"exemption": results["exemption"],
}
return context
context = {
"object_list": results_parsed,
"card": results["hits"]["total"]["value"],
"took": results["took"],
"redacted": results["redacted"],
"exemption": results["exemption"],
}
if query:
context["query"] = query
return context
def query_single_result(request):
@ -355,46 +432,3 @@ def query_single_result(request):
context["item"] = context["object_list"][0]
return context
def construct_query(query, size):
"""
Accept some query parameters and construct an OpenSearch query.
"""
if not size:
size = 5
query = {
"size": size,
"query": {
"bool": {
"must": [
{
"query_string": {
"query": query,
# "fields": fields,
# "default_field": "msg",
# "type": "best_fields",
"fuzziness": "AUTO",
"fuzzy_transpositions": True,
"fuzzy_max_expansions": 50,
"fuzzy_prefix_length": 0,
# "minimum_should_match": 1,
"default_operator": "or",
"analyzer": "standard",
"lenient": True,
"boost": 1,
"allow_leading_wildcard": True,
# "enable_position_increments": False,
"phrase_slop": 3,
# "max_determinized_states": 10000,
"quote_field_suffix": "",
"quote_analyzer": "standard",
"analyze_wildcard": False,
"auto_generate_synonyms_phrase_query": True,
}
}
]
}
},
}
return query

4
core/static/js/column-shifter.js
View File

@ -122,7 +122,7 @@ $(document).ready(function(){
// Show table content and hide spiner
var show_table_content = function($table_class_container){
$table_class_container.find("#loader").hide();
$table_class_container.find(".table-container").show();
$table_class_container.find("#table-container").show();
const event = new Event('restore-scroll');
document.dispatchEvent(event);
};
@ -222,7 +222,7 @@ $(document).ready(function(){
if(idx==undefined) {
idx = 0;
}
return $('.table-container').eq(idx).find('.btn-shift-column').filter(function(z) {
return $('#table-container').eq(idx).find('.btn-shift-column').filter(function(z) {
return $(this).data('state')=='off'
}).map(function(z) {
return $(this).data('td-class')

18
core/static/modal.js
View File

@ -2,12 +2,20 @@
var modal = document.getElementById("modal");
var html = document.querySelector('html');
var disableModal = function() {
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
var modal_refresh = document.getElementsByClassName("modal-refresh");
for(var i = 0; i < modal_refresh.length; i++) {
modal_refresh[i].remove();
}
}
var elements = document.querySelectorAll('.modal-background');
for(var i = 0; i < elements.length; i++) {
elements[i].addEventListener('click', function(e) {
// elements[i].preventDefault();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
disableModal();
});
}
@ -15,8 +23,7 @@ var elements = document.querySelectorAll('.modal-close');
for(var i = 0; i < elements.length; i++) {
elements[i].addEventListener('click', function(e) {
// elements[i].preventDefault();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
disableModal();
});
}
@ -25,8 +32,7 @@ function activateButtons() {
for(var i = 0; i < elements.length; i++) {
elements[i].addEventListener('click', function(e) {
// elements[i].preventDefault();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
disableModal();
});
}
}

85
core/templates/modals/context.html
View File

@ -4,10 +4,44 @@
<script src="{% static 'modal.js' %}"></script>
<link rel ="stylesheet" href="{% static 'tabs.css' %}">
<script src="{% static 'tabs.js' %}"></script>
<div
class="modal-refresh"
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "{{ net }}",
"num": "{{ num }}",
"src": "{{ src }}",
"channel": "{{ channel }}",
"time": "{{ time }}",
"date": "{{ date }}",
"index": "{{ index }}"
}'
hx-target="#modals-here"
hx-trigger="every 3s">
</div>
<script>
document.addEventListener("restore-modal-scroll", function(event) {
var scrollpos = localStorage.getItem('scrollpos_modal_content');
if (scrollpos) {
document.getElementsByClassName("modal-content")[0].scrollTop = scrollpos;
};
});
document.addEventListener("htmx:beforeSwap", function(event) {
var contentScroll = document.getElementsByClassName("modal-content")[0].scrollTop;
localStorage.setItem('scrollpos_modal_content', contentScroll);
});
</script>
<div id="modal" class="modal is-active is-clipped">
<div class="modal-background"></div>
<div class="modal-content">
<div class="box">
{% include 'partials/notify.html' %}
<div class="tabs is-toggle is-fullwidth is-info" id="tabs">
<ul>
<li class="is-active" data-tab="1">
@ -39,7 +73,52 @@
<div id="tab-content">
<div class="is-active" data-content="1">
<h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }}</h4>
scrollback
<article class="table-container">
<table class="table">
<thead>
<th></th>
</thead>
<tbody>
{% for item in object_list %}
{% if item.type == 'msg' %}
<tr>
<td>{{ item.time }}</td>
<td
class="has-tooltip-right {% if item.online is True %}has-text-success{% elif item.online is False %}has-text-danger{% else %}has-text-warning{% endif %}"
data-tooltip="{{ item.nick }}!{{ item.ident }}@{{ item.host }}">{{ item.nick }}</td>
<td class="wrap">{{ item.msg }}</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</article>
<div class="field has-addons">
<div class="control is-expanded has-icons-left">
<input
hx-post="{% url 'search' %}"
hx-trigger="keyup changed delay:200ms"
hx-target="#results"
hx-swap="innerHTML" id="query" name="query" value="{{ params.query }}" class="input" type="text" placeholder="msg: science AND nick: BillNye AND channel: #science">
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</div>
<div class="control">
<div class="field">
<button
id="search"
class="button is-info is-fullwidth"
hx-post="{% url 'search' %}"
hx-trigger="click"
hx-target="#results"
hx-swap="innerHTML">
Search
</button>
</div>
</div>
</div>
</div>
<div data-content="2">
<h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
@ -56,6 +135,10 @@
</div>
</div>
<button class="modal-close is-large" aria-label="close"></button>
<script>
var modal_event = new Event('restore-modal-scroll');
document.dispatchEvent(modal_event);
</script>
{# <script>activateButtons();</script> #}
</div>
</div>

16
core/templates/ui/drilldown/drilldown.html
View File

@ -2,6 +2,22 @@
{% load static %}
{% load joinsep %}
{% block content %}
{% if params.modal == 'context' %}
<div style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "{{ params.net|escapejs }}",
"num": "{{ params.num|escapejs }}",
"src": "{{ params.src|escapejs }}",
"channel": "{{ params.channel|escapejs }}",
"time": "{{ params.time|escapejs }}",
"date": "{{ params.date|escapejs }}",
"index": "{{ params.index }}"
}'
hx-target="#modals-here"
hx-trigger="load">
</div>
{% endif %}
<script src="{% static 'js/chart.js' %}"></script>
<script>
function setupTags() {

4
core/templates/ui/drilldown/table_results.html
View File

@ -2,6 +2,7 @@
{% load index %}
{% load joinsep %}
{% include 'partials/notify.html' %}
{% if table %}
<script src="{% static 'js/column-shifter.js' %}"></script>
<div style="display: none" id="jsonData" data-json="{{ data }}">
@ -26,7 +27,7 @@
{% endif %}
{% endif %}
</div>
{% if params.index == 'main' %}
{% if params.index != 'int' and params.index != 'meta' %}
<div class="box">
<div style="height: 30rem">
<canvas id="volume"></canvas>
@ -34,6 +35,7 @@
<script src="{% static 'chart.js' %}"></script>
</div>
{% endif %}
{{ params }}
{% include 'ui/drilldown/table_results_partial.html' %}
{% endif %}
{# Update the tags in case the user changed the query #}

6
core/templates/ui/drilldown/table_results_partial.html
View File

@ -52,7 +52,7 @@
});
</script>
<div class="table-container" style="display:none;">
<div id="table-container" class="table-container" style="display:none;">
<table {% render_attrs table.attrs class="table is-striped is-hoverable is-fullwidth" %}>
{% block table.thead %}
{% if table.show_header %}
@ -219,10 +219,10 @@
hx-vals='{"net": "{{ row.cells.net|escapejs }}",
"num": "{{ row.cells.num|escapejs }}",
"src": "{{ row.cells.src|escapejs }}",
"nick": "{{ row.cells.nick|escapejs }}",
"channel": "{{ row.cells.channel|escapejs }}",
"time": "{{ row.cells.time|escapejs }}",
"date": "{{ row.cells.date|escapejs }}"
"date": "{{ row.cells.date|escapejs }}",
"index": "{{ params.index }}"
}'
hx-target="#modals-here"
hx-trigger="click">

55
core/views/ui/drilldown.py
View File

@ -10,6 +10,7 @@ from django_tables2 import SingleTableView
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.lib.context import construct_query
from core.lib.opensearch import query_results
from core.lib.threshold import (
annotate_num_chans,
@ -114,10 +115,13 @@ def drilldown_search(request, return_context=False, template=None):
query_params["from_time"] = dates["from_time"]
query_params["to_time"] = dates["to_time"]
if request.GET:
context = query_results(request, query_params)
elif request.POST:
context = query_results(request, query_params)
if "query" in query_params:
if request.GET:
context = query_results(request, query_params)
elif request.POST:
context = query_results(request, query_params)
else:
context = {"object_list": []}
# URI we're passing to the template for linking
if "csrfmiddlewaretoken" in query_params:
@ -238,7 +242,10 @@ class DrilldownContextModal(APIView):
def post(self, request):
# if not request.user.has_plan(self.plan_name):
# return JsonResponse({"success": False})
num = ""
num = None
index = "main"
size = 10
# Check right fields exist
if "net" not in request.data:
return JsonResponse({"success": False})
if "channel" not in request.data:
@ -250,17 +257,49 @@ class DrilldownContextModal(APIView):
if "date" not in request.data:
return JsonResponse({"success": False})
if "num" in request.data:
num = request.data["num"]
if request.data["num"]:
if not request.data["num"] == "—":
num = request.data["num"]
if "index" in request.data:
if request.data["index"]:
index = request.data["index"]
net = request.data["net"]
channel = request.data["channel"]
# results =
# Create the query params from the POST arguments
blacklisted = ["date", "time"]
query_params = {
k: v for k, v in request.data.items() if v and k not in blacklisted
}
query_params["sorting"] = "desc"
# Create the query with the context helper
search_query = construct_query(net, channel, request.data["src"], num, size)
print("SEARCH QUERY", search_query)
results = query_results(
request, query_params, annotate=True, custom_query=search_query
)
if "message" in results:
return render(request, self.template_name, results)
results["object_list"] = results["object_list"][::-1]
# Make the time nicer
# for index, item in enumerate(results["object_list"]):
# results["object_list"][index]["time"] = item["time"]+"SSS"
context = {
"net": net,
"channel": channel,
"num": num,
"src": request.data["src"],
"ts": f"{request.data['date']} {request.data['time']}",
"object_list": results["object_list"],
"time": request.data["time"],
"date": request.data["date"],
"index": request.data["date"],
}
if index:
context["index"] = index
if num:
context["num"] = num
return render(request, self.template_name, context)