Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve modal and implement deduplication

This commit is contained in:
Mark Veidemanis 2022-08-16 00:15:36 +01:00
parent 7c94e27d22
commit 774ab800a0
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
9 changed files with 477 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/lib/manage/threshold.py
View File

@ -196,7 +196,6 @@ def send_irc_message(net, num, channel, msg, nick=None):
payload = {"msg": msg, "channel": channel}
if nick:
payload["nick"] = nick
print("SEND", payload)
messaged = threshold_request(url, payload, method="PUT")
return messaged

36
core/lib/opensearch.py
View File

@ -3,6 +3,7 @@ from opensearchpy import OpenSearch
from opensearchpy.exceptions import NotFoundError, RequestError
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
from core.views.helpers import dedup_list
def initialise_opensearch():
@ -49,7 +50,9 @@ def annotate_results(results_parsed):
[
x["nick"]
for x in results_parsed
if x["src"] == "irc" and x["net"] == net and "nick" in x
if {"nick", "src", "net"}.issubset(x)
and x["src"] == "irc"
and x["net"] == net
]
)
)
@ -58,7 +61,9 @@ def annotate_results(results_parsed):
[
x["channel"]
for x in results_parsed
if x["src"] == "irc" and x["net"] == net and "channel" in x
if {"channel", "src", "net"}.issubset(x)
and x["src"] == "irc"
and x["net"] == net
]
)
)
@ -244,7 +249,16 @@ def parse_results(results):
return results_parsed
def query_results(request, query_params, size=None, annotate=True, custom_query=False):
def query_results(
request,
query_params,
size=None,
annotate=True,
custom_query=False,
reverse=False,
dedup=False,
dedup_fields=None,
):
"""
API helper to alter the OpenSearch return format into something
a bit better to parse.
@ -403,8 +417,24 @@ def query_results(request, query_params, size=None, annotate=True, custom_query=
return {"message": message, "class": message_class}
results_parsed = parse_results(results)
if annotate:
annotate_results(results_parsed)
if "dedup" in query_params:
if query_params["dedup"] == "on":
dedup = True
else:
dedup = False
else:
dedup = False
if reverse:
results_parsed = results_parsed[::-1]
if dedup:
if not dedup_fields:
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
results_parsed = dedup_list(results_parsed, dedup_fields)
context = {
"object_list": results_parsed,

6
core/templates/manage/threshold/irc/network/relays.html
View File

@ -112,7 +112,8 @@
"index": "int",
"type": "znc",
"mtype": "None",
"nick": "*status"}'
"nick": "*status",
"dedup": "on"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small has-background-info has-text-white">
@ -201,7 +202,8 @@
"index": "int",
"type": "auth",
"mtype": "None",
"nick": "{{ sinst.entity }}"}'
"nick": "{{ sinst.entity }}",
"dedup": "on"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small has-background-info has-text-white">

145
core/templates/modals/context_table.html
View File

@ -1,27 +1,149 @@
<article class="table-container" id="modal-context-table">
<table class="table">
<table class="table is-fullwidth">
<thead>
<th></th>
<th></th>
<th></th>
</thead>
<tbody>
{% for item in object_list %}
{% if item.type == 'control' %}
<tr>
<td></td>
<td>
<span class="icon has-text-grey" data-tooltip="Hidden">
<i class="fa-solid fa-file-slash"></i>
</span>
</td>
<td>
<p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
</td>
</tr>
{% else %}
<tr>
<td>{{ item.time }}</td>
{% if query is True and item.type == 'self' %}
<td
class="has-tooltip-right {% if item.online is True %}has-text-success{% elif item.online is False %}has-text-danger{% else %}has-text-warning{% endif %}"
data-tooltip="You!">{{ item.channel }}</td>
<td>
{% if item.type != 'znc' and item.type != 'self' and query is not True %}
<article class="nowrap-parent">
<article class="nowrap-child">
{% if item.type == 'msg' %}
<span class="icon" data-tooltip="Message">
<i class="fa-solid fa-message"></i>
</span>
{% elif item.type == 'join' %}
<span class="icon" data-tooltip="Join">
<i class="fa-solid fa-person-to-portal"></i>
</span>
{% elif item.type == 'part' %}
<span class="icon" data-tooltip="Part">
<i class="fa-solid fa-person-from-portal"></i>
</span>
{% elif item.type == 'quit' %}
<span class="icon" data-tooltip="Quit">
<i class="fa-solid fa-circle-xmark"></i>
</span>
{% elif item.type == 'kick' %}
<span class="icon" data-tooltip="Kick">
<i class="fa-solid fa-user-slash"></i>
</span>
{% elif item.type == 'nick' %}
<span class="icon" data-tooltip="Nick">
<i class="fa-solid fa-signature"></i>
</span>
{% elif item.type == 'mode' %}
<span class="icon" data-tooltip="Mode">
<i class="fa-solid fa-gear"></i>
</span>
{% elif item.type == 'action' %}
<span class="icon" data-tooltip="Action">
<i class="fa-solid fa-exclamation"></i>
</span>
{% elif item.type == 'notice' %}
<span class="icon" data-tooltip="Notice">
<i class="fa-solid fa-message-code"></i>
</span>
{% elif item.type == 'conn' %}
<span class="icon" data-tooltip="Connection">
<i class="fa-solid fa-cloud-exclamation"></i>
</span>
{% elif item.type == 'znc' %}
<span class="icon" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
{% elif item.type == 'query' %}
<span class="icon" data-tooltip="Query">
<i class="fa-solid fa-message"></i>
</span>
{% elif item.type == 'highlight' %}
<span class="icon" data-tooltip="Highlight">
<i class="fa-solid fa-exclamation"></i>
</span>
{% elif item.type == 'who' %}
<span class="icon" data-tooltip="Who">
<i class="fa-solid fa-passport"></i>
</span>
{% else %}
<td
class="has-tooltip-right {% if item.online is True %}has-text-success{% elif item.online is False %}has-text-danger{% else %}has-text-warning{% endif %}"
data-tooltip="{{ item.nick }}!{{ item.ident }}@{{ item.host }}">{{ item.nick }}</td>
{{ item.type }}
{% endif %}
{% if item.online is True %}
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
<i class="fa-solid fa-circle"></i>
</span>
{% elif item.online is False %}
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
<i class="fa-solid fa-circle"></i>
</span>
{% else %}
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
<i class="fa-solid fa-circle"></i>
</span>
{% endif %}
{% if item.src == 'irc' %}
<a
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"
class="has-text-black">
<span class="icon" data-tooltip="Open drilldown modal">
<i class="fa-solid fa-album"></i>
</span>
</a>
{% endif %}
</article>
<a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
{{ item.nick }}
</a>
{% if item.num_chans != '—' %}
<article class="nowrap-child">
<span class="tag">
{{ item.num_chans }}
</span>
</article>
{% endif %}
</article>
{% endif %}
{% if item.type == 'self' %}
<span class="icon has-text-primary" data-tooltip="You">
<i class="fa-solid fa-message-check"></i>
</span>
{% elif item.type == 'znc' %}
<span class="icon has-text-info" data-tooltip="ZNC">
<i class="fa-brands fa-unity"></i>
</span>
{% elif query %}
<span class="icon has-text-info" data-tooltip="Auth">
<i class="fa-solid fa-passport"></i>
</span>
{% endif %}
</td>
<td class="wrap">{{ item.msg }}</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</article>
{% if object_list %}
<div
class="modal-refresh"
@ -37,11 +159,14 @@
"index": "{{ index }}",
"type": "{{ type }}",
"mtype": "{{ mtype }}",
"nick": "{{ nick }}"}'
"nick": "{{ nick }}",
"dedup": "{{ params.dedup }}"}'
hx-target="#modal-context-table"
hx-trigger="every 5s">
</div>
{% endif %}
</article>
<script>
var modal_event = new Event('restore-modal-scroll');
document.dispatchEvent(modal_event);

14
core/templates/ui/drilldown/drilldown.html
View File

@ -179,7 +179,7 @@
</div>
</div>
<div class="column is-narrow">
<div class="field has-addons">
<div class="field has-addons block">
<div class="control has-icons-left">
<span class="select">
<select id="source" name="source">
@ -215,6 +215,18 @@
</a>
</p>
</div>
<div class="control">
<label class="button has-text-link">
<input type="checkbox"
name="dedup"
{% if params.dedup == "on" %}
checked
{% endif %}>
<span class="icon" data-tooltip="Deduplicate results">
<i class="fa-solid fa-copy"></i>
</span>
</label>
</div>
</div>
<div class="column is-narrow">
<div id="sentiment block">

29
core/templates/ui/drilldown/table_results_partial.html
View File

@ -111,6 +111,19 @@
<tbody {{ table.attrs.tbody.as_html }}>
{% for row in table.paginated_rows %}
{% block table.tbody.row %}
{% if row.cells.type == 'control' %}
<tr>
<td></td>
<td>
<span class="icon has-text-grey" data-tooltip="Hidden">
<i class="fa-solid fa-file-slash"></i>
</span>
</td>
<td>
<p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
</td>
</tr>
{% else %}
<tr class="
{% if row.cells.exemption == True %}has-background-grey-lighter
{% elif cell == 'join' %}has-background-success-light
@ -134,7 +147,8 @@
</td>
{% elif column.name == 'src' %}
<td class="{{ column.name }}">
<a class="has-text-link is-underlined"
<a
class="has-text-link is-underlined"
onclick="populateSearch('src', '{{ cell|escapejs }}')">
{% if row.cells.src == 'irc' %}
<span class="icon" data-tooltip="IRC">
@ -154,7 +168,8 @@
</td>
{% elif column.name == 'type' or column.name == 'mtype' %}
<td class="{{ column.name }}">
<a class="has-text-link is-underlined"
<a
class="has-text-link is-underlined"
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
{% if cell == 'msg' %}
<span class="icon" data-tooltip="Message">
@ -232,7 +247,8 @@
"index": "{{ params.index }}",
"type": "{{ row.cells.type }}",
"mtype": "{{ row.cells.mtype }}",
"nick": "{{ row.cells.nick|escapejs }}"}'
"nick": "{{ row.cells.nick|escapejs }}",
"dedup": "{{ params.dedup }}"}'
hx-target="#modals-here"
hx-trigger="click"
href="/?modal=context&net={{row.cells.net|escapejs}}&num={{row.cells.num|escapejs}}&src={{row.cells.src|escapejs}}&channel={{row.cells.channel|urlsafe}}&time={{row.cells.time|escapejs}}&date={{row.cells.date|escapejs}}&index={{params.index}}&type={{row.cells.type}}&mtype={{row.cells.mtype}}&nick={{row.cells.mtype|escapejs}}">
@ -286,7 +302,8 @@
<td class="{{ column.name }}">
{% if cell != '—' %}
<div class="nowrap-parent">
<a class="nowrap-child has-text-link is-underlined"
<a
class="nowrap-child has-text-link is-underlined"
onclick="populateSearch('channel', '{{ cell|escapejs }}')">
{{ cell }}
</a>
@ -316,7 +333,8 @@
</td>
{% else %}
<td class="{{ column.name }}">
<a class="has-text-link is-underlined"
<a
class="has-text-link is-underlined"
onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
{{ cell }}
</a>
@ -326,6 +344,7 @@
{% endif %}
{% endfor %}
</tr>
{% endif %}
{% endblock table.tbody.row %}
{% empty %}
{% if table.empty_text %}

39
core/views/helpers.py Normal file
View File

@ -0,0 +1,39 @@
def dedup_list(data, check_keys):
"""
Remove duplicate dictionaries from list.
"""
seen = set()
out = []
dup_count = 0
for x in data:
dedupeKey = tuple(x[k] for k in check_keys if k in x)
if dedupeKey in seen:
dup_count += 1
continue
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
dup_count = 0
out.append(x)
seen.add(dedupeKey)
if dup_count > 0:
out.append({"type": "control", "hidden": dup_count})
return out
# from random import randint
# from timeit import timeit
# entries = 10000
# a = [
# {'ts': "sss", 'msg': randint(1, 2), str(randint(1, 2)): randint(1, 2)} for x in range(entries)
# ]
# kk = ["msg", "nick"]
# call = lambda: dedup_list(a, kk)
# #print(timeit(call, number=10))
# print(dedup_list(a, kk))
# # sh-5.1$ python helpers.py
# # 1.0805372429895215

15
core/views/ui/drilldown.py
View File

@ -254,14 +254,11 @@ class DrilldownContextModal(APIView):
nicks = None
query = False
# Create the query params from the POST arguments
mandatory = ["net", "channel", "num", "src", "index", "nick", "type"]
mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"]
invalid = [None, False, "—", "None"]
query_params = {k: v for k, v in request.data.items() if v}
if query_params["index"] == "int":
mandatory.append("mtype")
for key in query_params:
if query_params[key] in invalid:
query_params[key] = None
@ -314,13 +311,16 @@ class DrilldownContextModal(APIView):
if query_params["src"] == "irc":
if query_params["type"] in ["query", "notice", "msg", "highlight"]:
annotate = True
results = query_results(
request, query_params, annotate=annotate, custom_query=search_query
request,
query_params,
annotate=annotate,
custom_query=search_query,
reverse=True,
dedup_fields=["net", "type", "msg"],
)
if "message" in results:
return render(request, self.template_name, results)
results["object_list"] = results["object_list"][::-1]
# Make the time nicer
# for index, item in enumerate(results["object_list"]):
@ -339,6 +339,7 @@ class DrilldownContextModal(APIView):
"type": query_params["type"],
"mtype": query_params["mtype"],
"nick": query_params["nick"],
"params": query_params,
}
if request.user.is_superuser:
if query:

1
core/views/ui/tables.py
View File

@ -66,5 +66,6 @@ class DrilldownTable(Table):
server = Column()
mtype = Column()
tokens = Column()
hidden = Column()
template_name = "ui/drilldown/table_results.html"
paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE