Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement Insights page

This commit is contained in:
Mark Veidemanis 2022-07-21 13:51:55 +01:00
parent a56042376c
commit 185bda02ea
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
15 changed files with 652 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
app/urls.py
View File

@ -19,16 +19,18 @@ from django.contrib import admin
from django.urls import include, path
from django.views.generic import TemplateView
from core.api.views.threshold import (
ThresholdChans,
ThresholdInfoModal,
ThresholdOnline,
ThresholdUsers,
)
# Threshold API stuff
from core.api.views.threshold import ThresholdChans, ThresholdOnline, ThresholdUsers
# Main tool pages
from core.ui.views.drilldown import Drilldown
from core.ui.views.insights import Insights
from core.views import Billing, Cancel, Home, Order, Portal, Signup
from core.views.callbacks import Callback
from core.views.dynamic.search import APISearch, Search
# Dynamic elements
from core.views.dynamic.drilldown import DrilldownSearch, ThresholdInfoModal
from core.views.dynamic.insights import InsightsInfoModal, InsightsSearch
urlpatterns = [
path("", Home.as_view(), name="home"),
@ -49,10 +51,12 @@ urlpatterns = [
path("accounts/", include("django.contrib.auth.urls")),
path("accounts/signup/", Signup.as_view(), name="signup"),
path("ui/drilldown/", Drilldown.as_view(), name="drilldown"),
path("parts/search/", Search.as_view(), name="search"),
path("modal/info/", ThresholdInfoModal.as_view(), name="modal_info"),
path("ui/insights/", Insights.as_view(), name="insights"),
path("parts/search/drilldown/", DrilldownSearch.as_view(), name="search_drilldown"),
path("parts/search/insights/", InsightsSearch.as_view(), name="search_insights"),
path("modal/drilldown/", ThresholdInfoModal.as_view(), name="modal_drilldown"),
path("modal/insights/", InsightsInfoModal.as_view(), name="modal_insights"),
path("api/chans/", ThresholdChans.as_view(), name="chans"),
path("api/users/", ThresholdUsers.as_view(), name="users"),
path("api/online/", ThresholdOnline.as_view(), name="online"),
path("api/search/", APISearch.as_view(), name="api_search"),
] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

52
core/api/views/threshold.py
View File

@ -2,17 +2,10 @@ import logging
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponse, JsonResponse
from django.shortcuts import render
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.lib.threshold import (
annotate_num_chans,
annotate_num_users,
annotate_online,
get_chans,
get_users,
)
from core.lib.threshold import annotate_online, get_chans, get_users
logger = logging.getLogger(__name__)
@ -72,46 +65,3 @@ class ThresholdOnline(LoginRequiredMixin, APIView):
query = request.data["query"]
online_info = annotate_online(net, query)
return JsonResponse(online_info)
class ThresholdInfoModal(LoginRequiredMixin, APIView):
parser_classes = [FormParser]
plan_name = "drilldown"
template_name = "modals/info.html"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
if "net" not in request.data:
return JsonResponse({"success": False})
if "nick" not in request.data:
return JsonResponse({"success": False})
if "channel" not in request.data:
return JsonResponse({"success": False})
net = request.data["net"]
nick = request.data["nick"]
channel = request.data["channel"]
channels = get_chans(net, [nick])
users = get_users(net, [channel])
num_users = annotate_num_users(net, channels)
num_chans = annotate_num_chans(net, users)
if channels:
inter_users = get_users(net, channels)
else:
inter_users = []
if users:
inter_chans = get_chans(net, users)
else:
inter_chans = []
context = {
"net": net,
"nick": nick,
"channel": channel,
"chans": channels,
"users": users,
"inter_chans": inter_chans,
"inter_users": inter_users,
"num_users": num_users,
"num_chans": num_chans,
}
return render(request, self.template_name, context)

204
core/lib/opensearch.py
View File

@ -2,8 +2,13 @@ from django.conf import settings
from opensearchpy import OpenSearch
from opensearchpy.exceptions import RequestError
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
def initialise_opensearch():
"""
Inititialise the OpenSearch API endpoint.
"""
auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
client = OpenSearch(
# fmt: off
@ -22,9 +27,157 @@ def initialise_opensearch():
return client
def construct_query(query, fields, size):
if not fields:
fields = settings.OPENSEARCH_MAIN_SEARCH_FIELDS
client = initialise_opensearch()
def annotate_results(results_parsed):
"""
Accept a list of dict objects, search for the number of channels and users.
Add them to the object.
Mutate it in place. Does not return anything.
"""
# Figure out items with net (not discord)
nets = set()
for x in results_parsed:
if "net" in x:
nets.add(x["net"])
for net in nets:
# Annotate the online attribute from Threshold
online_info = annotate_online(
net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
)
# Annotate the number of users in the channel
num_users = annotate_num_users(
net, [x["channel"] for x in results_parsed if x["src"] == "irc"]
)
# Annotate the number channels the user is on
num_chans = annotate_num_chans(
net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
)
for item in results_parsed:
if item["nick"] in online_info:
item["online"] = online_info[item["nick"]]
if item["channel"] in num_users:
item["num_users"] = num_users[item["channel"]]
if item["nick"] in num_chans:
item["num_chans"] = num_chans[item["nick"]]
def filter_blacklisted(user, response):
"""
Low level filter to take the raw OpenSearch response and remove
objects from it we want to keep secret.
Does not return, the object is mutated in place.
"""
response["redacted"] = 0
response["exemption"] = None
# For every hit from ES
for item in list(response["hits"]["hits"]):
# For every blacklisted type
for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
# Check this field we are matching exists
if blacklisted_type in item["_source"].keys():
content = item["_source"][blacklisted_type]
# For every item in the blacklisted array for the type
for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
blacklisted_type
]:
if blacklisted_item in str(content):
# Remove the item
if item in response["hits"]["hits"]:
if not user.is_superuser:
response["hits"]["hits"].remove(item)
# Let the UI know something was redacted
response["redacted"] += 1
response["exemption"] = True
def run_main_query(client, user, query, size=None):
"""
Low level helper to run an ES query.
Accept a user to pass it to the filter, so we can
avoid filtering for superusers.
Accept fields and size, for the fields we want to match and the
number of results to return.
"""
search_query = construct_query(query, size)
try:
response = client.search(
body=search_query, index=settings.OPENSEARCH_INDEX_MAIN
)
except RequestError:
print("REQUEST ERROR")
return False
filter_blacklisted(user, response)
return response
def query_results(request, size=None):
"""
API helper to alter the OpenSearch return format into something
a bit better to parse.
Accept a HTTP request object. Run the query, and annotate the
results with the other data we have.
"""
if not size:
if "size" in request.POST:
size = request.POST["size"]
if size not in settings.OPENSEARCH_MAIN_SIZES:
return False
if "query" in request.POST:
query = request.POST["query"]
results = run_main_query(
client,
request.user,
query,
size,
)
if not results:
return False
results_parsed = []
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
element["id"] = item["_id"]
# Split the timestamp into date and time
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
results_parsed.append(element)
annotate_results(results_parsed)
context = {
"query": query,
"results": results_parsed,
"card": results["hits"]["total"]["value"],
"took": results["took"],
"redacted": results["redacted"],
"exemption": results["exemption"],
}
return context
def query_single_result(request):
context = query_results(request, 1)
dedup_set = {item["nick"] for item in context["results"]}
if len(dedup_set) == 1:
context["item"] = context["results"][0]
else:
return (len(dedup_set), context)
return (1, context)
def construct_query(query, size):
"""
Accept some query parameters and construct an OpenSearch query.
"""
if not size:
size = 5
query = {
@ -32,7 +185,7 @@ def construct_query(query, fields, size):
"query": {
"query_string": {
"query": query,
"fields": fields,
# "fields": fields,
# "default_field": "msg",
# "type": "best_fields",
"fuzziness": "AUTO",
@ -63,46 +216,3 @@ def construct_query(query, fields, size):
],
}
return query
def filter_blacklisted(user, response):
response["redacted"] = 0
response["exemption"] = None
# For every hit from ES
for item in list(response["hits"]["hits"]):
# For every blacklisted type
for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
# Check this field we are matching exists
if blacklisted_type in item["_source"].keys():
content = item["_source"][blacklisted_type]
# For every item in the blacklisted array for the type
for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
blacklisted_type
]:
if blacklisted_item in str(content):
# Remove the item
if item in response["hits"]["hits"]:
if not user.is_superuser:
response["hits"]["hits"].remove(item)
# Let the UI know something was redacted
response["redacted"] += 1
response["exemption"] = True
def run_main_query(client, user, query, fields=None, size=None):
if fields:
for field in fields:
if field not in settings.OPENSEARCH_MAIN_SEARCH_FIELDS:
return False
if size:
if size not in settings.OPENSEARCH_MAIN_SIZES:
return False
search_query = construct_query(query, fields, size)
# fmt: off
try:
response = client.search(body=search_query,
index=settings.OPENSEARCH_INDEX_MAIN)
except RequestError:
return False
filter_blacklisted(user, response)
return response

5
core/templates/base.html
View File

@ -65,7 +65,7 @@
{% if user.is_authenticated %}
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link">
Plans
Tools
</a>
<div class="navbar-dropdown">
@ -73,6 +73,9 @@
<a class="navbar-item" href="{% url 'drilldown' %}">
Drilldown
</a>
<a class="navbar-item" href="{% url 'insights' %}">
Insights
</a>
{% endif %}
<hr class="navbar-divider">
<a class="navbar-item" href="mailto:help@pathogen.is">

0
core/templates/modals/info.html → core/templates/modals/drilldown.html
View File

132
core/templates/modals/insights.html Normal file
View File

@ -0,0 +1,132 @@
{% load index %}
<script>
var modal = document.querySelector('.modal'); // assuming you have only 1
var html = document.querySelector('html');
modal.querySelector('.modal-background').addEventListener('click', function(e) {
e.preventDefault();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
});
modal.querySelector('.modal-close').addEventListener('click', function(e) {
e.preventDefault();
modal.classList.remove('is-active');
html.classList.remove('is-clipped');
});
var TABS = [...document.querySelectorAll('#tabs li')];
var CONTENT = [...document.querySelectorAll('#tab-content div')];
var ACTIVE_CLASS = 'is-active';
initTabs();
</script>
<style>
#tab-content div {
display: none;
}
#tab-content div.is-active {
display: block;
}
</style>
<div class="modal is-active is-clipped">
<div class="modal-background"></div>
<div class="modal-content">
<div class="box">
<div class="tabs is-toggle is-fullwidth" id="tabs">
<ul>
<li class="is-active" data-tab="1">
<a>
<span class="icon is-small"><i class="fa-solid fa-user"></i></span>
<span>Channels</span>
</a>
</li>
<li data-tab="2">
<a>
<span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
<span>Users</span>
</a>
</li>
<li data-tab="3">
<a>
<span class="icon is-small"><i class="fa-solid fa-people"></i></span>
<span>Intersection</span>
</a>
</li>
<li data-tab="4">
<a>
<span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
<span>Intersection</span>
</a>
</li>
</ul>
</div>
<div id="tab-content">
<div class="is-active" data-content="1">
<h4 class="subtitle is-4">Channels for {{ nick }} on {{ net }}</h4>
{% for channel in chans %}
<a class="panel-block">
<span class="panel-icon">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{{ channel }}
{% if nick in num_chans %}
<span class="tag">
{{ num_users|index:channel }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
<div data-content="2">
<h4 class="subtitle is-4">Users on {{ channel }} for {{ net }}</h4>
{% for user in users %}
<a class="panel-block">
<span class="panel-icon">
<i class="fa-solid fa-user" aria-hidden="true"></i>
</span>
{{ user }}
{% if channel in num_users %}
<span class="tag">
{{ num_chans|index:user }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
<div data-content="3">
<h4 class="subtitle is-4">Users sharing channels with {{ nick }} on {{ net }}</h4>
{% for user in inter_users %}
<a class="panel-block">
<span class="panel-icon">
<i class="fa-solid fa-user" aria-hidden="true"></i>
</span>
{{ user }}
{% if channel in num_users %}
<span class="tag">
{{ num_chans|index:user }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
<div data-content="4">
<h4 class="subtitle is-4">Channels sharing users with {{ channel }} on {{ net }}</h4>
{% for channel in inter_chans %}
<a class="panel-block">
<span class="panel-icon">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{{ channel }}
{% if nick in num_chans %}
<span class="tag">
{{ num_users|index:channel }}
</span>
{% endif %}
</a>
{% endfor %}
</div>
</div>
</div>
<button class="modal-close is-large" aria-label="close"></button>
</div>
</div>

39
core/templates/ui/drilldown.html → core/templates/ui/drilldown/drilldown.html
View File

@ -56,43 +56,6 @@
</div>
<div class="columns">
<div class="column">
<label class="label">Timescale</label>
<div class="field-body">
<div class="field">
<div class="control is-expanded has-icons-left">
<div class="select is-fullwidth">
<select name="timescale">
{% for timescale in timescales %}
<option value="{{ timescale }}">{{ timescale }}</option>
{% endfor %}
</select>
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</div>
</div>
</div>
</div>
</div>
<div class="column">
<label class="label">Fields</label>
<div class="field">
<div class="control is-expanded has-icons-left">
<div class="select is-fullwidth is-multiple">
<select multiple name="fields">
{% for field in fields %}
<option value="{{ field }}">{{ field }}</option>
{% endfor %}
</select>
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</div>
</div>
</div>
</div>
<div class="column">
<label class="label">Results</label>
<div class="field">
@ -114,7 +77,7 @@
<div class="field">
<div class="control">
<button class="button is-primary is-fullwidth" hx-post="{% url 'search' %}" hx-trigger="click" hx-target="#results" hx-swap="outerHTML">
<button class="button is-primary is-fullwidth" hx-post="{% url 'search_drilldown' %}" hx-trigger="click" hx-target="#results" hx-swap="outerHTML">
Search
</button>
</div>

3
core/templates/ui/results.html → core/templates/ui/drilldown/results.html
View File

@ -1,4 +1,5 @@
{% load static %}
{% load index %}
<div id="results">
{% if results is not None %}
@ -106,7 +107,7 @@
{% if item.src == 'irc' %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_info' %}"
hx-post="{% url 'modal_drilldown' %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"

85
core/templates/ui/insights/insights.html Normal file
View File

@ -0,0 +1,85 @@
{% extends "base.html" %}
{% load static %}
{% block content %}
<script>
// tabbed browsing for the modal
function initTabs() {
TABS.forEach((tab) => {
tab.addEventListener('click', (e) => {
let selected = tab.getAttribute('data-tab');
updateActiveTab(tab);
updateActiveContent(selected);
})
})
}
function updateActiveTab(selected) {
TABS.forEach((tab) => {
if (tab && tab.classList.contains(ACTIVE_CLASS)) {
tab.classList.remove(ACTIVE_CLASS);
}
});
selected.classList.add(ACTIVE_CLASS);
}
function updateActiveContent(selected) {
CONTENT.forEach((item) => {
if (item && item.classList.contains(ACTIVE_CLASS)) {
item.classList.remove(ACTIVE_CLASS);
}
let data = item.getAttribute('data-content');
if (data === selected) {
item.classList.add(ACTIVE_CLASS);
}
});
}
</script>
<style>
.icon { border-bottom: 0px !important;}
</style>
<div class="tile is-ancestor">
<div class="tile is-parent is-vertical">
<div class="tile is-child box">
<form method="POST">
{% csrf_token %}
<div class="field">
<label class="label">Search</label>
<div class="field-body">
<div class="field">
<div class="control is-expanded has-icons-left">
<input name="query" class="input" type="text" placeholder="nickname">
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
</div>
</div>
</div>
</div>
<div class="field">
<div class="control">
<button class="button is-primary is-fullwidth" hx-post="{% url 'search_insights' %}" hx-trigger="click" hx-target="#results" hx-swap="outerHTML">
Search
</button>
</div>
</div>
</form>
</div>
<div id="results">
</div>
</div>
<div class="tile is-parent">
<p> 2</p>
</div>
</div>
{% endblock %}

79
core/templates/ui/insights/results.html Normal file
View File

@ -0,0 +1,79 @@
{% load static %}
{% load index %}
<div class="tile is-child box">
<div id="results">
{% if item is not None %}
<div class="table-container">
<table class="table is-hoverable is-fullwidth">
<tr>
<th>src</th>
<td>
{% if item|index:'src' == 'irc' %}
<span class="icon" data-tooltip="IRC">
<i class="fa-solid fa-hashtag" aria-hidden="true"></i>
</span>
{% elif item|index:'src' == 'dis' %}
<span class="icon" data-tooltip="Discord">
<i class="fa-brands fa-discord" aria-hidden="true"></i>
</span>
{% endif %}
</td>
</tr>
<tr>
<th>nick</th>
<td>
{% if item|index:'online' is True %}
<span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
<i class="fa-solid fa-circle"></i>
</span>
{{ item.nick }}
{% elif item|index:'online' is False %}
<span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
<i class="fa-solid fa-circle"></i>
</span>
{{ item|index:'nick' }}
{% else %}
<span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
<i class="fa-solid fa-circle"></i>
</span>
{{ item|index:'nick'}}
{% endif %}
{% if item|index:'num_chans' is not None %}
<span class="tag">
{{ item|index:'num_chans' }}
</span>
{% endif %}
</td>
</tr>
<tr>
<th>host</th>
<td>{{ item.host }}</td>
</tr>
<tr>
<th>actions</th>
<td>
{% if item.src == 'irc' %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_insights' %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small">
Information
</button>
<div id="modals-here"></div>
{% endif %}
</td>
</tr>
<tr>
<th>net</th>
<td>{{ item.net }}</td>
</tr>
</table>
</div>
{% endif %}
</div>
</div>

8
core/ui/views/drilldown.py
View File

@ -3,21 +3,15 @@ from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import render
from django.views import View
from core.lib.opensearch import initialise_opensearch
client = initialise_opensearch()
class Drilldown(LoginRequiredMixin, View):
template_name = "ui/drilldown.html"
template_name = "ui/drilldown/drilldown.html"
plan_name = "drilldown"
def get(self, request):
if not request.user.has_plan(self.plan_name):
return render(request, "denied.html")
context = {
"fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS,
"sizes": settings.OPENSEARCH_MAIN_SIZES,
"timescales": settings.OPENSEARCH_MAIN_TIMESCALES,
}
return render(request, self.template_name, context)

13
core/ui/views/insights.py Normal file
View File

@ -0,0 +1,13 @@
from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import render
from django.views import View
class Insights(LoginRequiredMixin, View):
template_name = "ui/insights/insights.html"
plan_name = "drilldown"
def get(self, request):
if not request.user.has_plan(self.plan_name):
return render(request, "denied.html")
return render(request, self.template_name)

88
core/views/dynamic/drilldown.py Normal file
View File

@ -0,0 +1,88 @@
import json
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
from django.shortcuts import render
from django.views import View
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.lib.opensearch import query_results
from core.lib.threshold import (
annotate_num_chans,
annotate_num_users,
get_chans,
get_users,
)
class DrilldownSearch(LoginRequiredMixin, View):
# parser_classes = [JSONParser]
template_name = "ui/drilldown/results.html"
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return HttpResponseForbidden()
context = query_results(request)
if not context:
return HttpResponseForbidden()
context["data"] = json.dumps(
[
{
"text": item.get("msg", None) or item.get("id"),
"nick": item.get("nick", None),
"value": item.get("sentiment", None) or None,
"date": item.get("ts"),
}
for item in context["results"]
]
)
if context:
return render(request, self.template_name, context)
else:
return HttpResponse("No results")
class ThresholdInfoModal(LoginRequiredMixin, APIView):
parser_classes = [FormParser]
plan_name = "drilldown"
template_name = "modals/drilldown.html"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
if "net" not in request.data:
return JsonResponse({"success": False})
if "nick" not in request.data:
return JsonResponse({"success": False})
if "channel" not in request.data:
return JsonResponse({"success": False})
net = request.data["net"]
nick = request.data["nick"]
channel = request.data["channel"]
channels = get_chans(net, [nick])
users = get_users(net, [channel])
num_users = annotate_num_users(net, channels)
num_chans = annotate_num_chans(net, users)
if channels:
inter_users = get_users(net, channels)
else:
inter_users = []
if users:
inter_chans = get_chans(net, users)
else:
inter_chans = []
context = {
"net": net,
"nick": nick,
"channel": channel,
"chans": channels,
"users": users,
"inter_chans": inter_chans,
"inter_users": inter_users,
"num_users": num_users,
"num_chans": num_chans,
}
return render(request, self.template_name, context)

75
core/views/dynamic/insights.py Normal file
View File

@ -0,0 +1,75 @@
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
from django.shortcuts import render
from django.views import View
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.lib.opensearch import query_single_result
from core.lib.threshold import (
annotate_num_chans,
annotate_num_users,
get_chans,
get_users,
)
class InsightsSearch(LoginRequiredMixin, View):
# parser_classes = [JSONParser]
template_name = "ui/insights/results.html"
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return HttpResponseForbidden()
results, context = query_single_result(request)
if not context:
return HttpResponseForbidden()
if context:
return render(request, self.template_name, context)
else:
return HttpResponse("No results")
class InsightsInfoModal(LoginRequiredMixin, APIView):
parser_classes = [FormParser]
plan_name = "drilldown"
template_name = "modals/insights.html"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
if "net" not in request.data:
return JsonResponse({"success": False})
if "nick" not in request.data:
return JsonResponse({"success": False})
if "channel" not in request.data:
return JsonResponse({"success": False})
net = request.data["net"]
nick = request.data["nick"]
channel = request.data["channel"]
channels = get_chans(net, [nick])
users = get_users(net, [channel])
num_users = annotate_num_users(net, channels)
num_chans = annotate_num_chans(net, users)
if channels:
inter_users = get_users(net, channels)
else:
inter_users = []
if users:
inter_chans = get_chans(net, users)
else:
inter_chans = []
context = {
"net": net,
"nick": nick,
"channel": channel,
"chans": channels,
"users": users,
"inter_chans": inter_chans,
"inter_users": inter_users,
"num_users": num_users,
"num_chans": num_chans,
}
return render(request, self.template_name, context)

119
core/views/dynamic/search.py
View File

@ -1,119 +0,0 @@
import json
from django.conf import settings
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
from django.shortcuts import render
from django.views import View
from core.lib.opensearch import initialise_opensearch, run_main_query
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
client = initialise_opensearch()
def query_results(request, post_params, api=False):
fields = None
if "fields" in request.POST:
fields = request.POST.getlist("fields")
if "size" in request.POST:
size = request.POST["size"]
if "query" in request.POST:
query = request.POST["query"]
results = run_main_query(client, request.user, query, fields, size)
if not results:
return False
results_parsed = []
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
element["id"] = item["_id"]
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
results_parsed.append(element)
# Figure out items with net (not discord)
nets = set()
for x in results_parsed:
if "net" in x:
nets.add(x["net"])
for net in nets:
# Annotate the online attribute from Threshold
online_info = annotate_online(
net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
)
# Annotate the number of users in the channel
num_users = annotate_num_users(
net, [x["channel"] for x in results_parsed if x["src"] == "irc"]
)
# Annotate the number channels the user is on
num_chans = annotate_num_chans(
net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
)
for item in results_parsed:
if item["nick"] in online_info:
item["online"] = online_info[item["nick"]]
if item["channel"] in num_users:
item["num_users"] = num_users[item["channel"]]
if item["nick"] in num_chans:
item["num_chans"] = num_chans[item["nick"]]
context = {
"query": query,
"results": results_parsed,
"card": results["hits"]["total"]["value"],
"took": results["took"],
"redacted": results["redacted"],
"exemption": results["exemption"],
"fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS,
"sizes": settings.OPENSEARCH_MAIN_SIZES,
"timescales": settings.OPENSEARCH_MAIN_TIMESCALES,
}
return context
class Search(LoginRequiredMixin, View):
# parser_classes = [JSONParser]
template_name = "ui/results.html"
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return HttpResponseForbidden()
context = query_results(request, request.POST)
if not context:
return HttpResponseForbidden()
context["data"] = json.dumps(
[
{
"text": item.get("msg", None) or item.get("id"),
"nick": item.get("nick", None),
"value": item.get("sentiment", None) or None,
"date": item.get("ts"),
}
for item in context["results"]
]
)
if context:
return render(request, self.template_name, context)
else:
return HttpResponse("No results")
class APISearch(LoginRequiredMixin, View):
# parser_classes = [JSONParser]
template_name = "ui/results.html"
plan_name = "drilldown"
def post(self, request):
if not request.user.has_plan(self.plan_name):
return JsonResponse({"success": False})
context = query_results(request, request.POST)
return JsonResponse(context)