Implement paginated sortable table for results

app/urls.py

@@ -51,7 +51,10 @@ from core.views.manage.threshold.threshold import (
 )
 
 # Main tool pages
-from core.views.ui.drilldown import Drilldown, ThresholdInfoModal  # DrilldownTableView,
+from core.views.ui.drilldown import (  # DrilldownTableView,; Drilldown,
+    DrilldownTableView,
+    ThresholdInfoModal,
+)
 from core.views.ui.insights import (
     Insights,
     InsightsChannels,
@@ -62,7 +65,8 @@ from core.views.ui.insights import (
 )
 
 urlpatterns = [
-    path("", Drilldown.as_view(), name="home"),
+    path("", DrilldownTableView.as_view(), name="home"),
+    path("search/", DrilldownTableView.as_view(), name="search"),
     path("about/", About.as_view(), name="about"),
     path("callback", Callback.as_view(), name="callback"),
     path("billing/", Billing.as_view(), name="billing"),

core/lib/opensearch.py

@@ -299,7 +299,7 @@ def query_results(request, query_params, size=None):
 
         context = {
             "query": query,
-            "results": results_parsed,
+            "object_list": results_parsed,
             "card": results["hits"]["total"]["value"],
             "took": results["took"],
             "redacted": results["redacted"],

core/templates/ui/drilldown/drilldown.html

@@ -88,7 +88,7 @@
     }
   </script>
   <div>
-    <form method="POST" hx-post="{% url 'home' %}"
+    <form method="POST" hx-post="{% url 'search' %}"
           hx-trigger="change"
           hx-target="#results"
           hx-swap="innerHTML"
@@ -100,7 +100,7 @@
           <div class="field has-addons">
             <div class="control is-expanded has-icons-left">
               <input
-                hx-post="{% url 'home' %}"
+                hx-post="{% url 'search' %}"
                 hx-trigger="keyup changed delay:200ms"
                 hx-target="#results"
                 hx-swap="innerHTML" id="query" name="query" value="{{ params.query }}" class="input" type="text" placeholder="msg: science AND nick: BillNye AND channel: #science">
@@ -113,7 +113,7 @@
                 <button
                   id="search"
                   class="button is-info is-fullwidth"
-                  hx-post="{% url 'home' %}"
+                  hx-post="{% url 'search' %}"
                   hx-trigger="click"
                   hx-target="#results"
                   hx-swap="innerHTML">
@@ -365,8 +365,11 @@
   </div>
   <div class="block">
     <div id="results">
-      {% if results %}
+      <!-- {% if results %}
         {% include 'ui/drilldown/results.html' %}
+      {% endif %} -->
+      {% if table %}
+        {% include 'ui/drilldown/table_results.html' %}
       {% endif %}
     </div>
   </div>

core/templates/ui/drilldown/table_results.html

@@ -10,7 +10,7 @@
       <i class="fa-solid fa-chart-mixed"></i>
     </div>
     <div class="nowrap-child">
-      <p>fetched {{ results|length }} of {{ card }} hits in {{ took }}ms</p>
+      <p>fetched {{ table.data|length }} of {{ card }} hits in {{ took }}ms</p>
     </div>
     {% if exemption is not None %}
       <div class="nowrap-child">
@@ -24,6 +24,12 @@
       {% endif %}
     {% endif %}
   </div>
+  <div class="box">
+    <div style="height: 30rem">
+      <canvas id="volume"></canvas>
+    </div>
+    <script src="{% static 'chart.js' %}"></script>
+  </div>
   <div class="box">
     <div class="table-container">
       {% include 'ui/drilldown/table_results_partial.html' %}

core/templates/ui/drilldown/table_results_partial copy.html

@@ -0,0 +1,80 @@
+{% extends 'django-tables2/bulma.html' %}
+
+{% load django_tables2 %}
+
+{% load i18n %}
+
+{% block table.thead %}
+  {% if table.show_header %}
+    <thead {{ table.attrs.thead.as_html }}>
+      <tr>
+        {% for column in table.columns %}
+          <th
+            {{ column.attrs.th.as_html }}
+            hx-get="search/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-trigger="click"
+            hx-target="#results"
+            hx-swap="innerHTML"
+            hx-indicator=".progress"
+            style="cursor: pointer;">
+            {{ column.header }}
+          </th>
+        {% endfor %}
+      </tr>
+    </thead>
+  {% endif %}
+{% endblock table.thead %}
+
+{# Pagination block overrides #}
+{% block pagination.previous %}
+  {% if table.page.has_previous %}
+    <li class="pagination-previous">
+      <div
+        hx-get="search/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
+        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        hx-trigger="click"
+        hx-target="#results"
+        hx-swap="innerHTML"
+        hx-indicator=".progress"
+        class="pagination-link">
+        <span aria-hidden="true">&laquo;</span>
+        {% trans 'previous' %}
+      </div>
+    </li>
+  {% endif %}
+{% endblock pagination.previous %}
+
+{% block pagination.range %}
+  {% for p in table.page|table_page_range:table.paginator %}
+    <li class="page-item{% if table.page.number == p %} active{% endif %}">
+      <div
+        class="pagination-link" 
+        {% if p != '...' %}hx-get="search/{% querystring table.prefixed_page_field=p %}&{{ uri }}"{% endif %}
+        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        hx-trigger="click"
+        hx-target="#results"
+        hx-swap="innerHTML"
+        hx-indicator=".progress">
+        {{ p }}
+      </div>
+    </li>
+  {% endfor %}
+{% endblock pagination.range %}
+{% if table.page.has_next %}
+  {% block pagination.next %}
+    <li class="pagination-next">
+      <div
+        hx-get="search/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
+        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        hx-trigger="click"
+        hx-target="#results"
+        hx-swap="innerHTML"
+        hx-indicator=".progress"
+        class="pagination-link">
+        {% trans 'next' %}
+        <span aria-hidden="true">&raquo;</span>
+      </div>
+    </li>
+  {% endblock pagination.next %}
+{% endif %}

core/templates/ui/drilldown/table_results_partial.html

@@ -1,75 +1,163 @@
-{% extends 'django-tables2/bulma.html' %}
-
 {% load django_tables2 %}
-
 {% load i18n %}
-
+{% load django_tables2_bulma_template %}
-{% block table.thead %}
+{% block table-wrapper %}
-  {% if table.show_header %}
+  <div class="container">
-    <thead {{ table.attrs.thead.as_html }}>
+    {% block table %}
-      <tr>
+      <table {% render_attrs table.attrs class="table is-striped is-hoverable is-fullwidth" %}>
-        {% for column in table.columns %}
+        {% block table.thead %}
-          <th
+          {% if table.show_header %}
-            {{ column.attrs.th.as_html }}
+            <thead {% render_attrs table.attrs.thead class="" %}>
-            hx-post="{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}"
+              {% block table.thead.row %}
-            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                <tr>
-            hx-trigger="click"
+                  {% for column in table.columns %}
-            hx-target="div.table-container"
+                    {% block table.thead.th %}
-            hx-swap="outerHTML"
+                      <th {% render_attrs column.attrs.th class="" %}>
-            hx-indicator=".progress"
+                        {% if column.orderable %}
-            style="cursor: pointer;">
+                          {% if column.is_ordered %}
-            {{ column.header }}
+                            {% is_descending column.order_by as descending %}
-          </th>
+                            {% if descending %}
-        {% endfor %}
+                              <span aria-hidden="true">{% block table.desc_icon %}&darr;{% endblock table.desc_icon %}</span>
-      </tr>
+                            {% else %}
-    </thead>
+                              <span aria-hidden="true">{% block table.asc_icon %}&uarr;{% endblock table.asc_icon %}</span>
-  {% endif %}
+                            {% endif %}
-{% endblock table.thead %}
+                          {% else %}
-
+                            <span aria-hidden="true">{% block table.orderable_icon %}&ReverseUpEquilibrium;{% endblock table.orderable_icon %}</span>
-{# Pagination block overrides #}
+                          {% endif %}
-{% block pagination.previous %}
+                          <a
-  <li class="previous page-item">
+                            hx-get="search/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
-    <div
+                            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-      hx-post="{% querystring table.prefixed_page_field=table.page.previous_page_number %}"
+                            hx-trigger="click"
-      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                            hx-target="#results"
-      hx-trigger="click"
+                            hx-swap="innerHTML"
-      hx-target="div.table-container"
+                            hx-indicator=".progress"
-      hx-swap="outerHTML"
+                            style="cursor: pointer;">
-      hx-indicator=".progress"
+                            {{ column.header }}
-      class="page-link">
+                          </a>
-      <span aria-hidden="true">&laquo;</span>
+                        {% else %}
-      {% trans 'previous' %}
+                          {{ column.header }}
-    </div>
+                        {% endif %}
-  </li>
+                      </th>
-{% endblock pagination.previous %}
+                    {% endblock table.thead.th %}
-{% block pagination.range %}
+                  {% endfor %}
-  {% for p in table.page|table_page_range:table.paginator %}
+                </tr>
-    <li class="page-item{% if table.page.number == p %} active{% endif %}">
+              {% endblock table.thead.row %}
-      <div
+            </thead>
-        class="page-link" 
+          {% endif %}
-        {% if p != '...' %}hx-post="{% querystring table.prefixed_page_field=p %}"{% endif %}
+        {% endblock table.thead %}
-        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        {% block table.tbody %}
-        hx-trigger="click"
+          <tbody {{ table.attrs.tbody.as_html }}>
-        hx-target="div.table-container"
+            {% for row in table.paginated_rows %}
-        hx-swap="outerHTML"
+              {% block table.tbody.row %}
-        hx-indicator=".progress">
+                <tr {{ row.attrs.as_html }}>
-        {{ p }}
+                  {% for column, cell in row.items %}
-      </div>
+                    {% block table.tbody.td %}
-    </li>
+                      <td {{ column.attrs.td.as_html }}>{% if column.localize == None %}{{ cell }}{% else %}{% if column.localize %}{{ cell|localize }}{% else %}{{ cell|unlocalize }}{% endif %}{% endif %}</td>
-  {% endfor %}
+                    {% endblock table.tbody.td %}
-{% endblock pagination.range %}
+                  {% endfor %}
-{% block pagination.next %}
+                </tr>
-  <li class="next page-item">
+              {% endblock table.tbody.row %}
-    <div
+            {% empty %}
-      hx-post="{% querystring table.prefixed_page_field=table.page.next_page_number %}"
+              {% if table.empty_text %}
-      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                {% block table.tbody.empty_text %}
-      hx-trigger="click"
+                  <tr><td colspan="{{ table.columns|length }}">{{ table.empty_text }}</td></tr>
-      hx-target="div.table-container"
+                {% endblock table.tbody.empty_text %}
-      hx-swap="outerHTML"
+              {% endif %}
-      hx-indicator=".progress"
+            {% endfor %}
-      class="page-link">
+          </tbody>
-      {% trans 'next' %}
+        {% endblock table.tbody %}
-      <span aria-hidden="true">&raquo;</span>
+        {% block table.tfoot %}
-    </div>
+          {% if table.has_footer %}
-  </li>
+            <tfoot {{ table.attrs.tfoot.as_html }}>
-{% endblock pagination.next %}
+              {% block table.tfoot.row %}
+                <tr>
+                  {% for column in table.columns %}
+                    {% block table.tfoot.td %}
+                      <td {{ column.attrs.tf.as_html }}>{{ column.footer }}</td>
+                    {% endblock table.tfoot.td %}
+                  {% endfor %}
+                </tr>
+              {% endblock table.tfoot.row %}
+            </tfoot>
+          {% endif %}
+        {% endblock table.tfoot %}
+      </table>
+    {% endblock table %}
+    {% block pagination %}
+      {% if table.page and table.paginator.num_pages > 1 %}
+        <nav class="pagination is-justify-content-flex-end" role="navigation" aria-label="pagination">
+          {% block pagination.previous %}
+            <a
+              class="pagination-previous is-flex-grow-0 {% if not table.page.has_previous %}is-hidden-mobile{% endif %}"
+              {% if table.page.has_previous %}
+                hx-get="search/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
+                hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                hx-trigger="click"
+                hx-target="#results"
+                hx-swap="innerHTML"
+                hx-indicator=".progress"
+              {% else %}
+                href="#"
+                disabled
+              {% endif %}
+              style="order:1;">
+              {% block pagination.previous.text %}
+                <span aria-hidden="true">&laquo;</span>
+              {% endblock pagination.previous.text  %}
+            </a>
+          {% endblock pagination.previous %}
+          {% block pagination.next %}
+            <a class="pagination-next is-flex-grow-0 {% if not table.page.has_next %}is-hidden-mobile{% endif %}"
+               {% if table.page.has_next %}
+                 hx-get="search/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
+                 hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                 hx-trigger="click"
+                 hx-target="#results"
+                 hx-swap="innerHTML"
+                 hx-indicator=".progress"
+               {% else %}
+                 href="#"
+                 disabled
+               {% endif %}
+               style="order:3;"
+              >
+              {% block pagination.next.text %}
+                <span aria-hidden="true">&raquo;</span>
+              {% endblock pagination.next.text  %}
+            </a>
+          {% endblock pagination.next %}
+          {% if table.page.has_previous or table.page.has_next %}
+            {% block pagination.range %}
+              <ul class="pagination-list is-flex-grow-0" style="order:2;">
+                {% for p in table.page|table_page_range:table.paginator %}
+                  <li>
+                    <a class="pagination-link {% if p == table.page.number %}is-current{% endif %}"
+                       aria-label="Page {{ p }}" block
+                       {% if p == table.page.number %}aria-current="page"{% endif %}
+                       {% if p == table.page.number %}
+                         href="#"
+                       {% else %}
+                         hx-get="search/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
+                         hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                         hx-trigger="click"
+                         hx-target="#results"
+                         hx-swap="innerHTML"
+                         hx-indicator=".progress"
+                       {% endif %}
+                  >
+                      {% if p == '...' %}
+                        <span class="pagination-ellipsis">&hellip;</span>
+                      {% else %}
+                        {{ p }}
+                      {% endif %}
+                    </a>
+                  </li>
+                {% endfor %}
+              </ul>
+            {% endblock pagination.range %}
+          {% endif %}
+        </nav>
+      {% endif %}
+    {% endblock pagination %}
+  </div>
+{% endblock table-wrapper %}

core/views/ui/drilldown.py

@@ -6,7 +6,7 @@ from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
 from django.urls import reverse
 from django.views import View
-from django_tables2 import SingleTableMixin
+from django_tables2 import SingleTableView
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
@@ -20,30 +20,6 @@ from core.lib.threshold import (
 from core.views.ui.tables import DrilldownTable
 
 
-class DrilldownTableView(View, SingleTableMixin):
-    table_class = DrilldownTable
-    template_name = "ui/drilldown/table_results.html"
-    paginate_by = 5
-
-    def post(self, request):
-        context = query_results(request)
-        table = DrilldownTable(context["results"])
-        context["table"] = table
-        del context["results"]
-        if "message" in context:
-            return render(request, self.template_name, context)
-
-        if self.request.htmx:
-            template_name = "ui/drilldown/table_results.html"
-        else:
-            template_name = "ui/drilldown/table_results_partial.html"
-
-        if context:
-            return render(request, template_name, context)
-        else:
-            return HttpResponse("No results")
-
-
 def parse_dates(dates):
     spl = dates.split(" - ")
     if all(spl):
@@ -76,12 +52,52 @@ def create_tags(query):
     return tags
 
 
-def drilldown_search(request):
+def make_table(context):
-    template_name = "ui/drilldown/results.html"
+    table = DrilldownTable(context["object_list"])
+    context["table"] = table
+    # del context["results"]
+    return context
+
+
+def make_graph(results):
+    graph = json.dumps(
+        [
+            {
+                "text": item.get("msg", None) or item.get("id"),
+                "nick": item.get("nick", None),
+                "value": item.get("sentiment", None) or None,
+                "date": item.get("ts"),
+            }
+            for item in results
+        ]
+    )
+    return graph
+
+
+def drilldown_search(request, return_context=False, template=None):
+    if not template:
+        template_name = "ui/drilldown/table_results.html"
+    else:
+        template_name = template
+    if request.user.is_anonymous:
+        sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
+    else:
+        sizes = settings.OPENSEARCH_MAIN_SIZES
+
     if request.GET:
+        print("GET")
+        if not request.htmx:
+            print("NOT REQUEST HTMX")
+            template_name = "ui/drilldown/drilldown.html"
         query_params = request.GET.dict()
     elif request.POST:
+        print("POST")
         query_params = request.POST.dict()
+    else:
+        print("ELSE")
+        template_name = "ui/drilldown/drilldown.html"
+        context = {"sizes": sizes}
+        return render(request, template_name, context)
 
     # Parse the dates
     if "dates" in query_params:
@@ -99,54 +115,126 @@ def drilldown_search(request):
         context = query_results(request, query_params)
     elif request.POST:
         context = query_results(request, query_params)
+    print("QUERY PARAMS", query_params)
 
     # Turn the query into tags for populating the taglist
-    tags = create_tags(query_params["query"])
+    if "query" in query_params:
-    context["tags"] = tags
+        tags = create_tags(query_params["query"])
+        context["tags"] = tags
 
     context["params"] = query_params
     if "message" in context:
+        print("MESSAGE IN CONTEXT")
         return render(request, template_name, context)
 
-    context["data"] = json.dumps(
+    # Create data for chart.js sentiment graph
-        [
+    graph = make_graph(context["object_list"])
-            {
+    context["data"] = graph
-                "text": item.get("msg", None) or item.get("id"),
+    print("MADE GRAPH")
-                "nick": item.get("nick", None),
+
-                "value": item.get("sentiment", None) or None,
-                "date": item.get("ts"),
-            }
-            for item in context["results"]
-        ]
-    )
     if context:
+        context["sizes"] = sizes
+        context = make_table(context)
+
+        # URI we're passing to the template for linking
+        if "csrfmiddlewaretoken" in query_params:
+            del query_params["csrfmiddlewaretoken"]
+        url_params = urllib.parse.urlencode(query_params)
+        context["client_uri"] = url_params
+
+        # URI we're passing to the template for linking, table fields removed
+        table_fields = ["page", "sort"]
+        clean_params = {k: v for k, v in query_params.items() if k not in table_fields}
+        clean_url_params = urllib.parse.urlencode(clean_params)
+        context["uri"] = clean_url_params
+
         response = render(request, template_name, context)
         if request.GET:
-            return context
+            print("IS GET")
+            if request.htmx:
+                print("IS HTMX PUSHING", url_params)
+                response["HX-Push"] = reverse("home") + "?" + url_params
         elif request.POST:
-            del query_params["csrfmiddlewaretoken"]
+            print("IS POST")
-            url_params = urllib.parse.urlencode(query_params)
+            print("TEMPLATE", template_name)
             response["HX-Push"] = reverse("home") + "?" + url_params
+        if return_context:
+            return context
         return response
     else:
+        print("NO RESULTS", context)
         return HttpResponse("No results")
 
 
+class DrilldownTableView(SingleTableView):
+    table_class = DrilldownTable
+    template_name = "ui/drilldown/table_results.html"
+    paginate_by = 5
+
+    def get_queryset(self, request, **kwargs):
+        print("QUERYSET KWARGS", kwargs)
+        context = drilldown_search(request, return_context=True)
+        # Save the context as we will need to merge other attributes later
+        self.context = context
+        print(
+            "VIEW CONTEXT",
+            {k: v for k, v in context.items() if k not in ["object_list", "data"]},
+        )
+        if "object_list" in context:
+            return context["object_list"]
+        else:
+            print("NO OBJ LIST IN CONTEXT", context)
+            return []
+
+    def get(self, request, *args, **kwargs):
+        self.object_list = self.get_queryset(request)
+        allow_empty = self.get_allow_empty()
+
+        if not allow_empty:
+            # When pagination is enabled and object_list is a queryset,
+            # it's better to do a cheap query than to load the unpaginated
+            # queryset in memory.
+            if self.get_paginate_by(self.object_list) is not None and hasattr(
+                self.object_list, "exists"
+            ):
+                is_empty = not self.object_list.exists()  # noqa
+            else:
+                is_empty = not self.object_list  # noqa
+        context = self.get_context_data()
+        if isinstance(self.context, HttpResponse):
+            return self.context
+        print(
+            "TABLE CONTEXT",
+            {k: v for k, v in context.items() if k not in ["object_list", "data"]},
+        )
+        for k, v in self.context.items():
+            if k not in context:
+                context[k] = v
+        print(
+            "FINAL CONTEXT",
+            {k: v for k, v in context.items() if k not in ["object_list", "data"]},
+        )
+        if request.method == "GET":
+            if not request.htmx:
+                print("NOT REQUEST HTMX")
+                self.template_name = "ui/drilldown/drilldown.html"
+        response = self.render_to_response(context)
+        # if not request.method == "GET":
+        if "client_uri" in context:
+            print("PUSHING CLIENT URI", context["client_uri"])
+            response["HX-Push"] = reverse("home") + "?" + context["client_uri"]
+        return response
+
+    def post(self, request, *args, **kwargs):
+        return self.get(request, *args, **kwargs)
+
+
 class Drilldown(View):
     template_name = "ui/drilldown/drilldown.html"
     plan_name = "drilldown"
 
     def get(self, request):
-        if request.user.is_anonymous:
+        return drilldown_search(request)
-            sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
-        else:
-            sizes = settings.OPENSEARCH_MAIN_SIZES
-
-        context = {}
-        if request.GET:
-            context = drilldown_search(request)
-        context["sizes"] = sizes
-        return render(request, self.template_name, context)
 
     def post(self, request):
         return drilldown_search(request)

core/views/ui/tables.py

@@ -32,3 +32,5 @@ class DrilldownTable(Table):
     status = Column()
     user = Column()
     version_sentiment = Column()
+    template_name = "ui/drilldown/table_results.html"
+    paginate_by = 5