Improve context passing and implement superuser override for redactions

core/lib/opensearch.py

@@ -25,13 +25,13 @@ def initialise_opensearch():
     return client
 
 
-def construct_query(query, fields, results):
+def construct_query(query, fields, size):
     if not fields:
         fields = settings.OPENSEARCH_MAIN_SEARCH_FIELDS
-    if not results:
-        results = 5
+    if not size:
+        size = 5
     query = {
-        "size": results,
+        "size": size,
         "query": {
             "query_string": {
                 "query": query,
@@ -61,7 +61,7 @@ def construct_query(query, fields, results):
     return query
 
 
-def filter_blacklisted(response):
+def filter_blacklisted(user, response):
     pp.pprint(response["hits"]["hits"])
     print("LEN", len(response["hits"]["hits"]))
     response["redacted"] = 0
@@ -79,15 +79,24 @@ def filter_blacklisted(response):
                     if blacklisted_item in str(content):
                         # Remove the item
                         if item in response["hits"]["hits"]:
-                            response["hits"]["hits"].remove(item)
+                            if not user.is_superuser:
+                                response["hits"]["hits"].remove(item)
                             # Let the UI know something was redacted
                             response["redacted"] += 1
+                            response["exemption"] = True
 
 
-def run_main_query(client, query, fields=None, results=None):
-    search_query = construct_query(query, fields, results)
+def run_main_query(client, user, query, fields=None, size=None):
+    if fields:
+        for field in fields:
+            if field not in settings.OPENSEARCH_MAIN_SEARCH_FIELDS:
+                return False
+    if size:
+        if size not in settings.OPENSEARCH_MAIN_SIZES:
+            return False
+    search_query = construct_query(query, fields, size)
     # fmt: off
     response = client.search(body=search_query,
                              index=settings.OPENSEARCH_INDEX_MAIN)
-    filter_blacklisted(response)
+    filter_blacklisted(user, response)
     return response

core/templates/ui/drilldown.html

@@ -27,12 +27,9 @@
               <div class="control is-expanded has-icons-left">
                 <div class="select is-fullwidth">
                   <select name="timescale">
-                    <option value="minute">Minute</option>
-                    <option value="hour">Hour</option>
-                    <option value="day">Day</option>
-                    <option value="week">Week</option>
-                    <option value="month">Month</option>
-                    <option value="6months">6 months</option>
+                    {% for timescale in timescales %}
+                      <option value="{{ timescale }}">{{ timescale }}</option>
+                    {% endfor %}
                   </select>
                   <span class="icon is-small is-left">
                     <i class="fas fa-magnifying-glass"></i>
@@ -65,14 +62,10 @@
           <div class="field">
             <div class="control is-expanded has-icons-left">
               <div class="select is-fullwidth">
-                <select name="results">
-                  <option value="5">5</option>
-                  <option value="10">10</option>
-                  <option value="15">15</option>
-                  <option value="20">20</option>
-                  <option value="50">50</option>
-                  <option value="100">100</option>
-                  <option value="200">200</option>
+                <select name="size">
+                  {% for size in sizes %}
+                    <option value="{{ size }}">{{ size }}</option>
+                  {% endfor %}
                 </select>
                 <span class="icon is-small is-left">
                   <i class="fas fa-magnifying-glass"></i>
@@ -127,6 +120,9 @@
     </div>
     <p>{{ card }} hits</p>
     <p>{{ redacted }} redacted</p>
+    {% if exemption is not None %}
+      <p>redaction overriden by superuser</p>
+    {% endif %}
     <p>{{ took }}ms</p>
   {% endif %}
 

core/ui/views/drilldown.py

@@ -18,7 +18,11 @@ class Drilldown(LoginRequiredMixin, View):
     def get(self, request):
         if not request.user.has_plan(self.plan_name):
             return render(request, "denied.html")
-        context = {"fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS}
+        context = {
+            "fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS,
+            "sizes": settings.OPENSEARCH_MAIN_SIZES,
+            "timescales": settings.OPENSEARCH_MAIN_TIMESCALES,
+        }
         return render(request, self.template_name, context)
 
     def post(self, request):
@@ -27,13 +31,15 @@ class Drilldown(LoginRequiredMixin, View):
         fields = None
         if "fields" in request.POST:
             fields = request.POST.getlist("fields")
-        if "results" in request.POST:
-            results = request.POST["results"]
+        if "size" in request.POST:
+            size = request.POST["size"]
         if "query" in request.POST:
             query = request.POST["query"]
             # field = results.POST["field"]
             # print("FIELD ", field)
-            results = run_main_query(client, query, fields, results)
+            results = run_main_query(client, request.user, query, fields, size)
+            if not results:
+                return render(request, "denied.html")
             # pp.pprint(results)
             results_parsed = []
             if "hits" in results.keys():
@@ -46,7 +52,10 @@ class Drilldown(LoginRequiredMixin, View):
                 "card": results["hits"]["total"]["value"],
                 "took": results["took"],
                 "redacted": results["redacted"],
+                "exemption": results["exemption"],
                 "fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS,
+                "sizes": settings.OPENSEARCH_MAIN_SIZES,
+                "timescales": settings.OPENSEARCH_MAIN_TIMESCALES,
             }
             return render(request, self.template_name, context)
         return render(request, self.template_name)