Port Manticore and debug

core/db/__init__.py

@@ -253,7 +253,6 @@ class StorageBackend(ABC):
                     "took": time_took_rounded,
                     "cache": True,
                 }
-        print("S2", search_query)
         response = self.run_query(user, search_query, **kwargs)
 
         # For Elasticsearch

core/db/manticore.py

@@ -57,7 +57,6 @@ class ManticoreBackend(StorageBackend):
             "limit": size,
             "query": {"bool": {"must": []}},
         }
-        print("BASE", query_base)
         query_string = {
             "query_string": query,
         }
@@ -77,7 +76,6 @@ class ManticoreBackend(StorageBackend):
         raw = kwargs.get("raw")
         if search_query and not raw:
             search_query["index"] = index
-            pprint(search_query)
 
 
         path = kwargs.get("path", "json/search")
@@ -98,7 +96,6 @@ class ManticoreBackend(StorageBackend):
         """
         index = kwargs.get("index")
         search_query["index"] = index
-        pprint(search_query)
         async with httpx.AsyncClient() as client:
             response = await client.post(
                 f"{settings.MANTICORE_URL}/json/search", json=search_query

core/db/processing.py

@@ -94,7 +94,7 @@ def parse_results(results, meta=None):
                         del element[field]
                 # Unfold the tokens
                 if "tokens" in element:
-                    if element["tokens"].startswith('["'):
+                    if element["tokens"].startswith('["') or element["tokens"].startswith("['"):
                         tokens_parsed = ast.literal_eval(element["tokens"])
                         element["tokens"] = tokens_parsed
 

core/lib/context.py

@@ -85,5 +85,4 @@ def construct_query(index, net, channel, src, num, size, type=None, nicks=None):
     if extra_should2:
         query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should2]}})
 
-    print("CONTEXT QUERY", query)
     return query

core/lib/parsing.py

@@ -91,7 +91,6 @@ def parse_index(user, query_params, raise_error=False):
     else:
         index = settings.INDEX_MAIN
 
-    print("GOT INDEX", index)
     return index
 
 

core/management/commands/scheduling.py

@@ -44,8 +44,11 @@ class Command(BaseCommand):
         for interval in INTERVALS:
             log.debug(f"Scheduling {interval} second job")
             scheduler.add_job(job, "interval", seconds=interval, args=[interval])
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        scheduler._eventloop = loop
         scheduler.start()
-        loop = asyncio.get_event_loop()
         try:
             loop.run_forever()
         except (KeyboardInterrupt, SystemExit):

core/templates/partials/sentiment_chart.html

@@ -3,7 +3,7 @@
 </div>
 
 {% if params.index != 'int' and params.index != 'meta' %}
-  <div id="sentiment-container" {% if params.show_sentiment is None %} class="is-hidden" {% endif %}>
+  <div id="sentiment-container" {% if params.graph is None %} class="is-hidden" {% endif %}>
     <canvas id="sentiment-chart"></canvas>
   </div>
   <script src="{% static 'chart.js' %}"></script>

core/templates/window-content/search.html

@@ -258,7 +258,7 @@
               id="sentiment_graph_switch"
               type="checkbox"
               class="switch is-rounded is-info"
-              name="show_sentiment"
+              name="graph"
               data-script="on click toggle .is-hidden on #sentiment-container">
 
             <label

core/views/ui/drilldown.py

@@ -119,16 +119,20 @@ class DrilldownTableView(SingleTableView):
             sizes = settings.MAIN_SIZES
 
         if request.GET:
+            print("GET")
             self.template_name = "index.html"
             # GET arguments in URL like ?query=xyz
             query_params = request.GET.dict()
+            print("QUERY_PARAMS GET", query_params)
             if request.htmx:
                 if request.resolver_match.url_name == "search_partial":
                     self.template_name = "partials/results_table.html"
         elif request.POST:
+            print("POST")
             query_params = request.POST.dict()
         else:
             self.template_name = "index.html"
+            print("FRESH")
             # No query, this is a fresh page load
             # Don't try to search, since there's clearly nothing to do
             params_with_defaults = {}

docker/redis.conf

@@ -1,4 +1,5 @@
-# unixsocket /var/run/socks/redis.sock
-# unixsocketperm 777
-port 6379
-requirepass changeme
+unixsocket /var/run/neptune-redis.sock
+unixsocketperm 777
+port 0
+# port 6379
+# requirepass changeme

docker/uwsgi.ini

@@ -4,9 +4,19 @@ module=app.wsgi:application
 env=DJANGO_SETTINGS_MODULE=app.settings
 master=1
 pidfile=/tmp/project-master.pid
-socket=0.0.0.0:8000
+#socket=0.0.0.0:8000
+socket=/var/run/uwsgi-neptune.sock
+# socket 777
+chmod-socket=777
 harakiri=20
-max-requests=100000
+#max-requests=100000
+# Set a lower value for max-requests to prevent memory leaks from building up over time
+max-requests=1000
+# Ensure old worker processes are cleaned up properly
+reload-on-as=512
+reload-on-rss=256
 vacuum=1
 home=/venv
-processes=12
+processes=4
+threads=2
+log-level=debug