Add total hits to output

core/db/elastic.py

@@ -300,8 +300,8 @@ class ElasticsearchBackend(StorageBackend):
                 search_query,
                 index=index,
             )
-            self.log.debug(f"Running scheduled query on {index}: {search_query}")
-            self.log.debug(f"Response from scheduled query: {response}")
+            # self.log.debug(f"Running scheduled query on {index}: {search_query}")
+            # self.log.debug(f"Response from scheduled query: {response}")
             if isinstance(response, Exception):
                 error = response.info["error"]["root_cause"][0]["reason"]
                 self.log.error(f"Error running scheduled search: {error}")
@@ -310,7 +310,7 @@ class ElasticsearchBackend(StorageBackend):
                 # No results, skip
                 continue
             meta, response = self.parse(response, meta=True)
-            print("Parsed response", response)
+            # print("Parsed response", response)
             if "message" in response:
                 self.log.error(f"Error running scheduled search: {response['message']}")
                 continue

core/db/processing.py

@@ -125,6 +125,8 @@ def parse_results(results, meta=None):
             for field in ["avg_sentiment"]:  # Add other number fields here
                 if field in results["aggregations"]:
                     meta["aggs"][field] = results["aggregations"][field]
+        total_hits = results["hits"]["total"]["value"]
+        meta["total_hits"] = total_hits
         return (meta, results_parsed)
 
     return results_parsed