db.py

@@ -41,8 +41,8 @@ def store_message(msg):
     # print(body_post)
     try:
         # Bulk index operations
-        api_instance.bulk(body_post, async_req=True)
-        # print(api_response)
+        api_response = api_instance.bulk(body_post)  # , async_req=True
+        print(api_response)
     except ApiException as e:
         print("Exception when calling IndexApi->bulk: %s\n" % e)
 
@@ -82,8 +82,8 @@ def store_message_bulk(data):
         # print(body_post)
         try:
             # Bulk index operations
-            api_instance.bulk(body_post, async_req=True)
-            # print(api_response)
+            api_response = api_instance.bulk(body_post)  # , async_req=True
+            print(api_response)
         except ApiException as e:
             print("Exception when calling IndexApi->bulk: %s\n" % e)
         print("FINISHED PROCESSING SPLIT")

sources/ch4.py

@@ -4,21 +4,18 @@ import random
 import string
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
+from math import ceil
 
 import aiohttp
 import ujson
 from bs4 import BeautifulSoup
+from numpy import array_split
 from siphashc import siphash
 
 import db
 import util
 from schemas.ch4_s import ATTRMAP
 
-from numpy import array_split
-from math import ceil
-
-
-
 # CONFIGURATION #
 
 # Number of 4chan threads to request at once
@@ -40,6 +37,7 @@ CPU_THREADS = 2
 
 p = ProcessPoolExecutor(CPU_THREADS)
 
+
 class Chan4(object):
     """
     4chan indexer, crawler and ingester.
@@ -119,7 +117,7 @@ class Chan4(object):
                     i += 1
                     if i == n:
                         raise StopIteration
-        except:
+        except StopIteration:
             print("Take items took", i, "items")
 
     async def get_threads_content(self, thread_list):
@@ -213,6 +211,9 @@ class Chan4(object):
                     posts[key][index]["msg"] = msg
 
                 posts[key][index]["src"] = "4ch"
+                posts[key][index]["net"] = board
+                posts[key][index]["channel"] = thread
+
                 to_store.append(posts[key][index])
 
             # print({name_map[name]: val for name, val in post.items()})