Make crawler more efficient and implement configurable parameters

db.py

@@ -55,6 +55,8 @@ def store_message_bulk(data):
     print("BULK", len(data))
     if not data:
         return
+    # 10000: maximum inserts we can submit to
+    # Manticore as of Sept 2022
     split_posts = array_split(data, ceil(len(data) / 10000))
     for messages in split_posts:
         print("PROCESSING SPLIT OF", len(messages), "MESSAGES")