Send messages to Neptune Redis via PubSub

db.py

@@ -1,6 +1,7 @@
 from os import getenv
 
 import aioredis
+import msgpack
 import orjson
 import redis
 
@@ -21,6 +22,10 @@ r = redis.from_url("redis://ssdb:1289", db=0)
 # AIORedis
 ar = aioredis.from_url("redis://ssdb:1289", db=0)
 
+# Neptune redis for PubSub
+pr = aioredis.from_url("redis://redis_neptune:6379", db=10)
+
+
 TYPES_MAIN = [
     "msg",
     "notice",
@@ -43,6 +48,7 @@ MAIN_SRC_MAP = {
 TYPES_META = ["who"]
 TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
 KEYNAME = "queue"
+MESSAGE_KEY = "messages"
 
 ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "elastic")
 ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "changeme")
@@ -127,6 +133,10 @@ async def store_batch(data):
         else:
             indexmap[INDEX].append(msg)
 
+    # Pack the indexmap with msgpack and publish it to Neptune
+    packed_index = msgpack.packb(indexmap, use_bin_type=True)
+    await pr.publish(MESSAGE_KEY, packed_index)
+
     for index, index_messages in indexmap.items():
         for message in index_messages:
             result = await client.index(index=index, body=message)
@@ -139,6 +149,7 @@ async def queue_message(msg):
     """
     Queue a message on the Redis buffer.
     """
+    # TODO: msgpack
     message = orjson.dumps(msg)
     await ar.lpush(KEYNAME, message)
 
@@ -148,5 +159,6 @@ async def queue_message_bulk(data):
     Queue multiple messages on the Redis buffer.
     """
     for msg in data:
+        # TODO: msgpack
         message = orjson.dumps(msg)
         await ar.lpush(KEYNAME, message)