|
|
|
@ -40,7 +40,7 @@ MAIN_SRC_MAP = {
|
|
|
|
|
|
|
|
|
|
TYPES_META = ["who"]
|
|
|
|
|
TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
|
|
|
|
|
KEYPREFIX = "queue."
|
|
|
|
|
KEYNAME = "queue"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def store_kafka_batch(data):
|
|
|
|
@ -113,12 +113,12 @@ async def queue_message(msg):
|
|
|
|
|
"""
|
|
|
|
|
Queue a message on the Redis buffer.
|
|
|
|
|
"""
|
|
|
|
|
src = msg["src"]
|
|
|
|
|
# src = msg["src"]
|
|
|
|
|
message = orjson.dumps(msg)
|
|
|
|
|
|
|
|
|
|
key = f"{KEYPREFIX}{src}"
|
|
|
|
|
# key = f"{KEYPREFIX}{src}"
|
|
|
|
|
# log.debug(f"Queueing single message of string length {len(message)}")
|
|
|
|
|
await ar.sadd(key, message)
|
|
|
|
|
await ar.sadd(KEYNAME, message)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def queue_message_bulk(data):
|
|
|
|
@ -127,8 +127,8 @@ async def queue_message_bulk(data):
|
|
|
|
|
"""
|
|
|
|
|
# log.debug(f"Queueing message batch of length {len(data)}")
|
|
|
|
|
for msg in data:
|
|
|
|
|
src = msg["src"]
|
|
|
|
|
# src = msg["src"]
|
|
|
|
|
message = orjson.dumps(msg)
|
|
|
|
|
|
|
|
|
|
key = f"{KEYPREFIX}{src}"
|
|
|
|
|
await ar.sadd(key, message)
|
|
|
|
|
# key = f"{KEYPREFIX}{src}"
|
|
|
|
|
await ar.sadd(KEYNAME, message)
|
|
|
|
|