Begin switching away from Redis

db.py

@@ -5,7 +5,7 @@ import orjson
 
 # Kafka
 from aiokafka import AIOKafkaProducer
-from redis import StrictRedis
+import redis
 
 import util
 
@@ -14,10 +14,10 @@ import util
 log = util.get_logger("db")
 
 # Redis (legacy)
-r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
+r = redis.from_url("redis://ssdb:1289", db=0)
 
 # AIORedis
-ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
+ar = aioredis.from_url("redis://ssdb:1289", db=0)
 
 TYPES_MAIN = [
     "msg",
@@ -116,7 +116,7 @@ async def queue_message(msg):
     Queue a message on the Redis buffer.
     """
     message = orjson.dumps(msg)
-    await ar.sadd(KEYNAME, message)
+    await ar.zadd(KEYNAME, message)
 
 
 async def queue_message_bulk(data):
@@ -125,4 +125,4 @@ async def queue_message_bulk(data):
     """
     for msg in data:
         message = orjson.dumps(msg)
-        await ar.sadd(KEYNAME, message)
+        await ar.zadd(KEYNAME, message)

docker/docker-compose.prod.yml

@@ -4,22 +4,22 @@ services:
   app:
     image: pathogen/monolith:latest
     container_name: monolith
-    build: ./docker
+    build: ${PORTAINER_GIT_DIR}/docker
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
     env_file:
       - ../stack.env
-    volumes_from:
-      - tmp
-    depends_on:
+    # volumes_from:
+    #   - tmp
+    # depends_on:
       # broker:
       #   condition: service_started
       # kafka:
       #   condition: service_healthy
-      tmp:
-        condition: service_started
-      redis:
-        condition: service_healthy
+      # tmp:
+      #   condition: service_started
+      # redis:
+      #   condition: service_healthy
 
   threshold:
     image: pathogen/threshold:latest
@@ -39,40 +39,16 @@ services:
     # for development
     extra_hosts:
       - "host.docker.internal:host-gateway"
-    volumes_from:
-      - tmp
-    depends_on:
-      tmp:
-        condition: service_started
-      redis:
-        condition: service_healthy
 
-  tmp:
-    image: busybox
-    container_name: tmp_monolith
-    command: chmod -R 777 /var/run/redis
+  ssdb:
+    image: tsl0922/ssdb
+    container_name: ssdb_monolith
     volumes:
-        - /var/run/redis
-
-  redis:
-    image: redis
-    container_name: redis_monolith
-    command: redis-server /etc/redis.conf
-    ulimits:
-      nproc: 65535
-      nofile:
-         soft: 65535
-         hard: 65535
-    volumes:
-      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
-      - redis_data:/data
-    volumes_from:
-      - tmp
-    healthcheck:
-      test: "redis-cli -s /var/run/redis/redis.sock ping"
-      interval: 2s
-      timeout: 2s
-      retries: 15
+      - ssdb_data:/ssdb/var
+    ports:
+      - "1289:1289"
+    environment:
+      - SSDB_PORT=1289
 
 networks:
    default:
@@ -80,4 +56,4 @@ networks:
          name: pathogen
 
 volumes:
-  redis_data: {}
+  ssdb_data: {}

sources/ingest.py

@@ -36,7 +36,7 @@ class Ingest(object):
         items = []
         # for source in SOURCES:
         # key = f"{KEYPREFIX}{source}"
-        chunk = await db.ar.spop(KEYNAME, CHUNK_SIZE)
+        chunk = await db.ar.zpop(KEYNAME, CHUNK_SIZE)
         if not chunk:
             return
         for item in chunk: