Begin switching away from Redis

Makefile

@@ -0,0 +1,20 @@
+run:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env up -d
+
+build:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env build
+
+stop:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env down
+
+log:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env logs -f
+
+run-infra:
+	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env up -d
+
+stop-infra:
+	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env down
+
+log-infra:
+	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env logs -f

db.py

@@ -5,7 +5,7 @@ import orjson
 
 # Kafka
 from aiokafka import AIOKafkaProducer
-from redis import StrictRedis
+import redis
 
 import util
 
@@ -14,10 +14,10 @@ import util
 log = util.get_logger("db")
 
 # Redis (legacy)
-r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
+r = redis.from_url("redis://ssdb:1289", db=0)
 
 # AIORedis
-ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
+ar = aioredis.from_url("redis://ssdb:1289", db=0)
 
 TYPES_MAIN = [
     "msg",
@@ -44,6 +44,8 @@ KEYNAME = "queue"
 
 
 async def store_kafka_batch(data):
+    print("FAKE STORE KAFKA BATCH")
+    return
     # log.debug(f"Storing Kafka batch of {len(data)} messages")
     producer = AIOKafkaProducer(bootstrap_servers="kafka:9092")
     await producer.start()
@@ -114,7 +116,7 @@ async def queue_message(msg):
     Queue a message on the Redis buffer.
     """
     message = orjson.dumps(msg)
-    await ar.sadd(KEYNAME, message)
+    await ar.zadd(KEYNAME, message)
 
 
 async def queue_message_bulk(data):
@@ -123,4 +125,4 @@ async def queue_message_bulk(data):
     """
     for msg in data:
         message = orjson.dumps(msg)
-        await ar.sadd(KEYNAME, message)
+        await ar.zadd(KEYNAME, message)

docker-compose.yml

@@ -6,7 +6,7 @@ x-superset-depends-on: &superset-depends-on
   - redis_superset
 x-superset-volumes: &superset-volumes
   # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
-  - ./docker/superset:/app/docker
+  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
   - superset_home:/app/superset_home
 
 services:
@@ -21,7 +21,7 @@ services:
     volumes_from:
       - tmp
     depends_on:
-      broker:
+      druid:
         condition: service_started
       kafka:
         condition: service_healthy
@@ -100,7 +100,7 @@ services:
     container_name: superset_cache
     restart: unless-stopped
     volumes:
-      - redis:/data
+      - redis_superset:/data
   db:
     env_file: docker/.env-non-dev
     image: postgres:10
@@ -195,89 +195,117 @@ services:
       timeout: 30s
       retries: 45
 
-  coordinator:
+  druid:
-    image: apache/druid:0.23.0
+    image: pathogen/druid:0.23.0
-    container_name: coordinator
+    build: ./docker/druid/
+    container_name: druid
     volumes:
       - druid_shared:/opt/shared
-      - coordinator_var:/opt/druid/var
+      - druid_var:/opt/druid/var
     depends_on: 
       - zookeeper
       - postgres
     ports:
       - "8081:8081"
-    command:
-      - coordinator
-    env_file:
-      - environment
-
-  broker:
-    image: apache/druid:0.23.0
-    container_name: broker
-    volumes:
-      - broker_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
       - "8082:8082"
-    command:
-      - broker
-    env_file:
-      - environment
-
-  historical:
-    image: apache/druid:0.23.0
-    container_name: historical
-    volumes:
-      - druid_shared:/opt/shared
-      - historical_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
       - "8083:8083"
-    command:
-      - historical
-    env_file:
-      - environment
-
-  middlemanager:
-    image: apache/druid:0.23.0
-    container_name: middlemanager
-    volumes:
-      - druid_shared:/opt/shared
-      - middle_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
-      - "8091:8091"
-      - "8100-8105:8100-8105"
-    command:
-      - middleManager
-    env_file:
-      - environment
-
-  router:
-    image: apache/druid:0.23.0
-    container_name: router
-    volumes:
-      - router_var:/opt/druid/var
-    depends_on:
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
       - "8888:8888"
-    command:
-      - router
     env_file:
       - environment
 
+  # coordinator:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: coordinator
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - coordinator_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #   ports:
+  #     - "8081:8081"
+  #   command:
+  #     - coordinator
+  #   env_file:
+  #     - environment
+
+  # broker:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: broker
+  #   volumes:
+  #     - broker_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8082:8082"
+  #   command:
+  #     - broker
+  #   env_file:
+  #     - environment
+
+  # historical:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: historical
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - historical_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8083:8083"
+  #   command:
+  #     - historical
+  #   env_file:
+  #     - environment
+
+  # middlemanager:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: middlemanager
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - middle_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8091:8091"
+  #     - "8100-8105:8100-8105"
+  #   command:
+  #     - middleManager
+  #   env_file:
+  #     - environment
+
+  # router:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: router
+  #   volumes:
+  #     - router_var:/opt/druid/var
+  #   depends_on:
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8888:8888"
+  #   command:
+  #     - router
+  #   env_file:
+  #     - environment
+
   # db:
   #   #image: pathogen/manticore:kibana
   #   image: manticoresearch/manticore:dev
@@ -343,9 +371,9 @@ volumes:
     external: false
   redis_data: {}
   metadata_data: {}
-  middle_var: {}
+  # middle_var: {}
-  historical_var: {}
+  # historical_var: {}
-  broker_var: {}
+  # broker_var: {}
-  coordinator_var: {}
+  # coordinator_var: {}
-  router_var: {}
+  druid_var: {}
   druid_shared: {}

docker/.env-non-dev

@@ -41,6 +41,6 @@ REDIS_PORT=6379
 
 FLASK_ENV=production
 SUPERSET_ENV=production
-SUPERSET_LOAD_EXAMPLES=yes
+SUPERSET_LOAD_EXAMPLES=no
 CYPRESS_CONFIG=false
 SUPERSET_PORT=8088

docker/docker-compose.infra.yml

@@ -0,0 +1,142 @@
+version: "2.2"
+
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis_superset
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
+  - superset_home:/app/superset_home
+
+services:
+  redis_superset:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis:/data
+  db:
+    env_file: .env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: .env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: .env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - /block/store/metadata_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOO_MY_ID=1
+
+  kafka:
+    image: wurstmeister/kafka:latest
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_MESSAGE_MAX_BYTES: 2000000
+      #KAFKA_HEAP_OPTS: -Xmx2g
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
+      start_period: 15s
+      interval: 30s
+      timeout: 30s
+      retries: 45
+
+  druid:
+    image: pathogen/druid:0.23.0
+    build: ./druid/
+    container_name: druid
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/druid_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+      - "8082:8082"
+      - "8083:8083"
+      - "8888:8888"
+    env_file:
+      - environment
+
+networks:
+   default:
+      external:
+         name: pathogen
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker/docker-compose.prod.yml

@@ -1,44 +1,25 @@
 version: "2.2"
 
-# volumes:
-#   metadata_data: {}
-#   middle_var: {}
-#   historical_var: {}
-#   broker_var: {}
-#   coordinator_var: {}
-#   router_var: {}
-#   druid_shared: {}
-
-x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
-x-superset-depends-on: &superset-depends-on
-  - db
-  - redis_superset
-x-superset-volumes: &superset-volumes
-  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
-  - ./docker/superset:/app/docker
-  - superset_home:/app/superset_home
-
 services:
   app:
     image: pathogen/monolith:latest
     container_name: monolith
-    build: ./docker
+    build: ${PORTAINER_GIT_DIR}/docker
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
     env_file:
       - ../stack.env
-    volumes_from:
+    # volumes_from:
-      - tmp
+    #   - tmp
-    depends_on:
+    # depends_on:
-      broker:
+      # broker:
-        condition: service_started
+      #   condition: service_started
-      kafka:
+      # kafka:
-        condition: service_healthy
+      #   condition: service_healthy
-      tmp:
+      # tmp:
-        condition: service_started
+      #   condition: service_started
-      redis:
+      # redis:
-        condition: service_healthy
+      #   condition: service_healthy
-    #   - db
 
   threshold:
     image: pathogen/threshold:latest
@@ -55,283 +36,19 @@ services:
       - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
     env_file:
       - ../stack.env
-    volumes_from:
+    # for development
-        - tmp
+    extra_hosts:
-    depends_on:
+      - "host.docker.internal:host-gateway"
-      tmp:
-        condition: service_started
-      redis:
-        condition: service_healthy
 
-  # db:
+  ssdb:
-    #image: pathogen/manticore:kibana
+    image: tsl0922/ssdb
-    # image: manticoresearch/manticore:latest
+    container_name: ssdb_monolith
-    #build: 
-    #  context: ./docker/manticore
-    #  args:
-    #    DEV: 1
-    # restart: always
-
-
-  # turnilo:
-  #   container_name: turnilo
-  #   image: uchhatre/turnilo:latest
-  #   ports:
-  #     - 9093:9090
-  #   environment:
-  #     - DRUID_BROKER_URL=http://broker:8082
-  #   depends_on:
-  #     - broker
-
-  # metabase:
-  #   container_name: metabase
-  #   image: metabase/metabase:latest
-  #   ports:
-  #     - 3096:3000
-  #   environment:
-  #     JAVA_OPTS: -Xmx1g
-  #     MB_DB_TYPE: postgres
-  #     MB_DB_DBNAME: metabase
-  #     MB_DB_PORT: 5432
-  #     MB_DB_USER: druid
-  #     MB_DB_PASS: FoolishPassword
-  #     MB_DB_HOST: postgres
-  #   depends_on:
-  #     - broker
-
-  redis_superset:
-    image: redis:latest
-    container_name: superset_cache
-    restart: unless-stopped
     volumes:
-      - redis:/data
+      - ssdb_data:/ssdb/var
-  db:
-    env_file: .env-non-dev
-    image: postgres:10
-    container_name: superset_db
-    restart: unless-stopped
-    volumes:
-      - db_home:/var/lib/postgresql/data
-
-  superset:
-    env_file: .env-non-dev
-    image: *superset-image
-    container_name: superset_app
-    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
-    user: "root"
-    restart: unless-stopped
     ports:
-      - 8088:8088
+      - "1289:1289"
-    depends_on: *superset-depends-on
-    volumes: *superset-volumes
-
-  superset-init:
-    image: *superset-image
-    container_name: superset_init
-    command: ["/app/docker/docker-init.sh"]
-    env_file: .env-non-dev
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  superset-worker:
-    image: *superset-image
-    container_name: superset_worker
-    command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: .env-non-dev
-    restart: unless-stopped
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  superset-worker-beat:
-    image: *superset-image
-    container_name: superset_worker_beat
-    command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: .env-non-dev
-    restart: unless-stopped
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  postgres:
-    container_name: postgres
-    image: postgres:latest
-    volumes:
-      - /block/store/metadata_data:/var/lib/postgresql/data
     environment:
-      - POSTGRES_PASSWORD=FoolishPassword
+      - SSDB_PORT=1289
-      - POSTGRES_USER=druid
-      - POSTGRES_DB=druid
-
-  # Need 3.5 or later for container nodes
-  zookeeper:
-    container_name: zookeeper
-    image: zookeeper:3.5
-    ports:
-      - "2181:2181"
-    environment:
-      - ZOO_MY_ID=1
-
-  kafka:
-    image: wurstmeister/kafka:latest
-    container_name: kafka
-    depends_on:
-      - zookeeper
-    ports:
-      - 9092:9092
-      - 29092:29092
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      KAFKA_MESSAGE_MAX_BYTES: 2000000
-      #KAFKA_HEAP_OPTS: -Xmx2g
-    healthcheck:
-      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
-      start_period: 15s
-      interval: 30s
-      timeout: 30s
-      retries: 45
-
-  coordinator:
-    image: apache/druid:0.23.0
-    container_name: coordinator
-    volumes:
-      - /block/store/druid_shared:/opt/shared
-      - /block/store/coordinator_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-    ports:
-      - "8081:8081"
-    command:
-      - coordinator
-    env_file:
-      - environment
-
-  broker:
-    image: apache/druid:0.23.0
-    container_name: broker
-    volumes:
-      - /block/store/broker_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
-      - "8082:8082"
-    command:
-      - broker
-    env_file:
-      - environment
-
-  historical:
-    image: apache/druid:0.23.0
-    container_name: historical
-    volumes:
-      - /block/store/druid_shared:/opt/shared
-      - /block/store/historical_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
-      - "8083:8083"
-    command:
-      - historical
-    env_file:
-      - environment
-
-  middlemanager:
-    image: apache/druid:0.23.0
-    container_name: middlemanager
-    volumes:
-      - /block/store/druid_shared:/opt/shared
-      - /block/store/middle_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
-      - "8091:8091"
-      - "8100-8105:8100-8105"
-    command:
-      - middleManager
-    env_file:
-      - environment
-
-  router:
-    image: apache/druid:0.23.0
-    container_name: router
-    volumes:
-      - /block/store/router_var:/opt/druid/var
-    depends_on:
-      - zookeeper
-      - postgres
-      - coordinator
-    ports:
-      - "8888:8888"
-    command:
-      - router
-    env_file:
-      - environment
-
-  # db:
-  #   #image: pathogen/manticore:kibana
-  #   image: manticoresearch/manticore:dev
-  #   #build: 
-  #   #  context: ./docker/manticore
-  #   #  args:
-  #   #    DEV: 1
-  #   restart: always
-  #   ports:
-  #     - 9308
-  #     - 9312
-  #     - 9306
-  #   ulimits:
-  #     nproc: 65535
-  #     nofile:
-  #        soft: 65535
-  #        hard: 65535
-  #     memlock:
-  #       soft: -1
-  #       hard: -1
-  #   environment:
-  #     - MCL=1
-  #   volumes:
-  #     - ./docker/data:/var/lib/manticore
-  #     - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
-
-  tmp:
-    image: busybox
-    command: chmod -R 777 /var/run/redis
-    volumes:
-        - /var/run/redis
-
-  redis:
-    image: redis
-    command: redis-server /etc/redis.conf
-    ulimits:
-      nproc: 65535
-      nofile:
-         soft: 65535
-         hard: 65535
-    volumes:
-      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
-      - redis_data:/data
-    volumes_from:
-      - tmp
-    healthcheck:
-      test: "redis-cli -s /var/run/redis/redis.sock ping"
-      interval: 2s
-      timeout: 2s
-      retries: 15
 
 networks:
    default:
@@ -339,10 +56,4 @@ networks:
          name: pathogen
 
 volumes:
-  redis_data: {}
+  ssdb_data: {}
-  superset_home:
-    external: false
-  db_home:
-    external: false
-  redis:
-    external: false

docker/druid/Dockerfile

@@ -0,0 +1,22 @@
+ARG DRUID_VER=0.23.0
+
+FROM apache/druid:${DRUID_VER} AS druid
+
+FROM ubuntu:bionic
+RUN apt-get update && \
+    apt-get install --yes openjdk-8-jre-headless perl-modules && \
+    apt-get clean
+
+RUN addgroup --system -gid 1000 druid \
+ && adduser --system --uid 1000 --disabled-password --home /opt/druid --shell /bin/bash --group druid
+
+COPY --from=druid --chown=druid:druid /opt/druid /opt/druid
+
+WORKDIR /opt/druid
+
+USER druid
+
+EXPOSE 8888/tcp
+EXPOSE 8081/tcp
+
+CMD /opt/druid/bin/start-nano-quickstart

docker/turnilo.yaml

@@ -0,0 +1,3 @@
+clusters:
+  - name: druid
+    guardDataCubes: true

environment

@@ -1,87 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Java tuning
-#DRUID_XMX=1g
-#DRUID_XMS=1g
-#DRUID_MAXNEWSIZE=250m
-#DRUID_NEWSIZE=250m
-#DRUID_MAXDIRECTMEMORYSIZE=1g
-
-#druid_emitter_logging_logLevel=debug
-
-#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
-
-#druid_zk_service_host=zookeeper
-
-#druid_metadata_storage_host=
-#druid_metadata_storage_type=postgresql
-#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
-#druid_metadata_storage_connector_user=druid
-#druid_metadata_storage_connector_password=FoolishPassword
-
-#druid_coordinator_balancer_strategy=cachingCost
-
-#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
-#druid_processing_buffer_sizeBytes=268435456 # 256MiB
-
-#druid_storage_type=local
-#druid_storage_storageDirectory=/opt/shared/segments
-#druid_indexer_logs_type=file
-#druid_indexer_logs_directory=/opt/shared/indexing-logs
-
-#druid_processing_numThreads=1
-#druid_processing_numMergeBuffers=1
-
-#DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
-# Java tuning
-#DRUID_XMX=1g
-#DRUID_XMS=1g
-#DRUID_MAXNEWSIZE=250m
-#DRUID_NEWSIZE=250m
-#DRUID_MAXDIRECTMEMORYSIZE=6172m
-DRUID_SINGLE_NODE_CONF=nano-quickstart
-
-druid_emitter_logging_logLevel=debug
-
-druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
-
-druid_zk_service_host=zookeeper
-
-druid_metadata_storage_host=
-druid_metadata_storage_type=postgresql
-druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
-druid_metadata_storage_connector_user=druid
-druid_metadata_storage_connector_password=FoolishPassword
-
-druid_coordinator_balancer_strategy=cachingCost
-
-druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
-
-druid_storage_type=local
-druid_storage_storageDirectory=/opt/shared/segments
-druid_indexer_logs_type=file
-druid_indexer_logs_directory=/opt/shared/indexing-logs
-
-druid_processing_numThreads=2
-druid_processing_numMergeBuffers=2
-
-DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

sources/ingest.py

@@ -36,7 +36,7 @@ class Ingest(object):
         items = []
         # for source in SOURCES:
         # key = f"{KEYPREFIX}{source}"
-        chunk = await db.ar.spop(KEYNAME, CHUNK_SIZE)
+        chunk = await db.ar.zpop(KEYNAME, CHUNK_SIZE)
         if not chunk:
             return
         for item in chunk:

stack.env

@@ -0,0 +1,40 @@
+PORTAINER_GIT_DIR=..
+MODULES_ENABLED="dis"
+DISCORD_TOKEN="MTAwMzY5NjYyNTkxMjQwMTk2MQ.GRFrdq.tY_XyfBRyV0KlVAhGnPz1DH8URqE5jgV1_JJi0"
+THRESHOLD_LISTENER_HOST=0.0.0.0
+THRESHOLD_LISTENER_PORT=13867
+THRESHOLD_LISTENER_SSL=1
+
+THRESHOLD_RELAY_ENABLED=0
+THRESHOLD_RELAY_HOST=0.0.0.0
+THRESHOLD_RELAY_PORT=13868
+THRESHOLD_RELAY_SSL=1
+
+THRESHOLD_API_ENABLED=1
+THRESHOLD_API_HOST=0.0.0.0
+THRESHOLD_API_PORT=13869
+
+THRESHOLD_CONFIG_DIR=./legacy/conf/live/
+THRESHOLD_CERT_DIR=./legacy/conf/cert/
+
+# How many messages to ingest at once from Redis
+MONOLITH_INGEST_CHUNK_SIZE=8000
+
+# Time to wait between polling Redis again
+MONOLITH_INGEST_ITER_DELAY=0.5
+
+# Number of 4chan threads to request at once
+MONOLITH_CH4_THREADS_CONCURRENT=1000
+
+# Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
+MONOLITH_CH4_THREADS_DELAY=0.1
+
+# Time to wait after finishing a crawl before starting again
+MONOLITH_CH4_CRAWL_DELAY=30
+
+# Semaphore value
+MONOLITH_CH4_THREADS_SEMAPHORE=1000
+
+# Threads to use for data processing
+# Leave uncommented to use all available threads
+MONOLITH_PROCESS_THREADS=4