Switch quickstart setting to nano

This reverts commit 64e3e1160aa76d191740342ab3edc68807f890fb.

Makefile

@@ -1,20 +0,0 @@
-run:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env up -d
-
-build:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env build
-
-stop:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env down
-
-log:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env logs -f
-
-run-infra:
-	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env up -d
-
-stop-infra:
-	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env down
-
-log-infra:
-	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env logs -f

db.py

@@ -5,7 +5,7 @@ import orjson
 
 # Kafka
 from aiokafka import AIOKafkaProducer
-import redis
+from redis import StrictRedis
 
 import util
 
@@ -14,10 +14,10 @@ import util
 log = util.get_logger("db")
 
 # Redis (legacy)
-r = redis.from_url("redis://ssdb:1289", db=0)
+r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
 
 # AIORedis
-ar = aioredis.from_url("redis://ssdb:1289", db=0)
+ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
 
 TYPES_MAIN = [
     "msg",
@@ -44,8 +44,6 @@ KEYNAME = "queue"
 
 
 async def store_kafka_batch(data):
-    print("FAKE STORE KAFKA BATCH")
-    return
     # log.debug(f"Storing Kafka batch of {len(data)} messages")
     producer = AIOKafkaProducer(bootstrap_servers="kafka:9092")
     await producer.start()
@@ -116,7 +114,7 @@ async def queue_message(msg):
     Queue a message on the Redis buffer.
     """
     message = orjson.dumps(msg)
-    await ar.zadd(KEYNAME, message)
+    await ar.sadd(KEYNAME, message)
 
 
 async def queue_message_bulk(data):
@@ -125,4 +123,4 @@ async def queue_message_bulk(data):
     """
     for msg in data:
         message = orjson.dumps(msg)
-        await ar.zadd(KEYNAME, message)
+        await ar.sadd(KEYNAME, message)

docker-compose.yml

@@ -6,7 +6,7 @@ x-superset-depends-on: &superset-depends-on
   - redis_superset
 x-superset-volumes: &superset-volumes
   # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
-  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
+  - ./docker/superset:/app/docker
   - superset_home:/app/superset_home
 
 services:
@@ -21,7 +21,7 @@ services:
     volumes_from:
       - tmp
     depends_on:
-      druid:
+      broker:
         condition: service_started
       kafka:
         condition: service_healthy
@@ -100,7 +100,7 @@ services:
     container_name: superset_cache
     restart: unless-stopped
     volumes:
-      - redis_superset:/data
+      - redis:/data
   db:
     env_file: docker/.env-non-dev
     image: postgres:10
@@ -195,116 +195,88 @@ services:
       timeout: 30s
       retries: 45
 
-  druid:
+  coordinator:
-    image: pathogen/druid:0.23.0
+    image: apache/druid:0.23.0
-    build: ./docker/druid/
+    container_name: coordinator
-    container_name: druid
     volumes:
       - druid_shared:/opt/shared
-      - druid_var:/opt/druid/var
+      - coordinator_var:/opt/druid/var
     depends_on: 
       - zookeeper
       - postgres
     ports:
       - "8081:8081"
-      - "8082:8082"
+    command:
-      - "8083:8083"
+      - coordinator
-      - "8888:8888"
     env_file:
       - environment
 
-  # coordinator:
+  broker:
-  #   #image: apache/druid:0.23.0
+    image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
+    container_name: broker
-  #   build: ./docker/druid/
+    volumes:
-  #   container_name: coordinator
+      - broker_var:/opt/druid/var
-  #   volumes:
+    depends_on: 
-  #     - druid_shared:/opt/shared
+      - zookeeper
-  #     - coordinator_var:/opt/druid/var
+      - postgres
-  #   depends_on: 
+      - coordinator
-  #     - zookeeper
+    ports:
-  #     - postgres
+      - "8082:8082"
-  #   ports:
+    command:
-  #     - "8081:8081"
+      - broker
-  #   command:
+    env_file:
-  #     - coordinator
+      - environment
-  #   env_file:
-  #     - environment
 
-  # broker:
+  historical:
-  #   #image: apache/druid:0.23.0
+    image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
+    container_name: historical
-  #   build: ./docker/druid/
+    volumes:
-  #   container_name: broker
+      - druid_shared:/opt/shared
-  #   volumes:
+      - historical_var:/opt/druid/var
-  #     - broker_var:/opt/druid/var
+    depends_on: 
-  #   depends_on: 
+      - zookeeper
-  #     - zookeeper
+      - postgres
-  #     - postgres
+      - coordinator
-  #     - coordinator
+    ports:
-  #   ports:
+      - "8083:8083"
-  #     - "8082:8082"
+    command:
-  #   command:
+      - historical
-  #     - broker
+    env_file:
-  #   env_file:
+      - environment
-  #     - environment
 
-  # historical:
+  middlemanager:
-  #   #image: apache/druid:0.23.0
+    image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
+    container_name: middlemanager
-  #   build: ./docker/druid/
+    volumes:
-  #   container_name: historical
+      - druid_shared:/opt/shared
-  #   volumes:
+      - middle_var:/opt/druid/var
-  #     - druid_shared:/opt/shared
+    depends_on: 
-  #     - historical_var:/opt/druid/var
+      - zookeeper
-  #   depends_on: 
+      - postgres
-  #     - zookeeper
+      - coordinator
-  #     - postgres
+    ports:
-  #     - coordinator
+      - "8091:8091"
-  #   ports:
+      - "8100-8105:8100-8105"
-  #     - "8083:8083"
+    command:
-  #   command:
+      - middleManager
-  #     - historical
+    env_file:
-  #   env_file:
+      - environment
-  #     - environment
 
-  # middlemanager:
+  router:
-  #   #image: apache/druid:0.23.0
+    image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
+    container_name: router
-  #   build: ./docker/druid/
+    volumes:
-  #   container_name: middlemanager
+      - router_var:/opt/druid/var
-  #   volumes:
+    depends_on:
-  #     - druid_shared:/opt/shared
+      - zookeeper
-  #     - middle_var:/opt/druid/var
+      - postgres
-  #   depends_on: 
+      - coordinator
-  #     - zookeeper
+    ports:
-  #     - postgres
+      - "8888:8888"
-  #     - coordinator
+    command:
-  #   ports:
+      - router
-  #     - "8091:8091"
+    env_file:
-  #     - "8100-8105:8100-8105"
+      - environment
-  #   command:
-  #     - middleManager
-  #   env_file:
-  #     - environment
-
-  # router:
-  #   #image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
-  #   build: ./docker/druid/
-  #   container_name: router
-  #   volumes:
-  #     - router_var:/opt/druid/var
-  #   depends_on:
-  #     - zookeeper
-  #     - postgres
-  #     - coordinator
-  #   ports:
-  #     - "8888:8888"
-  #   command:
-  #     - router
-  #   env_file:
-  #     - environment
 
   # db:
   #   #image: pathogen/manticore:kibana
@@ -371,9 +343,9 @@ volumes:
     external: false
   redis_data: {}
   metadata_data: {}
-  # middle_var: {}
+  middle_var: {}
-  # historical_var: {}
+  historical_var: {}
-  # broker_var: {}
+  broker_var: {}
-  # coordinator_var: {}
+  coordinator_var: {}
-  druid_var: {}
+  router_var: {}
   druid_shared: {}

docker/.env-non-dev

@@ -41,6 +41,6 @@ REDIS_PORT=6379
 
 FLASK_ENV=production
 SUPERSET_ENV=production
-SUPERSET_LOAD_EXAMPLES=no
+SUPERSET_LOAD_EXAMPLES=yes
 CYPRESS_CONFIG=false
 SUPERSET_PORT=8088

docker/docker-compose.infra.yml

@@ -1,142 +0,0 @@
-version: "2.2"
-
-x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
-x-superset-depends-on: &superset-depends-on
-  - db
-  - redis_superset
-x-superset-volumes: &superset-volumes
-  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
-  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
-  - superset_home:/app/superset_home
-
-services:
-  redis_superset:
-    image: redis:latest
-    container_name: superset_cache
-    restart: unless-stopped
-    volumes:
-      - redis:/data
-  db:
-    env_file: .env-non-dev
-    image: postgres:10
-    container_name: superset_db
-    restart: unless-stopped
-    volumes:
-      - db_home:/var/lib/postgresql/data
-
-  superset:
-    env_file: .env-non-dev
-    image: *superset-image
-    container_name: superset_app
-    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
-    user: "root"
-    restart: unless-stopped
-    ports:
-      - 8088:8088
-    depends_on: *superset-depends-on
-    volumes: *superset-volumes
-
-  superset-init:
-    image: *superset-image
-    container_name: superset_init
-    command: ["/app/docker/docker-init.sh"]
-    env_file: .env-non-dev
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  superset-worker:
-    image: *superset-image
-    container_name: superset_worker
-    command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: .env-non-dev
-    restart: unless-stopped
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  superset-worker-beat:
-    image: *superset-image
-    container_name: superset_worker_beat
-    command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: .env-non-dev
-    restart: unless-stopped
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  postgres:
-    container_name: postgres
-    image: postgres:latest
-    volumes:
-      - /block/store/metadata_data:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD=FoolishPassword
-      - POSTGRES_USER=druid
-      - POSTGRES_DB=druid
-
-  # Need 3.5 or later for container nodes
-  zookeeper:
-    container_name: zookeeper
-    image: zookeeper:3.5
-    ports:
-      - "2181:2181"
-    environment:
-      - ZOO_MY_ID=1
-
-  kafka:
-    image: wurstmeister/kafka:latest
-    container_name: kafka
-    depends_on:
-      - zookeeper
-    ports:
-      - 9092:9092
-      - 29092:29092
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      KAFKA_MESSAGE_MAX_BYTES: 2000000
-      #KAFKA_HEAP_OPTS: -Xmx2g
-    healthcheck:
-      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
-      start_period: 15s
-      interval: 30s
-      timeout: 30s
-      retries: 45
-
-  druid:
-    image: pathogen/druid:0.23.0
-    build: ./druid/
-    container_name: druid
-    volumes:
-      - /block/store/druid_shared:/opt/shared
-      - /block/store/druid_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-    ports:
-      - "8081:8081"
-      - "8082:8082"
-      - "8083:8083"
-      - "8888:8888"
-    env_file:
-      - environment
-
-networks:
-   default:
-      external:
-         name: pathogen
-
-volumes:
-  superset_home:
-    external: false
-  db_home:
-    external: false
-  redis:
-    external: false

docker/docker-compose.prod.yml

@@ -1,25 +1,44 @@
 version: "2.2"
 
+# volumes:
+#   metadata_data: {}
+#   middle_var: {}
+#   historical_var: {}
+#   broker_var: {}
+#   coordinator_var: {}
+#   router_var: {}
+#   druid_shared: {}
+
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis_superset
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ./docker/superset:/app/docker
+  - superset_home:/app/superset_home
+
 services:
   app:
     image: pathogen/monolith:latest
     container_name: monolith
-    build: ${PORTAINER_GIT_DIR}/docker
+    build: ./docker
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
     env_file:
       - ../stack.env
-    # volumes_from:
+    volumes_from:
-    #   - tmp
+      - tmp
-    # depends_on:
+    depends_on:
-      # broker:
+      broker:
-      #   condition: service_started
+        condition: service_started
-      # kafka:
+      kafka:
-      #   condition: service_healthy
+        condition: service_healthy
-      # tmp:
+      tmp:
-      #   condition: service_started
+        condition: service_started
-      # redis:
+      redis:
-      #   condition: service_healthy
+        condition: service_healthy
+    #   - db
 
   threshold:
     image: pathogen/threshold:latest
@@ -36,19 +55,283 @@ services:
       - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
     env_file:
       - ../stack.env
-    # for development
+    volumes_from:
-    extra_hosts:
+        - tmp
-      - "host.docker.internal:host-gateway"
+    depends_on:
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
 
-  ssdb:
+  # db:
-    image: tsl0922/ssdb
+    #image: pathogen/manticore:kibana
-    container_name: ssdb_monolith
+    # image: manticoresearch/manticore:latest
+    #build: 
+    #  context: ./docker/manticore
+    #  args:
+    #    DEV: 1
+    # restart: always
+
+
+  # turnilo:
+  #   container_name: turnilo
+  #   image: uchhatre/turnilo:latest
+  #   ports:
+  #     - 9093:9090
+  #   environment:
+  #     - DRUID_BROKER_URL=http://broker:8082
+  #   depends_on:
+  #     - broker
+
+  # metabase:
+  #   container_name: metabase
+  #   image: metabase/metabase:latest
+  #   ports:
+  #     - 3096:3000
+  #   environment:
+  #     JAVA_OPTS: -Xmx1g
+  #     MB_DB_TYPE: postgres
+  #     MB_DB_DBNAME: metabase
+  #     MB_DB_PORT: 5432
+  #     MB_DB_USER: druid
+  #     MB_DB_PASS: FoolishPassword
+  #     MB_DB_HOST: postgres
+  #   depends_on:
+  #     - broker
+
+  redis_superset:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
     volumes:
-      - ssdb_data:/ssdb/var
+      - redis:/data
+  db:
+    env_file: .env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: .env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
     ports:
-      - "1289:1289"
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: .env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - /block/store/metadata_data:/var/lib/postgresql/data
     environment:
-      - SSDB_PORT=1289
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOO_MY_ID=1
+
+  kafka:
+    image: wurstmeister/kafka:latest
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_MESSAGE_MAX_BYTES: 2000000
+      #KAFKA_HEAP_OPTS: -Xmx2g
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
+      start_period: 15s
+      interval: 30s
+      timeout: 30s
+      retries: 45
+
+  coordinator:
+    image: apache/druid:0.23.0
+    container_name: coordinator
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/coordinator_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+    command:
+      - coordinator
+    env_file:
+      - environment
+
+  broker:
+    image: apache/druid:0.23.0
+    container_name: broker
+    volumes:
+      - /block/store/broker_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8082:8082"
+    command:
+      - broker
+    env_file:
+      - environment
+
+  historical:
+    image: apache/druid:0.23.0
+    container_name: historical
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/historical_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8083:8083"
+    command:
+      - historical
+    env_file:
+      - environment
+
+  middlemanager:
+    image: apache/druid:0.23.0
+    container_name: middlemanager
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/middle_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8091:8091"
+      - "8100-8105:8100-8105"
+    command:
+      - middleManager
+    env_file:
+      - environment
+
+  router:
+    image: apache/druid:0.23.0
+    container_name: router
+    volumes:
+      - /block/store/router_var:/opt/druid/var
+    depends_on:
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8888:8888"
+    command:
+      - router
+    env_file:
+      - environment
+
+  # db:
+  #   #image: pathogen/manticore:kibana
+  #   image: manticoresearch/manticore:dev
+  #   #build: 
+  #   #  context: ./docker/manticore
+  #   #  args:
+  #   #    DEV: 1
+  #   restart: always
+  #   ports:
+  #     - 9308
+  #     - 9312
+  #     - 9306
+  #   ulimits:
+  #     nproc: 65535
+  #     nofile:
+  #        soft: 65535
+  #        hard: 65535
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   environment:
+  #     - MCL=1
+  #   volumes:
+  #     - ./docker/data:/var/lib/manticore
+  #     - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+        - /var/run/redis
+
+  redis:
+    image: redis
+    command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
+    volumes_from:
+      - tmp
+    healthcheck:
+      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
 
 networks:
    default:
@@ -56,4 +339,10 @@ networks:
          name: pathogen
 
 volumes:
-  ssdb_data: {}
+  redis_data: {}
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker/druid/Dockerfile

@@ -1,22 +0,0 @@
-ARG DRUID_VER=0.23.0
-
-FROM apache/druid:${DRUID_VER} AS druid
-
-FROM ubuntu:bionic
-RUN apt-get update && \
-    apt-get install --yes openjdk-8-jre-headless perl-modules && \
-    apt-get clean
-
-RUN addgroup --system -gid 1000 druid \
- && adduser --system --uid 1000 --disabled-password --home /opt/druid --shell /bin/bash --group druid
-
-COPY --from=druid --chown=druid:druid /opt/druid /opt/druid
-
-WORKDIR /opt/druid
-
-USER druid
-
-EXPOSE 8888/tcp
-EXPOSE 8081/tcp
-
-CMD /opt/druid/bin/start-nano-quickstart

docker/turnilo.yaml

@@ -1,3 +0,0 @@
-clusters:
-  - name: druid
-    guardDataCubes: true

environment

@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=1g
+
+#druid_emitter_logging_logLevel=debug
+
+#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+#druid_zk_service_host=zookeeper
+
+#druid_metadata_storage_host=
+#druid_metadata_storage_type=postgresql
+#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+#druid_metadata_storage_connector_user=druid
+#druid_metadata_storage_connector_password=FoolishPassword
+
+#druid_coordinator_balancer_strategy=cachingCost
+
+#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
+#druid_processing_buffer_sizeBytes=268435456 # 256MiB
+
+#druid_storage_type=local
+#druid_storage_storageDirectory=/opt/shared/segments
+#druid_indexer_logs_type=file
+#druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+#druid_processing_numThreads=1
+#druid_processing_numMergeBuffers=1
+
+#DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=6172m
+DRUID_SINGLE_NODE_CONF=nano-quickstart
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/shared/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

sources/ingest.py

@@ -36,7 +36,7 @@ class Ingest(object):
         items = []
         # for source in SOURCES:
         # key = f"{KEYPREFIX}{source}"
-        chunk = await db.ar.zpop(KEYNAME, CHUNK_SIZE)
+        chunk = await db.ar.spop(KEYNAME, CHUNK_SIZE)
         if not chunk:
             return
         for item in chunk: