Switch quickstart setting to nano

This reverts commit 64e3e1160aa76d191740342ab3edc68807f890fb.

.gitignore

@@ -158,5 +158,4 @@ cython_debug/
 docker/data
 *.pem
 legacy/conf/live/
-legacy/conf/cert/
-stack.env
+legacy/conf/cert/

Makefile

@@ -1,20 +0,0 @@
-run:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env up -d
-
-build:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env build
-
-stop:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env down
-
-log:
-	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env logs -f
-
-run-infra:
-	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env up -d
-
-stop-infra:
-	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env down
-
-log-infra:
-	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env logs -f

db.py

@@ -1,28 +1,23 @@
 import random
-from os import getenv
 
 import aioredis
 import orjson
-import redis
 
 # Kafka
 from aiokafka import AIOKafkaProducer
+from redis import StrictRedis
 
 import util
 
-trues = ("true", "1", "t", True)
-
-MONOLITH_KAFKA_ENABLED = getenv("MONOLITH_KAFKA_ENABLED", "false").lower() in trues
-
 # KAFKA_TOPIC = "msg"
 
 log = util.get_logger("db")
 
 # Redis (legacy)
-r = redis.from_url("redis://ssdb:1289", db=0)
+r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
 
 # AIORedis
-ar = aioredis.from_url("redis://ssdb:1289", db=0)
+ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
 
 TYPES_MAIN = [
     "msg",
@@ -49,9 +44,6 @@ KEYNAME = "queue"
 
 
 async def store_kafka_batch(data):
-    if not MONOLITH_KAFKA_ENABLED:
-        log.info(f"Not storing Kafka batch of length {len(data)}, Kafka is disabled.")
-        return
     # log.debug(f"Storing Kafka batch of {len(data)} messages")
     producer = AIOKafkaProducer(bootstrap_servers="kafka:9092")
     await producer.start()
@@ -122,7 +114,7 @@ async def queue_message(msg):
     Queue a message on the Redis buffer.
     """
     message = orjson.dumps(msg)
-    await ar.lpush(KEYNAME, message)
+    await ar.sadd(KEYNAME, message)
 
 
 async def queue_message_bulk(data):
@@ -131,4 +123,4 @@ async def queue_message_bulk(data):
     """
     for msg in data:
         message = orjson.dumps(msg)
-        await ar.lpush(KEYNAME, message)
+        await ar.sadd(KEYNAME, message)

docker-compose.yml

@@ -6,7 +6,7 @@ x-superset-depends-on: &superset-depends-on
   - redis_superset
 x-superset-volumes: &superset-volumes
   # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
-  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
+  - ./docker/superset:/app/docker
   - superset_home:/app/superset_home
 
 services:
@@ -21,7 +21,7 @@ services:
     volumes_from:
       - tmp
     depends_on:
-      druid:
+      broker:
         condition: service_started
       kafka:
         condition: service_healthy
@@ -100,7 +100,7 @@ services:
     container_name: superset_cache
     restart: unless-stopped
     volumes:
-      - redis_superset:/data
+      - redis:/data
   db:
     env_file: docker/.env-non-dev
     image: postgres:10
@@ -195,116 +195,88 @@ services:
       timeout: 30s
       retries: 45
 
-  druid:
-    image: pathogen/druid:0.23.0
-    build: ./docker/druid/
-    container_name: druid
+  coordinator:
+    image: apache/druid:0.23.0
+    container_name: coordinator
     volumes:
       - druid_shared:/opt/shared
-      - druid_var:/opt/druid/var
+      - coordinator_var:/opt/druid/var
     depends_on: 
       - zookeeper
       - postgres
     ports:
       - "8081:8081"
-      - "8082:8082"
-      - "8083:8083"
-      - "8888:8888"
+    command:
+      - coordinator
     env_file:
       - environment
 
-  # coordinator:
-  #   #image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
-  #   build: ./docker/druid/
-  #   container_name: coordinator
-  #   volumes:
-  #     - druid_shared:/opt/shared
-  #     - coordinator_var:/opt/druid/var
-  #   depends_on: 
-  #     - zookeeper
-  #     - postgres
-  #   ports:
-  #     - "8081:8081"
-  #   command:
-  #     - coordinator
-  #   env_file:
-  #     - environment
+  broker:
+    image: apache/druid:0.23.0
+    container_name: broker
+    volumes:
+      - broker_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8082:8082"
+    command:
+      - broker
+    env_file:
+      - environment
 
-  # broker:
-  #   #image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
-  #   build: ./docker/druid/
-  #   container_name: broker
-  #   volumes:
-  #     - broker_var:/opt/druid/var
-  #   depends_on: 
-  #     - zookeeper
-  #     - postgres
-  #     - coordinator
-  #   ports:
-  #     - "8082:8082"
-  #   command:
-  #     - broker
-  #   env_file:
-  #     - environment
+  historical:
+    image: apache/druid:0.23.0
+    container_name: historical
+    volumes:
+      - druid_shared:/opt/shared
+      - historical_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8083:8083"
+    command:
+      - historical
+    env_file:
+      - environment
 
-  # historical:
-  #   #image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
-  #   build: ./docker/druid/
-  #   container_name: historical
-  #   volumes:
-  #     - druid_shared:/opt/shared
-  #     - historical_var:/opt/druid/var
-  #   depends_on: 
-  #     - zookeeper
-  #     - postgres
-  #     - coordinator
-  #   ports:
-  #     - "8083:8083"
-  #   command:
-  #     - historical
-  #   env_file:
-  #     - environment
+  middlemanager:
+    image: apache/druid:0.23.0
+    container_name: middlemanager
+    volumes:
+      - druid_shared:/opt/shared
+      - middle_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8091:8091"
+      - "8100-8105:8100-8105"
+    command:
+      - middleManager
+    env_file:
+      - environment
 
-  # middlemanager:
-  #   #image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
-  #   build: ./docker/druid/
-  #   container_name: middlemanager
-  #   volumes:
-  #     - druid_shared:/opt/shared
-  #     - middle_var:/opt/druid/var
-  #   depends_on: 
-  #     - zookeeper
-  #     - postgres
-  #     - coordinator
-  #   ports:
-  #     - "8091:8091"
-  #     - "8100-8105:8100-8105"
-  #   command:
-  #     - middleManager
-  #   env_file:
-  #     - environment
-
-  # router:
-  #   #image: apache/druid:0.23.0
-  #   image: pathogen/druid:0.23.0
-  #   build: ./docker/druid/
-  #   container_name: router
-  #   volumes:
-  #     - router_var:/opt/druid/var
-  #   depends_on:
-  #     - zookeeper
-  #     - postgres
-  #     - coordinator
-  #   ports:
-  #     - "8888:8888"
-  #   command:
-  #     - router
-  #   env_file:
-  #     - environment
+  router:
+    image: apache/druid:0.23.0
+    container_name: router
+    volumes:
+      - router_var:/opt/druid/var
+    depends_on:
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8888:8888"
+    command:
+      - router
+    env_file:
+      - environment
 
   # db:
   #   #image: pathogen/manticore:kibana
@@ -371,9 +343,9 @@ volumes:
     external: false
   redis_data: {}
   metadata_data: {}
-  # middle_var: {}
-  # historical_var: {}
-  # broker_var: {}
-  # coordinator_var: {}
-  druid_var: {}
+  middle_var: {}
+  historical_var: {}
+  broker_var: {}
+  coordinator_var: {}
+  router_var: {}
   druid_shared: {}

docker/.env-non-dev

@@ -41,6 +41,6 @@ REDIS_PORT=6379
 
 FLASK_ENV=production
 SUPERSET_ENV=production
-SUPERSET_LOAD_EXAMPLES=no
+SUPERSET_LOAD_EXAMPLES=yes
 CYPRESS_CONFIG=false
 SUPERSET_PORT=8088

docker/docker-compose.infra.yml

@@ -1,142 +0,0 @@
-version: "2.2"
-
-x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
-x-superset-depends-on: &superset-depends-on
-  - db
-  - redis_superset
-x-superset-volumes: &superset-volumes
-  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
-  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
-  - superset_home:/app/superset_home
-
-services:
-  redis_superset:
-    image: redis:latest
-    container_name: superset_cache
-    restart: unless-stopped
-    volumes:
-      - redis:/data
-  db:
-    env_file: .env-non-dev
-    image: postgres:10
-    container_name: superset_db
-    restart: unless-stopped
-    volumes:
-      - db_home:/var/lib/postgresql/data
-
-  superset:
-    env_file: .env-non-dev
-    image: *superset-image
-    container_name: superset_app
-    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
-    user: "root"
-    restart: unless-stopped
-    ports:
-      - 8088:8088
-    depends_on: *superset-depends-on
-    volumes: *superset-volumes
-
-  superset-init:
-    image: *superset-image
-    container_name: superset_init
-    command: ["/app/docker/docker-init.sh"]
-    env_file: .env-non-dev
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  superset-worker:
-    image: *superset-image
-    container_name: superset_worker
-    command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: .env-non-dev
-    restart: unless-stopped
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  superset-worker-beat:
-    image: *superset-image
-    container_name: superset_worker_beat
-    command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: .env-non-dev
-    restart: unless-stopped
-    depends_on: *superset-depends-on
-    user: "root"
-    volumes: *superset-volumes
-
-  postgres:
-    container_name: postgres
-    image: postgres:latest
-    volumes:
-      - /block/store/metadata_data:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD=FoolishPassword
-      - POSTGRES_USER=druid
-      - POSTGRES_DB=druid
-
-  # Need 3.5 or later for container nodes
-  zookeeper:
-    container_name: zookeeper
-    image: zookeeper:3.5
-    ports:
-      - "2181:2181"
-    environment:
-      - ZOO_MY_ID=1
-
-  kafka:
-    image: wurstmeister/kafka:latest
-    container_name: kafka
-    depends_on:
-      - zookeeper
-    ports:
-      - 9092:9092
-      - 29092:29092
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      KAFKA_MESSAGE_MAX_BYTES: 2000000
-      #KAFKA_HEAP_OPTS: -Xmx2g
-    healthcheck:
-      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
-      start_period: 15s
-      interval: 30s
-      timeout: 30s
-      retries: 45
-
-  druid:
-    image: pathogen/druid:0.23.0
-    build: ./druid/
-    container_name: druid
-    volumes:
-      - /block/store/druid_shared:/opt/shared
-      - /block/store/druid_var:/opt/druid/var
-    depends_on: 
-      - zookeeper
-      - postgres
-    ports:
-      - "8081:8081"
-      - "8082:8082"
-      - "8083:8083"
-      - "8888:8888"
-    env_file:
-      - environment
-
-networks:
-   default:
-      external:
-         name: pathogen
-
-volumes:
-  superset_home:
-    external: false
-  db_home:
-    external: false
-  redis:
-    external: false

docker/docker-compose.prod.yml

@@ -1,14 +1,44 @@
 version: "2.2"
 
+# volumes:
+#   metadata_data: {}
+#   middle_var: {}
+#   historical_var: {}
+#   broker_var: {}
+#   coordinator_var: {}
+#   router_var: {}
+#   druid_shared: {}
+
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis_superset
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ./docker/superset:/app/docker
+  - superset_home:/app/superset_home
+
 services:
   app:
     image: pathogen/monolith:latest
     container_name: monolith
-    build: ${PORTAINER_GIT_DIR}/docker
+    build: ./docker
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
     env_file:
       - ../stack.env
+    volumes_from:
+      - tmp
+    depends_on:
+      broker:
+        condition: service_started
+      kafka:
+        condition: service_healthy
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
+    #   - db
 
   threshold:
     image: pathogen/threshold:latest
@@ -17,7 +47,7 @@ services:
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
       - ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
-      - ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
       - ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
     ports:
       - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
@@ -25,19 +55,283 @@ services:
       - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
     env_file:
       - ../stack.env
-    # for development
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
+    volumes_from:
+        - tmp
+    depends_on:
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
 
-  ssdb:
-    image: tsl0922/ssdb
-    container_name: ssdb_monolith
+  # db:
+    #image: pathogen/manticore:kibana
+    # image: manticoresearch/manticore:latest
+    #build: 
+    #  context: ./docker/manticore
+    #  args:
+    #    DEV: 1
+    # restart: always
+
+
+  # turnilo:
+  #   container_name: turnilo
+  #   image: uchhatre/turnilo:latest
+  #   ports:
+  #     - 9093:9090
+  #   environment:
+  #     - DRUID_BROKER_URL=http://broker:8082
+  #   depends_on:
+  #     - broker
+
+  # metabase:
+  #   container_name: metabase
+  #   image: metabase/metabase:latest
+  #   ports:
+  #     - 3096:3000
+  #   environment:
+  #     JAVA_OPTS: -Xmx1g
+  #     MB_DB_TYPE: postgres
+  #     MB_DB_DBNAME: metabase
+  #     MB_DB_PORT: 5432
+  #     MB_DB_USER: druid
+  #     MB_DB_PASS: FoolishPassword
+  #     MB_DB_HOST: postgres
+  #   depends_on:
+  #     - broker
+
+  redis_superset:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
     volumes:
-      - ssdb_data:/ssdb/var
+      - redis:/data
+  db:
+    env_file: .env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: .env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
     ports:
-      - "1289:1289"
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: .env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - /block/store/metadata_data:/var/lib/postgresql/data
     environment:
-      - SSDB_PORT=1289
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOO_MY_ID=1
+
+  kafka:
+    image: wurstmeister/kafka:latest
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_MESSAGE_MAX_BYTES: 2000000
+      #KAFKA_HEAP_OPTS: -Xmx2g
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
+      start_period: 15s
+      interval: 30s
+      timeout: 30s
+      retries: 45
+
+  coordinator:
+    image: apache/druid:0.23.0
+    container_name: coordinator
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/coordinator_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+    command:
+      - coordinator
+    env_file:
+      - environment
+
+  broker:
+    image: apache/druid:0.23.0
+    container_name: broker
+    volumes:
+      - /block/store/broker_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8082:8082"
+    command:
+      - broker
+    env_file:
+      - environment
+
+  historical:
+    image: apache/druid:0.23.0
+    container_name: historical
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/historical_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8083:8083"
+    command:
+      - historical
+    env_file:
+      - environment
+
+  middlemanager:
+    image: apache/druid:0.23.0
+    container_name: middlemanager
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/middle_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8091:8091"
+      - "8100-8105:8100-8105"
+    command:
+      - middleManager
+    env_file:
+      - environment
+
+  router:
+    image: apache/druid:0.23.0
+    container_name: router
+    volumes:
+      - /block/store/router_var:/opt/druid/var
+    depends_on:
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8888:8888"
+    command:
+      - router
+    env_file:
+      - environment
+
+  # db:
+  #   #image: pathogen/manticore:kibana
+  #   image: manticoresearch/manticore:dev
+  #   #build: 
+  #   #  context: ./docker/manticore
+  #   #  args:
+  #   #    DEV: 1
+  #   restart: always
+  #   ports:
+  #     - 9308
+  #     - 9312
+  #     - 9306
+  #   ulimits:
+  #     nproc: 65535
+  #     nofile:
+  #        soft: 65535
+  #        hard: 65535
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   environment:
+  #     - MCL=1
+  #   volumes:
+  #     - ./docker/data:/var/lib/manticore
+  #     - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+        - /var/run/redis
+
+  redis:
+    image: redis
+    command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
+    volumes_from:
+      - tmp
+    healthcheck:
+      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
 
 networks:
    default:
@@ -45,4 +339,10 @@ networks:
          name: pathogen
 
 volumes:
-  ssdb_data: {}
+  redis_data: {}
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker/druid/Dockerfile

@@ -1,22 +0,0 @@
-ARG DRUID_VER=0.23.0
-
-FROM apache/druid:${DRUID_VER} AS druid
-
-FROM ubuntu:bionic
-RUN apt-get update && \
-    apt-get install --yes openjdk-8-jre-headless perl-modules && \
-    apt-get clean
-
-RUN addgroup --system -gid 1000 druid \
- && adduser --system --uid 1000 --disabled-password --home /opt/druid --shell /bin/bash --group druid
-
-COPY --from=druid --chown=druid:druid /opt/druid /opt/druid
-
-WORKDIR /opt/druid
-
-USER druid
-
-EXPOSE 8888/tcp
-EXPOSE 8081/tcp
-
-CMD /opt/druid/bin/start-nano-quickstart

docker/turnilo.yaml

@@ -1,3 +0,0 @@
-clusters:
-  - name: druid
-    guardDataCubes: true

environment

@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=1g
+
+#druid_emitter_logging_logLevel=debug
+
+#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+#druid_zk_service_host=zookeeper
+
+#druid_metadata_storage_host=
+#druid_metadata_storage_type=postgresql
+#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+#druid_metadata_storage_connector_user=druid
+#druid_metadata_storage_connector_password=FoolishPassword
+
+#druid_coordinator_balancer_strategy=cachingCost
+
+#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
+#druid_processing_buffer_sizeBytes=268435456 # 256MiB
+
+#druid_storage_type=local
+#druid_storage_storageDirectory=/opt/shared/segments
+#druid_indexer_logs_type=file
+#druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+#druid_processing_numThreads=1
+#druid_processing_numMergeBuffers=1
+
+#DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=6172m
+DRUID_SINGLE_NODE_CONF=nano-quickstart
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/shared/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

legacy/modules/monitor.py

@@ -67,7 +67,7 @@ def parsemeta(numName, c):
 
 def queue_message(c):
     message = json.dumps(c)
-    main.g.lpush("queue", message)
+    main.g.sadd("queue", message)
 
 
 def event(

processing/process.py

@@ -15,7 +15,6 @@ from concurrent.futures import ProcessPoolExecutor
 # For timestamp processing
 from datetime import datetime
 from math import ceil
-from os import getenv
 
 import orjson
 import regex
@@ -52,12 +51,6 @@ import util
 # 4chan schema
 from schemas.ch4_s import ATTRMAP
 
-trues = ("true", "1", "t", True)
-
-MONOLITH_PROCESS_PERFSTATS = (
-    getenv("MONOLITH_PROCESS_PERFSTATS", "false").lower() in trues
-)
-
 CUSTOM_FILTERS = [
     lambda x: x.lower(),
     strip_tags,  #
@@ -274,19 +267,17 @@ def process_data(data):
         # Add the mutated message to the return buffer
         to_store.append(msg)
         total_time += (time.process_time() - total_start) * 1000
-
-    if MONOLITH_PROCESS_PERFSTATS:
-        log.debug("=====================================")
-        log.debug(f"Sentiment: {sentiment_time}")
-        log.debug(f"Regex: {regex_time}")
-        log.debug(f"Polyglot: {polyglot_time}")
-        log.debug(f"Date: {date_time}")
-        log.debug(f"NLP: {nlp_time}")
-        log.debug(f"Normalise: {normalise_time}")
-        log.debug(f"Hash: {hash_time}")
-        log.debug(f"Normal2: {normal2_time}")
-        log.debug(f"Soup: {soup_time}")
-        log.debug(f"Total: {total_time}")
-        log.debug("=====================================")
+    log.debug("=====================================")
+    log.debug(f"Sentiment: {sentiment_time}")
+    log.debug(f"Regex: {regex_time}")
+    log.debug(f"Polyglot: {polyglot_time}")
+    log.debug(f"Date: {date_time}")
+    log.debug(f"NLP: {nlp_time}")
+    log.debug(f"Normalise: {normalise_time}")
+    log.debug(f"Hash: {hash_time}")
+    log.debug(f"Normal2: {normal2_time}")
+    log.debug(f"Soup: {soup_time}")
+    log.debug(f"Total: {total_time}")
+    log.debug("=====================================")
 
     return to_store

sources/ingest.py

@@ -36,10 +36,7 @@ class Ingest(object):
         items = []
         # for source in SOURCES:
         # key = f"{KEYPREFIX}{source}"
-        length = await db.ar.llen(KEYNAME)
-        start_num = length - CHUNK_SIZE
-        chunk = await db.ar.lrange(KEYNAME, start_num, -1)
-        # chunk = await db.ar.rpop(KEYNAME, CHUNK_SIZE)
+        chunk = await db.ar.spop(KEYNAME, CHUNK_SIZE)
         if not chunk:
             return
         for item in chunk: