db.py

@@ -47,6 +47,7 @@ KEYNAME = "queue"
 ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "elastic")
 ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "changeme")
 ELASTICSEARCH_HOST = getenv("ELASTICSEARCH_HOST", "localhost")
+ELASTICSEARCH_PORT = int(getenv("ELASTICSEARCH_PORT", "9200"))
 ELASTICSEARCH_TLS = getenv("ELASTICSEARCH_TLS", "false") in trues
 
 client = None

docker-compose.yml

@@ -0,0 +1,379 @@
+version: "2.2"
+
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis_superset
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
+  - superset_home:/app/superset_home
+
+services:
+  app:
+    image: pathogen/monolith:latest
+    container_name: monolith
+    build: ./docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+    env_file:
+      - .env
+    volumes_from:
+      - tmp
+    depends_on:
+      druid:
+        condition: service_started
+      kafka:
+        condition: service_healthy
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
+    #   - db
+
+  threshold:
+    image: pathogen/threshold:latest
+    container_name: threshold
+    build: ./legacy/docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      - ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
+    ports:
+      - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
+      - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
+      - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
+    env_file:
+      - .env
+    # for development
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes_from:
+        - tmp
+    depends_on:
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
+
+  # db:
+    #image: pathogen/manticore:kibana
+    # image: manticoresearch/manticore:latest
+    #build: 
+    #  context: ./docker/manticore
+    #  args:
+    #    DEV: 1
+    # restart: always
+
+  # turnilo:
+  #   container_name: turnilo
+  #   image: uchhatre/turnilo:latest
+  #   ports:
+  #     - 9093:9090
+  #   environment:
+  #     - DRUID_BROKER_URL=http://broker:8082
+  #     - CONFIG_FILE=/config.yaml
+  #   volumes:
+  #     - ${PORTAINER_GIT_DIR}/docker/turnilo.yaml:/config.yaml
+  #   depends_on:
+  #     - broker
+
+  # metabase:
+  #   container_name: metabase
+  #   image: metabase/metabase:latest
+  #   ports:
+  #     - 3096:3000
+  #   environment:
+  #     JAVA_OPTS: -Xmx1g
+  #     MB_DB_TYPE: postgres
+  #     MB_DB_DBNAME: metabase
+  #     MB_DB_PORT: 5432
+  #     MB_DB_USER: druid
+  #     MB_DB_PASS: FoolishPassword
+  #     MB_DB_HOST: postgres
+  #   depends_on:
+  #     - broker
+
+  redis_superset:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis_superset:/data
+  db:
+    env_file: docker/.env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: docker/.env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: docker/.env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: docker/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: docker/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - metadata_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: FoolishPassword
+      POSTGRES_USER: druid
+      POSTGRES_DB: druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOO_MY_ID=1
+
+  kafka:
+    image: wurstmeister/kafka:latest
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_MESSAGE_MAX_BYTES: 2000000
+      #KAFKA_HEAP_OPTS: -Xmx2g
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
+      start_period: 15s
+      interval: 30s
+      timeout: 30s
+      retries: 45
+
+  druid:
+    image: pathogen/druid:0.23.0
+    build: ./docker/druid/
+    container_name: druid
+    volumes:
+      - druid_shared:/opt/shared
+      - druid_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+      - "8082:8082"
+      - "8083:8083"
+      - "8888:8888"
+    env_file:
+      - environment
+
+  # coordinator:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: coordinator
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - coordinator_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #   ports:
+  #     - "8081:8081"
+  #   command:
+  #     - coordinator
+  #   env_file:
+  #     - environment
+
+  # broker:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: broker
+  #   volumes:
+  #     - broker_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8082:8082"
+  #   command:
+  #     - broker
+  #   env_file:
+  #     - environment
+
+  # historical:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: historical
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - historical_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8083:8083"
+  #   command:
+  #     - historical
+  #   env_file:
+  #     - environment
+
+  # middlemanager:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: middlemanager
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - middle_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8091:8091"
+  #     - "8100-8105:8100-8105"
+  #   command:
+  #     - middleManager
+  #   env_file:
+  #     - environment
+
+  # router:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: router
+  #   volumes:
+  #     - router_var:/opt/druid/var
+  #   depends_on:
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8888:8888"
+  #   command:
+  #     - router
+  #   env_file:
+  #     - environment
+
+  # db:
+  #   #image: pathogen/manticore:kibana
+  #   image: manticoresearch/manticore:dev
+  #   #build: 
+  #   #  context: ./docker/manticore
+  #   #  args:
+  #   #    DEV: 1
+  #   restart: always
+  #   ports:
+  #     - 9308
+  #     - 9312
+  #     - 9306
+  #   ulimits:
+  #     nproc: 65535
+  #     nofile:
+  #        soft: 65535
+  #        hard: 65535
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   environment:
+  #     - MCL=1
+  #   volumes:
+  #     - ./docker/data:/var/lib/manticore
+  #     - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+        - /var/run/redis
+
+  redis:
+    image: redis
+    command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
+    volumes_from:
+      - tmp
+    healthcheck:
+      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
+
+networks:
+   default:
+      external:
+         name: pathogen
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis_superset:
+    external: false
+  redis_data: {}
+  metadata_data: {}
+  # middle_var: {}
+  # historical_var: {}
+  # broker_var: {}
+  # coordinator_var: {}
+  druid_var: {}
+  druid_shared: {}

env.example

@@ -1,6 +1,6 @@
-PORTAINER_GIT_DIR=..
+PORTAINER_GIT_DIR=.
 MODULES_ENABLED="dis"
-DISCORD_TOKEN=
+DISCORD_TOKEN="xx"
 THRESHOLD_LISTENER_HOST=0.0.0.0
 THRESHOLD_LISTENER_PORT=13867
 THRESHOLD_LISTENER_SSL=1
@@ -13,16 +13,16 @@ THRESHOLD_RELAY_SSL=1
 THRESHOLD_API_ENABLED=1
 THRESHOLD_API_HOST=0.0.0.0
 THRESHOLD_API_PORT=13869
+PORTAINER_GIT_DIR=.
 
-THRESHOLD_CONFIG_DIR=../legacy/conf/live/
+THRESHOLD_CONFIG_DIR=./legacy/conf/live/
-#THRESHOLD_TEMPLATE_DIR=../legacy/conf/templates/
+THRESHOLD_CERT_DIR=./legacy/conf/cert/
-THRESHOLD_CERT_DIR=../legacy/conf/cert/
 
 # How many messages to ingest at once from Redis
-MONOLITH_INGEST_CHUNK_SIZE=70000
+MONOLITH_INGEST_CHUNK_SIZE=900
 
 # Time to wait between polling Redis again
-MONOLITH_INGEST_ITER_DELAY=2
+MONOLITH_INGEST_ITER_DELAY=0.5
 
 # Number of 4chan threads to request at once
 MONOLITH_CH4_THREADS_CONCURRENT=1000
@@ -31,20 +31,11 @@ MONOLITH_CH4_THREADS_CONCURRENT=1000
 MONOLITH_CH4_THREADS_DELAY=0.1
 
 # Time to wait after finishing a crawl before starting again
-MONOLITH_CH4_CRAWL_DELAY=60
+MONOLITH_CH4_CRAWL_DELAY=30
 
 # Semaphore value
 MONOLITH_CH4_THREADS_SEMAPHORE=1000
 
 # Threads to use for data processing
 # Leave uncommented to use all available threads
-MONOLITH_PROCESS_THREADS=7
+# MONOLITH_PROCESS_THREADS=4
-
-# Enable performance metrics after message processing
-MONOLITH_PROCESS_PERFSTATS=0
-
-# Elasticsearch
-ELASTICSEARCH_USERNAME=elastic
-ELASTICSEARCH_PASSWORD=
-ELASTICSEARCH_HOST=https://es01:9200
-ELASTICSEARCH_TLS=1