Compare commits
3 Commits
49f46c33ba
...
c53438d07b
| Author | SHA1 | Date | |
|---|---|---|---|
c53438d07b
|
|||
|
93353f34e7
|
|||
|
6b1604b724
|
1
db.py
1
db.py
@ -47,7 +47,6 @@ KEYNAME = "queue"
|
|||||||
ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "elastic")
|
ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "elastic")
|
||||||
ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "changeme")
|
ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "changeme")
|
||||||
ELASTICSEARCH_HOST = getenv("ELASTICSEARCH_HOST", "localhost")
|
ELASTICSEARCH_HOST = getenv("ELASTICSEARCH_HOST", "localhost")
|
||||||
ELASTICSEARCH_PORT = int(getenv("ELASTICSEARCH_PORT", "9200"))
|
|
||||||
ELASTICSEARCH_TLS = getenv("ELASTICSEARCH_TLS", "false") in trues
|
ELASTICSEARCH_TLS = getenv("ELASTICSEARCH_TLS", "false") in trues
|
||||||
|
|
||||||
client = None
|
client = None
|
||||||
|
|||||||
@ -1,379 +0,0 @@
|
|||||||
version: "2.2"
|
|
||||||
|
|
||||||
x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
|
|
||||||
x-superset-depends-on: &superset-depends-on
|
|
||||||
- db
|
|
||||||
- redis_superset
|
|
||||||
x-superset-volumes: &superset-volumes
|
|
||||||
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
|
|
||||||
- ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
|
|
||||||
- superset_home:/app/superset_home
|
|
||||||
|
|
||||||
services:
|
|
||||||
app:
|
|
||||||
image: pathogen/monolith:latest
|
|
||||||
container_name: monolith
|
|
||||||
build: ./docker
|
|
||||||
volumes:
|
|
||||||
- ${PORTAINER_GIT_DIR}:/code
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
volumes_from:
|
|
||||||
- tmp
|
|
||||||
depends_on:
|
|
||||||
druid:
|
|
||||||
condition: service_started
|
|
||||||
kafka:
|
|
||||||
condition: service_healthy
|
|
||||||
tmp:
|
|
||||||
condition: service_started
|
|
||||||
redis:
|
|
||||||
condition: service_healthy
|
|
||||||
# - db
|
|
||||||
|
|
||||||
threshold:
|
|
||||||
image: pathogen/threshold:latest
|
|
||||||
container_name: threshold
|
|
||||||
build: ./legacy/docker
|
|
||||||
volumes:
|
|
||||||
- ${PORTAINER_GIT_DIR}:/code
|
|
||||||
- ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
|
|
||||||
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
|
|
||||||
- ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
|
|
||||||
ports:
|
|
||||||
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
|
|
||||||
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
|
|
||||||
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
# for development
|
|
||||||
extra_hosts:
|
|
||||||
- "host.docker.internal:host-gateway"
|
|
||||||
volumes_from:
|
|
||||||
- tmp
|
|
||||||
depends_on:
|
|
||||||
tmp:
|
|
||||||
condition: service_started
|
|
||||||
redis:
|
|
||||||
condition: service_healthy
|
|
||||||
|
|
||||||
# db:
|
|
||||||
#image: pathogen/manticore:kibana
|
|
||||||
# image: manticoresearch/manticore:latest
|
|
||||||
#build:
|
|
||||||
# context: ./docker/manticore
|
|
||||||
# args:
|
|
||||||
# DEV: 1
|
|
||||||
# restart: always
|
|
||||||
|
|
||||||
# turnilo:
|
|
||||||
# container_name: turnilo
|
|
||||||
# image: uchhatre/turnilo:latest
|
|
||||||
# ports:
|
|
||||||
# - 9093:9090
|
|
||||||
# environment:
|
|
||||||
# - DRUID_BROKER_URL=http://broker:8082
|
|
||||||
# - CONFIG_FILE=/config.yaml
|
|
||||||
# volumes:
|
|
||||||
# - ${PORTAINER_GIT_DIR}/docker/turnilo.yaml:/config.yaml
|
|
||||||
# depends_on:
|
|
||||||
# - broker
|
|
||||||
|
|
||||||
# metabase:
|
|
||||||
# container_name: metabase
|
|
||||||
# image: metabase/metabase:latest
|
|
||||||
# ports:
|
|
||||||
# - 3096:3000
|
|
||||||
# environment:
|
|
||||||
# JAVA_OPTS: -Xmx1g
|
|
||||||
# MB_DB_TYPE: postgres
|
|
||||||
# MB_DB_DBNAME: metabase
|
|
||||||
# MB_DB_PORT: 5432
|
|
||||||
# MB_DB_USER: druid
|
|
||||||
# MB_DB_PASS: FoolishPassword
|
|
||||||
# MB_DB_HOST: postgres
|
|
||||||
# depends_on:
|
|
||||||
# - broker
|
|
||||||
|
|
||||||
redis_superset:
|
|
||||||
image: redis:latest
|
|
||||||
container_name: superset_cache
|
|
||||||
restart: unless-stopped
|
|
||||||
volumes:
|
|
||||||
- redis_superset:/data
|
|
||||||
db:
|
|
||||||
env_file: docker/.env-non-dev
|
|
||||||
image: postgres:10
|
|
||||||
container_name: superset_db
|
|
||||||
restart: unless-stopped
|
|
||||||
volumes:
|
|
||||||
- db_home:/var/lib/postgresql/data
|
|
||||||
|
|
||||||
superset:
|
|
||||||
env_file: docker/.env-non-dev
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_app
|
|
||||||
command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
|
|
||||||
user: "root"
|
|
||||||
restart: unless-stopped
|
|
||||||
ports:
|
|
||||||
- 8088:8088
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
superset-init:
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_init
|
|
||||||
command: ["/app/docker/docker-init.sh"]
|
|
||||||
env_file: docker/.env-non-dev
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
user: "root"
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
superset-worker:
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_worker
|
|
||||||
command: ["/app/docker/docker-bootstrap.sh", "worker"]
|
|
||||||
env_file: docker/.env-non-dev
|
|
||||||
restart: unless-stopped
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
user: "root"
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
superset-worker-beat:
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_worker_beat
|
|
||||||
command: ["/app/docker/docker-bootstrap.sh", "beat"]
|
|
||||||
env_file: docker/.env-non-dev
|
|
||||||
restart: unless-stopped
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
user: "root"
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
postgres:
|
|
||||||
container_name: postgres
|
|
||||||
image: postgres:latest
|
|
||||||
volumes:
|
|
||||||
- metadata_data:/var/lib/postgresql/data
|
|
||||||
environment:
|
|
||||||
POSTGRES_PASSWORD: FoolishPassword
|
|
||||||
POSTGRES_USER: druid
|
|
||||||
POSTGRES_DB: druid
|
|
||||||
|
|
||||||
# Need 3.5 or later for container nodes
|
|
||||||
zookeeper:
|
|
||||||
container_name: zookeeper
|
|
||||||
image: zookeeper:3.5
|
|
||||||
ports:
|
|
||||||
- "2181:2181"
|
|
||||||
environment:
|
|
||||||
- ZOO_MY_ID=1
|
|
||||||
|
|
||||||
kafka:
|
|
||||||
image: wurstmeister/kafka:latest
|
|
||||||
container_name: kafka
|
|
||||||
depends_on:
|
|
||||||
- zookeeper
|
|
||||||
ports:
|
|
||||||
- 9092:9092
|
|
||||||
- 29092:29092
|
|
||||||
environment:
|
|
||||||
KAFKA_BROKER_ID: 1
|
|
||||||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
|
||||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
|
||||||
KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
|
||||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
|
||||||
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
|
||||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
||||||
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
|
||||||
KAFKA_MESSAGE_MAX_BYTES: 2000000
|
|
||||||
#KAFKA_HEAP_OPTS: -Xmx2g
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
|
|
||||||
start_period: 15s
|
|
||||||
interval: 30s
|
|
||||||
timeout: 30s
|
|
||||||
retries: 45
|
|
||||||
|
|
||||||
druid:
|
|
||||||
image: pathogen/druid:0.23.0
|
|
||||||
build: ./docker/druid/
|
|
||||||
container_name: druid
|
|
||||||
volumes:
|
|
||||||
- druid_shared:/opt/shared
|
|
||||||
- druid_var:/opt/druid/var
|
|
||||||
depends_on:
|
|
||||||
- zookeeper
|
|
||||||
- postgres
|
|
||||||
ports:
|
|
||||||
- "8081:8081"
|
|
||||||
- "8082:8082"
|
|
||||||
- "8083:8083"
|
|
||||||
- "8888:8888"
|
|
||||||
env_file:
|
|
||||||
- environment
|
|
||||||
|
|
||||||
# coordinator:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./docker/druid/
|
|
||||||
# container_name: coordinator
|
|
||||||
# volumes:
|
|
||||||
# - druid_shared:/opt/shared
|
|
||||||
# - coordinator_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# ports:
|
|
||||||
# - "8081:8081"
|
|
||||||
# command:
|
|
||||||
# - coordinator
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# broker:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./docker/druid/
|
|
||||||
# container_name: broker
|
|
||||||
# volumes:
|
|
||||||
# - broker_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8082:8082"
|
|
||||||
# command:
|
|
||||||
# - broker
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# historical:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./docker/druid/
|
|
||||||
# container_name: historical
|
|
||||||
# volumes:
|
|
||||||
# - druid_shared:/opt/shared
|
|
||||||
# - historical_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8083:8083"
|
|
||||||
# command:
|
|
||||||
# - historical
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# middlemanager:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./docker/druid/
|
|
||||||
# container_name: middlemanager
|
|
||||||
# volumes:
|
|
||||||
# - druid_shared:/opt/shared
|
|
||||||
# - middle_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8091:8091"
|
|
||||||
# - "8100-8105:8100-8105"
|
|
||||||
# command:
|
|
||||||
# - middleManager
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# router:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./docker/druid/
|
|
||||||
# container_name: router
|
|
||||||
# volumes:
|
|
||||||
# - router_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8888:8888"
|
|
||||||
# command:
|
|
||||||
# - router
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# db:
|
|
||||||
# #image: pathogen/manticore:kibana
|
|
||||||
# image: manticoresearch/manticore:dev
|
|
||||||
# #build:
|
|
||||||
# # context: ./docker/manticore
|
|
||||||
# # args:
|
|
||||||
# # DEV: 1
|
|
||||||
# restart: always
|
|
||||||
# ports:
|
|
||||||
# - 9308
|
|
||||||
# - 9312
|
|
||||||
# - 9306
|
|
||||||
# ulimits:
|
|
||||||
# nproc: 65535
|
|
||||||
# nofile:
|
|
||||||
# soft: 65535
|
|
||||||
# hard: 65535
|
|
||||||
# memlock:
|
|
||||||
# soft: -1
|
|
||||||
# hard: -1
|
|
||||||
# environment:
|
|
||||||
# - MCL=1
|
|
||||||
# volumes:
|
|
||||||
# - ./docker/data:/var/lib/manticore
|
|
||||||
# - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
|
||||||
|
|
||||||
tmp:
|
|
||||||
image: busybox
|
|
||||||
command: chmod -R 777 /var/run/redis
|
|
||||||
volumes:
|
|
||||||
- /var/run/redis
|
|
||||||
|
|
||||||
redis:
|
|
||||||
image: redis
|
|
||||||
command: redis-server /etc/redis.conf
|
|
||||||
ulimits:
|
|
||||||
nproc: 65535
|
|
||||||
nofile:
|
|
||||||
soft: 65535
|
|
||||||
hard: 65535
|
|
||||||
volumes:
|
|
||||||
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
|
||||||
- redis_data:/data
|
|
||||||
volumes_from:
|
|
||||||
- tmp
|
|
||||||
healthcheck:
|
|
||||||
test: "redis-cli -s /var/run/redis/redis.sock ping"
|
|
||||||
interval: 2s
|
|
||||||
timeout: 2s
|
|
||||||
retries: 15
|
|
||||||
|
|
||||||
networks:
|
|
||||||
default:
|
|
||||||
external:
|
|
||||||
name: pathogen
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
superset_home:
|
|
||||||
external: false
|
|
||||||
db_home:
|
|
||||||
external: false
|
|
||||||
redis_superset:
|
|
||||||
external: false
|
|
||||||
redis_data: {}
|
|
||||||
metadata_data: {}
|
|
||||||
# middle_var: {}
|
|
||||||
# historical_var: {}
|
|
||||||
# broker_var: {}
|
|
||||||
# coordinator_var: {}
|
|
||||||
druid_var: {}
|
|
||||||
druid_shared: {}
|
|
||||||
27
env.example
27
env.example
@ -1,6 +1,6 @@
|
|||||||
PORTAINER_GIT_DIR=.
|
PORTAINER_GIT_DIR=..
|
||||||
MODULES_ENABLED="dis"
|
MODULES_ENABLED="dis"
|
||||||
DISCORD_TOKEN="xx"
|
DISCORD_TOKEN=
|
||||||
THRESHOLD_LISTENER_HOST=0.0.0.0
|
THRESHOLD_LISTENER_HOST=0.0.0.0
|
||||||
THRESHOLD_LISTENER_PORT=13867
|
THRESHOLD_LISTENER_PORT=13867
|
||||||
THRESHOLD_LISTENER_SSL=1
|
THRESHOLD_LISTENER_SSL=1
|
||||||
@ -13,16 +13,16 @@ THRESHOLD_RELAY_SSL=1
|
|||||||
THRESHOLD_API_ENABLED=1
|
THRESHOLD_API_ENABLED=1
|
||||||
THRESHOLD_API_HOST=0.0.0.0
|
THRESHOLD_API_HOST=0.0.0.0
|
||||||
THRESHOLD_API_PORT=13869
|
THRESHOLD_API_PORT=13869
|
||||||
PORTAINER_GIT_DIR=.
|
|
||||||
|
|
||||||
THRESHOLD_CONFIG_DIR=./legacy/conf/live/
|
THRESHOLD_CONFIG_DIR=../legacy/conf/live/
|
||||||
THRESHOLD_CERT_DIR=./legacy/conf/cert/
|
#THRESHOLD_TEMPLATE_DIR=../legacy/conf/templates/
|
||||||
|
THRESHOLD_CERT_DIR=../legacy/conf/cert/
|
||||||
|
|
||||||
# How many messages to ingest at once from Redis
|
# How many messages to ingest at once from Redis
|
||||||
MONOLITH_INGEST_CHUNK_SIZE=900
|
MONOLITH_INGEST_CHUNK_SIZE=70000
|
||||||
|
|
||||||
# Time to wait between polling Redis again
|
# Time to wait between polling Redis again
|
||||||
MONOLITH_INGEST_ITER_DELAY=0.5
|
MONOLITH_INGEST_ITER_DELAY=2
|
||||||
|
|
||||||
# Number of 4chan threads to request at once
|
# Number of 4chan threads to request at once
|
||||||
MONOLITH_CH4_THREADS_CONCURRENT=1000
|
MONOLITH_CH4_THREADS_CONCURRENT=1000
|
||||||
@ -31,11 +31,20 @@ MONOLITH_CH4_THREADS_CONCURRENT=1000
|
|||||||
MONOLITH_CH4_THREADS_DELAY=0.1
|
MONOLITH_CH4_THREADS_DELAY=0.1
|
||||||
|
|
||||||
# Time to wait after finishing a crawl before starting again
|
# Time to wait after finishing a crawl before starting again
|
||||||
MONOLITH_CH4_CRAWL_DELAY=30
|
MONOLITH_CH4_CRAWL_DELAY=60
|
||||||
|
|
||||||
# Semaphore value
|
# Semaphore value
|
||||||
MONOLITH_CH4_THREADS_SEMAPHORE=1000
|
MONOLITH_CH4_THREADS_SEMAPHORE=1000
|
||||||
|
|
||||||
# Threads to use for data processing
|
# Threads to use for data processing
|
||||||
# Leave uncommented to use all available threads
|
# Leave uncommented to use all available threads
|
||||||
# MONOLITH_PROCESS_THREADS=4
|
MONOLITH_PROCESS_THREADS=7
|
||||||
|
|
||||||
|
# Enable performance metrics after message processing
|
||||||
|
MONOLITH_PROCESS_PERFSTATS=0
|
||||||
|
|
||||||
|
# Elasticsearch
|
||||||
|
ELASTICSEARCH_USERNAME=elastic
|
||||||
|
ELASTICSEARCH_PASSWORD=
|
||||||
|
ELASTICSEARCH_HOST=https://es01:9200
|
||||||
|
ELASTICSEARCH_TLS=1
|
||||||
Loading…
Reference in New Issue
Block a user