Separate out infra in production
This commit is contained in:
parent
83e8fb0e38
commit
8cc1a48a25
142
docker/docker-compose.infra.yml
Normal file
142
docker/docker-compose.infra.yml
Normal file
@ -0,0 +1,142 @@
|
|||||||
|
version: "2.2"
|
||||||
|
|
||||||
|
x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
|
||||||
|
x-superset-depends-on: &superset-depends-on
|
||||||
|
- db
|
||||||
|
- redis_superset
|
||||||
|
x-superset-volumes: &superset-volumes
|
||||||
|
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
|
||||||
|
- ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
|
||||||
|
- superset_home:/app/superset_home
|
||||||
|
|
||||||
|
services:
|
||||||
|
redis_superset:
|
||||||
|
image: redis:latest
|
||||||
|
container_name: superset_cache
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- redis:/data
|
||||||
|
db:
|
||||||
|
env_file: .env-non-dev
|
||||||
|
image: postgres:10
|
||||||
|
container_name: superset_db
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- db_home:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
superset:
|
||||||
|
env_file: .env-non-dev
|
||||||
|
image: *superset-image
|
||||||
|
container_name: superset_app
|
||||||
|
command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
|
||||||
|
user: "root"
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- 8088:8088
|
||||||
|
depends_on: *superset-depends-on
|
||||||
|
volumes: *superset-volumes
|
||||||
|
|
||||||
|
superset-init:
|
||||||
|
image: *superset-image
|
||||||
|
container_name: superset_init
|
||||||
|
command: ["/app/docker/docker-init.sh"]
|
||||||
|
env_file: .env-non-dev
|
||||||
|
depends_on: *superset-depends-on
|
||||||
|
user: "root"
|
||||||
|
volumes: *superset-volumes
|
||||||
|
|
||||||
|
superset-worker:
|
||||||
|
image: *superset-image
|
||||||
|
container_name: superset_worker
|
||||||
|
command: ["/app/docker/docker-bootstrap.sh", "worker"]
|
||||||
|
env_file: .env-non-dev
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on: *superset-depends-on
|
||||||
|
user: "root"
|
||||||
|
volumes: *superset-volumes
|
||||||
|
|
||||||
|
superset-worker-beat:
|
||||||
|
image: *superset-image
|
||||||
|
container_name: superset_worker_beat
|
||||||
|
command: ["/app/docker/docker-bootstrap.sh", "beat"]
|
||||||
|
env_file: .env-non-dev
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on: *superset-depends-on
|
||||||
|
user: "root"
|
||||||
|
volumes: *superset-volumes
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
container_name: postgres
|
||||||
|
image: postgres:latest
|
||||||
|
volumes:
|
||||||
|
- /block/store/metadata_data:/var/lib/postgresql/data
|
||||||
|
environment:
|
||||||
|
- POSTGRES_PASSWORD=FoolishPassword
|
||||||
|
- POSTGRES_USER=druid
|
||||||
|
- POSTGRES_DB=druid
|
||||||
|
|
||||||
|
# Need 3.5 or later for container nodes
|
||||||
|
zookeeper:
|
||||||
|
container_name: zookeeper
|
||||||
|
image: zookeeper:3.5
|
||||||
|
ports:
|
||||||
|
- "2181:2181"
|
||||||
|
environment:
|
||||||
|
- ZOO_MY_ID=1
|
||||||
|
|
||||||
|
kafka:
|
||||||
|
image: wurstmeister/kafka:latest
|
||||||
|
container_name: kafka
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
ports:
|
||||||
|
- 9092:9092
|
||||||
|
- 29092:29092
|
||||||
|
environment:
|
||||||
|
KAFKA_BROKER_ID: 1
|
||||||
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||||
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
||||||
|
KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
||||||
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||||
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
||||||
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||||
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
||||||
|
KAFKA_MESSAGE_MAX_BYTES: 2000000
|
||||||
|
#KAFKA_HEAP_OPTS: -Xmx2g
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
|
||||||
|
start_period: 15s
|
||||||
|
interval: 30s
|
||||||
|
timeout: 30s
|
||||||
|
retries: 45
|
||||||
|
|
||||||
|
druid:
|
||||||
|
image: pathogen/druid:0.23.0
|
||||||
|
build: ./druid/
|
||||||
|
container_name: druid
|
||||||
|
volumes:
|
||||||
|
- /block/store/druid_shared:/opt/shared
|
||||||
|
- /block/store/druid_var:/opt/druid/var
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- postgres
|
||||||
|
ports:
|
||||||
|
- "8081:8081"
|
||||||
|
- "8082:8082"
|
||||||
|
- "8083:8083"
|
||||||
|
- "8888:8888"
|
||||||
|
env_file:
|
||||||
|
- environment
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
external:
|
||||||
|
name: pathogen
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
superset_home:
|
||||||
|
external: false
|
||||||
|
db_home:
|
||||||
|
external: false
|
||||||
|
redis:
|
||||||
|
external: false
|
||||||
@ -1,23 +1,5 @@
|
|||||||
version: "2.2"
|
version: "2.2"
|
||||||
|
|
||||||
# volumes:
|
|
||||||
# metadata_data: {}
|
|
||||||
# middle_var: {}
|
|
||||||
# historical_var: {}
|
|
||||||
# broker_var: {}
|
|
||||||
# coordinator_var: {}
|
|
||||||
# router_var: {}
|
|
||||||
# druid_shared: {}
|
|
||||||
|
|
||||||
x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
|
|
||||||
x-superset-depends-on: &superset-depends-on
|
|
||||||
- db
|
|
||||||
- redis_superset
|
|
||||||
x-superset-volumes: &superset-volumes
|
|
||||||
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
|
|
||||||
- ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
|
|
||||||
- superset_home:/app/superset_home
|
|
||||||
|
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
image: pathogen/monolith:latest
|
image: pathogen/monolith:latest
|
||||||
@ -38,7 +20,6 @@ services:
|
|||||||
condition: service_started
|
condition: service_started
|
||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
# - db
|
|
||||||
|
|
||||||
threshold:
|
threshold:
|
||||||
image: pathogen/threshold:latest
|
image: pathogen/threshold:latest
|
||||||
@ -63,279 +44,6 @@ services:
|
|||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
||||||
# db:
|
|
||||||
#image: pathogen/manticore:kibana
|
|
||||||
# image: manticoresearch/manticore:latest
|
|
||||||
#build:
|
|
||||||
# context: ./docker/manticore
|
|
||||||
# args:
|
|
||||||
# DEV: 1
|
|
||||||
# restart: always
|
|
||||||
|
|
||||||
|
|
||||||
# turnilo:
|
|
||||||
# container_name: turnilo
|
|
||||||
# image: uchhatre/turnilo:latest
|
|
||||||
# ports:
|
|
||||||
# - 9093:9090
|
|
||||||
# environment:
|
|
||||||
# - DRUID_BROKER_URL=http://broker:8082
|
|
||||||
# depends_on:
|
|
||||||
# - broker
|
|
||||||
|
|
||||||
# metabase:
|
|
||||||
# container_name: metabase
|
|
||||||
# image: metabase/metabase:latest
|
|
||||||
# ports:
|
|
||||||
# - 3096:3000
|
|
||||||
# environment:
|
|
||||||
# JAVA_OPTS: -Xmx1g
|
|
||||||
# MB_DB_TYPE: postgres
|
|
||||||
# MB_DB_DBNAME: metabase
|
|
||||||
# MB_DB_PORT: 5432
|
|
||||||
# MB_DB_USER: druid
|
|
||||||
# MB_DB_PASS: FoolishPassword
|
|
||||||
# MB_DB_HOST: postgres
|
|
||||||
# depends_on:
|
|
||||||
# - broker
|
|
||||||
|
|
||||||
redis_superset:
|
|
||||||
image: redis:latest
|
|
||||||
container_name: superset_cache
|
|
||||||
restart: unless-stopped
|
|
||||||
volumes:
|
|
||||||
- redis:/data
|
|
||||||
db:
|
|
||||||
env_file: .env-non-dev
|
|
||||||
image: postgres:10
|
|
||||||
container_name: superset_db
|
|
||||||
restart: unless-stopped
|
|
||||||
volumes:
|
|
||||||
- db_home:/var/lib/postgresql/data
|
|
||||||
|
|
||||||
superset:
|
|
||||||
env_file: .env-non-dev
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_app
|
|
||||||
command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
|
|
||||||
user: "root"
|
|
||||||
restart: unless-stopped
|
|
||||||
ports:
|
|
||||||
- 8088:8088
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
superset-init:
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_init
|
|
||||||
command: ["/app/docker/docker-init.sh"]
|
|
||||||
env_file: .env-non-dev
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
user: "root"
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
superset-worker:
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_worker
|
|
||||||
command: ["/app/docker/docker-bootstrap.sh", "worker"]
|
|
||||||
env_file: .env-non-dev
|
|
||||||
restart: unless-stopped
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
user: "root"
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
superset-worker-beat:
|
|
||||||
image: *superset-image
|
|
||||||
container_name: superset_worker_beat
|
|
||||||
command: ["/app/docker/docker-bootstrap.sh", "beat"]
|
|
||||||
env_file: .env-non-dev
|
|
||||||
restart: unless-stopped
|
|
||||||
depends_on: *superset-depends-on
|
|
||||||
user: "root"
|
|
||||||
volumes: *superset-volumes
|
|
||||||
|
|
||||||
postgres:
|
|
||||||
container_name: postgres
|
|
||||||
image: postgres:latest
|
|
||||||
volumes:
|
|
||||||
- /block/store/metadata_data:/var/lib/postgresql/data
|
|
||||||
environment:
|
|
||||||
- POSTGRES_PASSWORD=FoolishPassword
|
|
||||||
- POSTGRES_USER=druid
|
|
||||||
- POSTGRES_DB=druid
|
|
||||||
|
|
||||||
# Need 3.5 or later for container nodes
|
|
||||||
zookeeper:
|
|
||||||
container_name: zookeeper
|
|
||||||
image: zookeeper:3.5
|
|
||||||
ports:
|
|
||||||
- "2181:2181"
|
|
||||||
environment:
|
|
||||||
- ZOO_MY_ID=1
|
|
||||||
|
|
||||||
kafka:
|
|
||||||
image: wurstmeister/kafka:latest
|
|
||||||
container_name: kafka
|
|
||||||
depends_on:
|
|
||||||
- zookeeper
|
|
||||||
ports:
|
|
||||||
- 9092:9092
|
|
||||||
- 29092:29092
|
|
||||||
environment:
|
|
||||||
KAFKA_BROKER_ID: 1
|
|
||||||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
|
||||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
|
||||||
KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
|
||||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
|
||||||
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
|
||||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
||||||
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
|
||||||
KAFKA_MESSAGE_MAX_BYTES: 2000000
|
|
||||||
#KAFKA_HEAP_OPTS: -Xmx2g
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
|
|
||||||
start_period: 15s
|
|
||||||
interval: 30s
|
|
||||||
timeout: 30s
|
|
||||||
retries: 45
|
|
||||||
|
|
||||||
druid:
|
|
||||||
image: pathogen/druid:0.23.0
|
|
||||||
build: ./druid/
|
|
||||||
container_name: druid
|
|
||||||
volumes:
|
|
||||||
- /block/store/druid_shared:/opt/shared
|
|
||||||
- /block/store/druid_var:/opt/druid/var
|
|
||||||
depends_on:
|
|
||||||
- zookeeper
|
|
||||||
- postgres
|
|
||||||
ports:
|
|
||||||
- "8081:8081"
|
|
||||||
- "8082:8082"
|
|
||||||
- "8083:8083"
|
|
||||||
- "8888:8888"
|
|
||||||
env_file:
|
|
||||||
- environment
|
|
||||||
|
|
||||||
# coordinator:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./druid
|
|
||||||
# container_name: coordinator
|
|
||||||
# volumes:
|
|
||||||
# - /block/store/druid_shared:/opt/shared
|
|
||||||
# - /block/store/coordinator_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# ports:
|
|
||||||
# - "8081:8081"
|
|
||||||
# command:
|
|
||||||
# - coordinator
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# broker:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./druid
|
|
||||||
# container_name: broker
|
|
||||||
# volumes:
|
|
||||||
# - /block/store/broker_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8082:8082"
|
|
||||||
# command:
|
|
||||||
# - broker
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# historical:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./druid
|
|
||||||
# container_name: historical
|
|
||||||
# volumes:
|
|
||||||
# - /block/store/druid_shared:/opt/shared
|
|
||||||
# - /block/store/historical_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8083:8083"
|
|
||||||
# command:
|
|
||||||
# - historical
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# middlemanager:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./druid
|
|
||||||
# container_name: middlemanager
|
|
||||||
# volumes:
|
|
||||||
# - /block/store/druid_shared:/opt/shared
|
|
||||||
# - /block/store/middle_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8091:8091"
|
|
||||||
# - "8100-8105:8100-8105"
|
|
||||||
# command:
|
|
||||||
# - middleManager
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# router:
|
|
||||||
# #image: apache/druid:0.23.0
|
|
||||||
# image: pathogen/druid:0.23.0
|
|
||||||
# build: ./druid
|
|
||||||
# container_name: router
|
|
||||||
# volumes:
|
|
||||||
# - /block/store/router_var:/opt/druid/var
|
|
||||||
# depends_on:
|
|
||||||
# - zookeeper
|
|
||||||
# - postgres
|
|
||||||
# - coordinator
|
|
||||||
# ports:
|
|
||||||
# - "8888:8888"
|
|
||||||
# command:
|
|
||||||
# - router
|
|
||||||
# env_file:
|
|
||||||
# - environment
|
|
||||||
|
|
||||||
# db:
|
|
||||||
# #image: pathogen/manticore:kibana
|
|
||||||
# image: manticoresearch/manticore:dev
|
|
||||||
# #build:
|
|
||||||
# # context: ./docker/manticore
|
|
||||||
# # args:
|
|
||||||
# # DEV: 1
|
|
||||||
# restart: always
|
|
||||||
# ports:
|
|
||||||
# - 9308
|
|
||||||
# - 9312
|
|
||||||
# - 9306
|
|
||||||
# ulimits:
|
|
||||||
# nproc: 65535
|
|
||||||
# nofile:
|
|
||||||
# soft: 65535
|
|
||||||
# hard: 65535
|
|
||||||
# memlock:
|
|
||||||
# soft: -1
|
|
||||||
# hard: -1
|
|
||||||
# environment:
|
|
||||||
# - MCL=1
|
|
||||||
# volumes:
|
|
||||||
# - ./docker/data:/var/lib/manticore
|
|
||||||
# - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
|
||||||
|
|
||||||
tmp:
|
tmp:
|
||||||
image: busybox
|
image: busybox
|
||||||
command: chmod -R 777 /var/run/redis
|
command: chmod -R 777 /var/run/redis
|
||||||
@ -368,9 +76,3 @@ networks:
|
|||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
redis_data: {}
|
redis_data: {}
|
||||||
superset_home:
|
|
||||||
external: false
|
|
||||||
db_home:
|
|
||||||
external: false
|
|
||||||
redis:
|
|
||||||
external: false
|
|
||||||
3
docker/turnilo.yaml
Normal file
3
docker/turnilo.yaml
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
clusters:
|
||||||
|
- name: druid
|
||||||
|
guardDataCubes: true
|
||||||
Loading…
Reference in New Issue
Block a user