From 8cc1a48a25a4500783292de5524190c8ae428078 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 11 Oct 2022 11:04:03 +0100 Subject: [PATCH] Separate out infra in production --- docker/docker-compose.infra.yml | 142 +++++++++++++++ docker/docker-compose.prod.yml | 298 -------------------------------- docker/turnilo.yaml | 3 + 3 files changed, 145 insertions(+), 298 deletions(-) create mode 100644 docker/docker-compose.infra.yml create mode 100644 docker/turnilo.yaml diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml new file mode 100644 index 0000000..81477d3 --- /dev/null +++ b/docker/docker-compose.infra.yml @@ -0,0 +1,142 @@ +version: "2.2" + +x-superset-image: &superset-image apache/superset:${TAG:-latest-dev} +x-superset-depends-on: &superset-depends-on + - db + - redis_superset +x-superset-volumes: &superset-volumes + # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container + - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker + - superset_home:/app/superset_home + +services: + redis_superset: + image: redis:latest + container_name: superset_cache + restart: unless-stopped + volumes: + - redis:/data + db: + env_file: .env-non-dev + image: postgres:10 + container_name: superset_db + restart: unless-stopped + volumes: + - db_home:/var/lib/postgresql/data + + superset: + env_file: .env-non-dev + image: *superset-image + container_name: superset_app + command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] + user: "root" + restart: unless-stopped + ports: + - 8088:8088 + depends_on: *superset-depends-on + volumes: *superset-volumes + + superset-init: + image: *superset-image + container_name: superset_init + command: ["/app/docker/docker-init.sh"] + env_file: .env-non-dev + depends_on: *superset-depends-on + user: "root" + volumes: *superset-volumes + + superset-worker: + image: *superset-image + container_name: superset_worker + command: ["/app/docker/docker-bootstrap.sh", "worker"] + env_file: .env-non-dev + restart: unless-stopped + depends_on: *superset-depends-on + user: "root" + volumes: *superset-volumes + + superset-worker-beat: + image: *superset-image + container_name: superset_worker_beat + command: ["/app/docker/docker-bootstrap.sh", "beat"] + env_file: .env-non-dev + restart: unless-stopped + depends_on: *superset-depends-on + user: "root" + volumes: *superset-volumes + + postgres: + container_name: postgres + image: postgres:latest + volumes: + - /block/store/metadata_data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=FoolishPassword + - POSTGRES_USER=druid + - POSTGRES_DB=druid + + # Need 3.5 or later for container nodes + zookeeper: + container_name: zookeeper + image: zookeeper:3.5 + ports: + - "2181:2181" + environment: + - ZOO_MY_ID=1 + + kafka: + image: wurstmeister/kafka:latest + container_name: kafka + depends_on: + - zookeeper + ports: + - 9092:9092 + - 29092:29092 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_MESSAGE_MAX_BYTES: 2000000 + #KAFKA_HEAP_OPTS: -Xmx2g + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"] + start_period: 15s + interval: 30s + timeout: 30s + retries: 45 + + druid: + image: pathogen/druid:0.23.0 + build: ./druid/ + container_name: druid + volumes: + - /block/store/druid_shared:/opt/shared + - /block/store/druid_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + ports: + - "8081:8081" + - "8082:8082" + - "8083:8083" + - "8888:8888" + env_file: + - environment + +networks: + default: + external: + name: pathogen + +volumes: + superset_home: + external: false + db_home: + external: false + redis: + external: false \ No newline at end of file diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 2339858..27901a6 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -1,23 +1,5 @@ version: "2.2" -# volumes: -# metadata_data: {} -# middle_var: {} -# historical_var: {} -# broker_var: {} -# coordinator_var: {} -# router_var: {} -# druid_shared: {} - -x-superset-image: &superset-image apache/superset:${TAG:-latest-dev} -x-superset-depends-on: &superset-depends-on - - db - - redis_superset -x-superset-volumes: &superset-volumes - # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container - - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker - - superset_home:/app/superset_home - services: app: image: pathogen/monolith:latest @@ -38,7 +20,6 @@ services: condition: service_started redis: condition: service_healthy - # - db threshold: image: pathogen/threshold:latest @@ -63,279 +44,6 @@ services: redis: condition: service_healthy - # db: - #image: pathogen/manticore:kibana - # image: manticoresearch/manticore:latest - #build: - # context: ./docker/manticore - # args: - # DEV: 1 - # restart: always - - - # turnilo: - # container_name: turnilo - # image: uchhatre/turnilo:latest - # ports: - # - 9093:9090 - # environment: - # - DRUID_BROKER_URL=http://broker:8082 - # depends_on: - # - broker - - # metabase: - # container_name: metabase - # image: metabase/metabase:latest - # ports: - # - 3096:3000 - # environment: - # JAVA_OPTS: -Xmx1g - # MB_DB_TYPE: postgres - # MB_DB_DBNAME: metabase - # MB_DB_PORT: 5432 - # MB_DB_USER: druid - # MB_DB_PASS: FoolishPassword - # MB_DB_HOST: postgres - # depends_on: - # - broker - - redis_superset: - image: redis:latest - container_name: superset_cache - restart: unless-stopped - volumes: - - redis:/data - db: - env_file: .env-non-dev - image: postgres:10 - container_name: superset_db - restart: unless-stopped - volumes: - - db_home:/var/lib/postgresql/data - - superset: - env_file: .env-non-dev - image: *superset-image - container_name: superset_app - command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] - user: "root" - restart: unless-stopped - ports: - - 8088:8088 - depends_on: *superset-depends-on - volumes: *superset-volumes - - superset-init: - image: *superset-image - container_name: superset_init - command: ["/app/docker/docker-init.sh"] - env_file: .env-non-dev - depends_on: *superset-depends-on - user: "root" - volumes: *superset-volumes - - superset-worker: - image: *superset-image - container_name: superset_worker - command: ["/app/docker/docker-bootstrap.sh", "worker"] - env_file: .env-non-dev - restart: unless-stopped - depends_on: *superset-depends-on - user: "root" - volumes: *superset-volumes - - superset-worker-beat: - image: *superset-image - container_name: superset_worker_beat - command: ["/app/docker/docker-bootstrap.sh", "beat"] - env_file: .env-non-dev - restart: unless-stopped - depends_on: *superset-depends-on - user: "root" - volumes: *superset-volumes - - postgres: - container_name: postgres - image: postgres:latest - volumes: - - /block/store/metadata_data:/var/lib/postgresql/data - environment: - - POSTGRES_PASSWORD=FoolishPassword - - POSTGRES_USER=druid - - POSTGRES_DB=druid - - # Need 3.5 or later for container nodes - zookeeper: - container_name: zookeeper - image: zookeeper:3.5 - ports: - - "2181:2181" - environment: - - ZOO_MY_ID=1 - - kafka: - image: wurstmeister/kafka:latest - container_name: kafka - depends_on: - - zookeeper - ports: - - 9092:9092 - - 29092:29092 - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' - KAFKA_MESSAGE_MAX_BYTES: 2000000 - #KAFKA_HEAP_OPTS: -Xmx2g - healthcheck: - test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"] - start_period: 15s - interval: 30s - timeout: 30s - retries: 45 - - druid: - image: pathogen/druid:0.23.0 - build: ./druid/ - container_name: druid - volumes: - - /block/store/druid_shared:/opt/shared - - /block/store/druid_var:/opt/druid/var - depends_on: - - zookeeper - - postgres - ports: - - "8081:8081" - - "8082:8082" - - "8083:8083" - - "8888:8888" - env_file: - - environment - - # coordinator: - # #image: apache/druid:0.23.0 - # image: pathogen/druid:0.23.0 - # build: ./druid - # container_name: coordinator - # volumes: - # - /block/store/druid_shared:/opt/shared - # - /block/store/coordinator_var:/opt/druid/var - # depends_on: - # - zookeeper - # - postgres - # ports: - # - "8081:8081" - # command: - # - coordinator - # env_file: - # - environment - - # broker: - # #image: apache/druid:0.23.0 - # image: pathogen/druid:0.23.0 - # build: ./druid - # container_name: broker - # volumes: - # - /block/store/broker_var:/opt/druid/var - # depends_on: - # - zookeeper - # - postgres - # - coordinator - # ports: - # - "8082:8082" - # command: - # - broker - # env_file: - # - environment - - # historical: - # #image: apache/druid:0.23.0 - # image: pathogen/druid:0.23.0 - # build: ./druid - # container_name: historical - # volumes: - # - /block/store/druid_shared:/opt/shared - # - /block/store/historical_var:/opt/druid/var - # depends_on: - # - zookeeper - # - postgres - # - coordinator - # ports: - # - "8083:8083" - # command: - # - historical - # env_file: - # - environment - - # middlemanager: - # #image: apache/druid:0.23.0 - # image: pathogen/druid:0.23.0 - # build: ./druid - # container_name: middlemanager - # volumes: - # - /block/store/druid_shared:/opt/shared - # - /block/store/middle_var:/opt/druid/var - # depends_on: - # - zookeeper - # - postgres - # - coordinator - # ports: - # - "8091:8091" - # - "8100-8105:8100-8105" - # command: - # - middleManager - # env_file: - # - environment - - # router: - # #image: apache/druid:0.23.0 - # image: pathogen/druid:0.23.0 - # build: ./druid - # container_name: router - # volumes: - # - /block/store/router_var:/opt/druid/var - # depends_on: - # - zookeeper - # - postgres - # - coordinator - # ports: - # - "8888:8888" - # command: - # - router - # env_file: - # - environment - - # db: - # #image: pathogen/manticore:kibana - # image: manticoresearch/manticore:dev - # #build: - # # context: ./docker/manticore - # # args: - # # DEV: 1 - # restart: always - # ports: - # - 9308 - # - 9312 - # - 9306 - # ulimits: - # nproc: 65535 - # nofile: - # soft: 65535 - # hard: 65535 - # memlock: - # soft: -1 - # hard: -1 - # environment: - # - MCL=1 - # volumes: - # - ./docker/data:/var/lib/manticore - # - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf - tmp: image: busybox command: chmod -R 777 /var/run/redis @@ -368,9 +76,3 @@ networks: volumes: redis_data: {} - superset_home: - external: false - db_home: - external: false - redis: - external: false \ No newline at end of file diff --git a/docker/turnilo.yaml b/docker/turnilo.yaml new file mode 100644 index 0000000..d3105dd --- /dev/null +++ b/docker/turnilo.yaml @@ -0,0 +1,3 @@ +clusters: + - name: druid + guardDataCubes: true \ No newline at end of file