From 7482064aeeacd956c548d089a1ec9ea542b97def Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 19 Oct 2022 16:45:18 +0100 Subject: [PATCH] Clean up docker environment --- Makefile | 20 ++++++++ db.py | 2 + docker-compose.yml | 2 +- docker/docker-compose.prod.yml | 7 ++- environment | 87 ---------------------------------- 5 files changed, 29 insertions(+), 89 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8e4e6de --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +run: + docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env up -d + +build: + docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env build + +stop: + docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env down + +log: + docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env logs -f + +run-infra: + docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env up -d + +stop-infra: + docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env down + +log-infra: + docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env logs -f \ No newline at end of file diff --git a/db.py b/db.py index 6b20baf..715751c 100644 --- a/db.py +++ b/db.py @@ -44,6 +44,8 @@ KEYNAME = "queue" async def store_kafka_batch(data): + print("FAKE STORE KAFKA BATCH") + return # log.debug(f"Storing Kafka batch of {len(data)} messages") producer = AIOKafkaProducer(bootstrap_servers="kafka:9092") await producer.start() diff --git a/docker-compose.yml b/docker-compose.yml index ab3f4f3..18a0239 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: volumes_from: - tmp depends_on: - broker: + druid: condition: service_started kafka: condition: service_healthy diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 0c6248d..f189125 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -36,8 +36,11 @@ services: - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" env_file: - ../stack.env + # for development + extra_hosts: + - "host.docker.internal:host-gateway" volumes_from: - - tmp + - tmp depends_on: tmp: condition: service_started @@ -46,12 +49,14 @@ services: tmp: image: busybox + container_name: tmp_monolith command: chmod -R 777 /var/run/redis volumes: - /var/run/redis redis: image: redis + container_name: redis_monolith command: redis-server /etc/redis.conf ulimits: nproc: 65535 diff --git a/environment b/environment index bac4cd4..e69de29 100644 --- a/environment +++ b/environment @@ -1,87 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Java tuning -#DRUID_XMX=1g -#DRUID_XMS=1g -#DRUID_MAXNEWSIZE=250m -#DRUID_NEWSIZE=250m -#DRUID_MAXDIRECTMEMORYSIZE=1g - -#druid_emitter_logging_logLevel=debug - -#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -#druid_zk_service_host=zookeeper - -#druid_metadata_storage_host= -#druid_metadata_storage_type=postgresql -#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -#druid_metadata_storage_connector_user=druid -#druid_metadata_storage_connector_password=FoolishPassword - -#druid_coordinator_balancer_strategy=cachingCost - -#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB -#druid_processing_buffer_sizeBytes=268435456 # 256MiB - -#druid_storage_type=local -#druid_storage_storageDirectory=/opt/shared/segments -#druid_indexer_logs_type=file -#druid_indexer_logs_directory=/opt/shared/indexing-logs - -#druid_processing_numThreads=1 -#druid_processing_numMergeBuffers=1 - -#DRUID_LOG4J= -# Java tuning -#DRUID_XMX=1g -#DRUID_XMS=1g -#DRUID_MAXNEWSIZE=250m -#DRUID_NEWSIZE=250m -#DRUID_MAXDIRECTMEMORYSIZE=6172m -DRUID_SINGLE_NODE_CONF=nano-quickstart - -druid_emitter_logging_logLevel=debug - -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -druid_zk_service_host=zookeeper - -druid_metadata_storage_host= -druid_metadata_storage_type=postgresql -druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=druid -druid_metadata_storage_connector_password=FoolishPassword - -druid_coordinator_balancer_strategy=cachingCost - -druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB - -druid_storage_type=local -druid_storage_storageDirectory=/opt/shared/segments -druid_indexer_logs_type=file -druid_indexer_logs_directory=/opt/shared/indexing-logs - -druid_processing_numThreads=2 -druid_processing_numMergeBuffers=2 - -DRUID_LOG4J=