From 395dfb1e7b1f6fa24dcb19f35badf0118e9896bf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 21 Sep 2022 21:11:13 +0100 Subject: [PATCH] Decrease memory requirements further and switch Kafka image --- docker-compose.yml | 24 ++++++++++-------------- docker/docker-compose.prod.yml | 18 +++++++----------- docker/environment | 9 +++++---- environment | 9 +++++---- 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 051d782..e015f45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,9 +93,9 @@ services: volumes: - metadata_data:/var/lib/postgresql/data environment: - - POSTGRES_PASSWORD=FoolishPassword - - POSTGRES_USER=druid - - POSTGRES_DB=druid + POSTGRES_PASSWORD: FoolishPassword + POSTGRES_USER: druid + POSTGRES_DB: druid # Need 3.5 or later for container nodes zookeeper: @@ -107,28 +107,24 @@ services: - ZOO_MY_ID=1 kafka: - image: bitnami/kafka + image: wurstmeister/kafka:latest container_name: kafka depends_on: - zookeeper - - broker ports: - - 29092:29092 - 9092:9092 + - 29092:29092 environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - #KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_MESSAGE_MAX_BYTES: 2000000 KAFKA_HEAP_OPTS: -Xmx2g - # healthcheck: - # test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"] - # interval: 2s - # timeout: 2s - # retries: 15 healthcheck: test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"] start_period: 15s diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 64fca6c..674608b 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -104,28 +104,24 @@ services: - ZOO_MY_ID=1 kafka: - image: bitnami/kafka + image: wurstmeister/kafka:latest container_name: kafka depends_on: - zookeeper - - broker ports: - - 29092:29092 - 9092:9092 + - 29092:29092 environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - #KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_MESSAGE_MAX_BYTES: 2000000 KAFKA_HEAP_OPTS: -Xmx2g - # healthcheck: - # test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"] - # interval: 2s - # timeout: 2s - # retries: 15 healthcheck: test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"] start_period: 15s diff --git a/docker/environment b/docker/environment index 6244524..6c76521 100644 --- a/docker/environment +++ b/docker/environment @@ -22,7 +22,7 @@ DRUID_XMX=1g DRUID_XMS=1g DRUID_MAXNEWSIZE=250m DRUID_NEWSIZE=250m -DRUID_MAXDIRECTMEMORYSIZE=2500m +DRUID_MAXDIRECTMEMORYSIZE=500m druid_emitter_logging_logLevel=debug @@ -39,14 +39,15 @@ druid_metadata_storage_connector_password=FoolishPassword druid_coordinator_balancer_strategy=cachingCost druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=134217728 # 128MiB +druid_processing_buffer_sizeBytes=134217728 # 128MiB druid_storage_type=local druid_storage_storageDirectory=/opt/shared/segments druid_indexer_logs_type=file druid_indexer_logs_directory=/opt/shared/indexing-logs -druid_processing_numThreads=2 -druid_processing_numMergeBuffers=2 +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=1 DRUID_LOG4J= diff --git a/environment b/environment index 6244524..6c76521 100644 --- a/environment +++ b/environment @@ -22,7 +22,7 @@ DRUID_XMX=1g DRUID_XMS=1g DRUID_MAXNEWSIZE=250m DRUID_NEWSIZE=250m -DRUID_MAXDIRECTMEMORYSIZE=2500m +DRUID_MAXDIRECTMEMORYSIZE=500m druid_emitter_logging_logLevel=debug @@ -39,14 +39,15 @@ druid_metadata_storage_connector_password=FoolishPassword druid_coordinator_balancer_strategy=cachingCost druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=134217728 # 128MiB +druid_processing_buffer_sizeBytes=134217728 # 128MiB druid_storage_type=local druid_storage_storageDirectory=/opt/shared/segments druid_indexer_logs_type=file druid_indexer_logs_directory=/opt/shared/indexing-logs -druid_processing_numThreads=2 -druid_processing_numMergeBuffers=2 +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=1 DRUID_LOG4J=