Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Decrease memory requirements further and switch Kafka image

This commit is contained in:
Mark Veidemanis 2022-09-21 21:11:13 +01:00
parent ee79762c73
commit 395dfb1e7b
4 changed files with 27 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
docker-compose.yml
View File

@ -93,9 +93,9 @@ services:
volumes: volumes:
- metadata_data:/var/lib/postgresql/data - metadata_data:/var/lib/postgresql/data
environment: environment:
- POSTGRES_PASSWORD=FoolishPassword POSTGRES_PASSWORD: FoolishPassword
- POSTGRES_USER=druid POSTGRES_USER: druid
- POSTGRES_DB=druid POSTGRES_DB: druid
# Need 3.5 or later for container nodes # Need 3.5 or later for container nodes
zookeeper: zookeeper:
@ -107,28 +107,24 @@ services:
- ZOO_MY_ID=1 - ZOO_MY_ID=1
kafka: kafka:
image: bitnami/kafka image: wurstmeister/kafka:latest
container_name: kafka container_name: kafka
depends_on: depends_on:
- zookeeper - zookeeper
- broker
ports: ports:
- 29092:29092
- 9092:9092 - 9092:9092
- 29092:29092
environment: environment:
KAFKA_BROKER_ID: 1 KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
#KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
ALLOW_PLAINTEXT_LISTENER: yes KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_MESSAGE_MAX_BYTES: 2000000
KAFKA_HEAP_OPTS: -Xmx2g KAFKA_HEAP_OPTS: -Xmx2g
# healthcheck:
# test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"]
# interval: 2s
# timeout: 2s
# retries: 15
healthcheck: healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"] test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
start_period: 15s start_period: 15s

18
docker/docker-compose.prod.yml
View File

@ -104,28 +104,24 @@ services:
- ZOO_MY_ID=1 - ZOO_MY_ID=1
kafka: kafka:
image: bitnami/kafka image: wurstmeister/kafka:latest
container_name: kafka container_name: kafka
depends_on: depends_on:
- zookeeper - zookeeper
- broker
ports: ports:
- 29092:29092
- 9092:9092 - 9092:9092
- 29092:29092
environment: environment:
KAFKA_BROKER_ID: 1 KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
#KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
ALLOW_PLAINTEXT_LISTENER: yes KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_MESSAGE_MAX_BYTES: 2000000
KAFKA_HEAP_OPTS: -Xmx2g KAFKA_HEAP_OPTS: -Xmx2g
# healthcheck:
# test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"]
# interval: 2s
# timeout: 2s
# retries: 15
healthcheck: healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"] test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
start_period: 15s start_period: 15s

9
docker/environment
View File

@ -22,7 +22,7 @@ DRUID_XMX=1g
DRUID_XMS=1g DRUID_XMS=1g
DRUID_MAXNEWSIZE=250m DRUID_MAXNEWSIZE=250m
DRUID_NEWSIZE=250m DRUID_NEWSIZE=250m
DRUID_MAXDIRECTMEMORYSIZE=2500m DRUID_MAXDIRECTMEMORYSIZE=500m
druid_emitter_logging_logLevel=debug druid_emitter_logging_logLevel=debug
@ -39,14 +39,15 @@ druid_metadata_storage_connector_password=FoolishPassword
druid_coordinator_balancer_strategy=cachingCost druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB druid_indexer_fork_property_druid_processing_buffer_sizeBytes=134217728 # 128MiB
druid_processing_buffer_sizeBytes=134217728 # 128MiB
druid_storage_type=local druid_storage_type=local
druid_storage_storageDirectory=/opt/shared/segments druid_storage_storageDirectory=/opt/shared/segments
druid_indexer_logs_type=file druid_indexer_logs_type=file
druid_indexer_logs_directory=/opt/shared/indexing-logs druid_indexer_logs_directory=/opt/shared/indexing-logs
druid_processing_numThreads=2 druid_processing_numThreads=1
druid_processing_numMergeBuffers=2 druid_processing_numMergeBuffers=1
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration> DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

9
environment
View File

@ -22,7 +22,7 @@ DRUID_XMX=1g
DRUID_XMS=1g DRUID_XMS=1g
DRUID_MAXNEWSIZE=250m DRUID_MAXNEWSIZE=250m
DRUID_NEWSIZE=250m DRUID_NEWSIZE=250m
DRUID_MAXDIRECTMEMORYSIZE=2500m DRUID_MAXDIRECTMEMORYSIZE=500m
druid_emitter_logging_logLevel=debug druid_emitter_logging_logLevel=debug
@ -39,14 +39,15 @@ druid_metadata_storage_connector_password=FoolishPassword
druid_coordinator_balancer_strategy=cachingCost druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB druid_indexer_fork_property_druid_processing_buffer_sizeBytes=134217728 # 128MiB
druid_processing_buffer_sizeBytes=134217728 # 128MiB
druid_storage_type=local druid_storage_type=local
druid_storage_storageDirectory=/opt/shared/segments druid_storage_storageDirectory=/opt/shared/segments
druid_indexer_logs_type=file druid_indexer_logs_type=file
druid_indexer_logs_directory=/opt/shared/indexing-logs druid_indexer_logs_directory=/opt/shared/indexing-logs
druid_processing_numThreads=2 druid_processing_numThreads=1
druid_processing_numMergeBuffers=2 druid_processing_numMergeBuffers=1
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration> DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>