diff --git a/docker-compose.yml b/docker-compose.yml
index 051d782..e015f45 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -93,9 +93,9 @@ services:
volumes:
- metadata_data:/var/lib/postgresql/data
environment:
- - POSTGRES_PASSWORD=FoolishPassword
- - POSTGRES_USER=druid
- - POSTGRES_DB=druid
+ POSTGRES_PASSWORD: FoolishPassword
+ POSTGRES_USER: druid
+ POSTGRES_DB: druid
# Need 3.5 or later for container nodes
zookeeper:
@@ -107,28 +107,24 @@ services:
- ZOO_MY_ID=1
kafka:
- image: bitnami/kafka
+ image: wurstmeister/kafka:latest
container_name: kafka
depends_on:
- zookeeper
- - broker
ports:
- - 29092:29092
- 9092:9092
+ - 29092:29092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
- #KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+ KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- ALLOW_PLAINTEXT_LISTENER: yes
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+ KAFKA_MESSAGE_MAX_BYTES: 2000000
KAFKA_HEAP_OPTS: -Xmx2g
- # healthcheck:
- # test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"]
- # interval: 2s
- # timeout: 2s
- # retries: 15
healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
start_period: 15s
diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml
index 64fca6c..674608b 100644
--- a/docker/docker-compose.prod.yml
+++ b/docker/docker-compose.prod.yml
@@ -104,28 +104,24 @@ services:
- ZOO_MY_ID=1
kafka:
- image: bitnami/kafka
+ image: wurstmeister/kafka:latest
container_name: kafka
depends_on:
- zookeeper
- - broker
ports:
- - 29092:29092
- 9092:9092
+ - 29092:29092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
- #KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+ KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- ALLOW_PLAINTEXT_LISTENER: yes
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+ KAFKA_MESSAGE_MAX_BYTES: 2000000
KAFKA_HEAP_OPTS: -Xmx2g
- # healthcheck:
- # test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"]
- # interval: 2s
- # timeout: 2s
- # retries: 15
healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
start_period: 15s
diff --git a/docker/environment b/docker/environment
index 6244524..6c76521 100644
--- a/docker/environment
+++ b/docker/environment
@@ -22,7 +22,7 @@ DRUID_XMX=1g
DRUID_XMS=1g
DRUID_MAXNEWSIZE=250m
DRUID_NEWSIZE=250m
-DRUID_MAXDIRECTMEMORYSIZE=2500m
+DRUID_MAXDIRECTMEMORYSIZE=500m
druid_emitter_logging_logLevel=debug
@@ -39,14 +39,15 @@ druid_metadata_storage_connector_password=FoolishPassword
druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=134217728 # 128MiB
+druid_processing_buffer_sizeBytes=134217728 # 128MiB
druid_storage_type=local
druid_storage_storageDirectory=/opt/shared/segments
druid_indexer_logs_type=file
druid_indexer_logs_directory=/opt/shared/indexing-logs
-druid_processing_numThreads=2
-druid_processing_numMergeBuffers=2
+druid_processing_numThreads=1
+druid_processing_numMergeBuffers=1
DRUID_LOG4J=
diff --git a/environment b/environment
index 6244524..6c76521 100644
--- a/environment
+++ b/environment
@@ -22,7 +22,7 @@ DRUID_XMX=1g
DRUID_XMS=1g
DRUID_MAXNEWSIZE=250m
DRUID_NEWSIZE=250m
-DRUID_MAXDIRECTMEMORYSIZE=2500m
+DRUID_MAXDIRECTMEMORYSIZE=500m
druid_emitter_logging_logLevel=debug
@@ -39,14 +39,15 @@ druid_metadata_storage_connector_password=FoolishPassword
druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=1gb", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=134217728 # 128MiB
+druid_processing_buffer_sizeBytes=134217728 # 128MiB
druid_storage_type=local
druid_storage_storageDirectory=/opt/shared/segments
druid_indexer_logs_type=file
druid_indexer_logs_directory=/opt/shared/indexing-logs
-druid_processing_numThreads=2
-druid_processing_numMergeBuffers=2
+druid_processing_numThreads=1
+druid_processing_numMergeBuffers=1
DRUID_LOG4J=