From cc6340acabffa768139ca989711713cc357d6a3e Mon Sep 17 00:00:00 2001
From: Mark Veidemanis <m@zm.is>
Date: Tue, 4 Oct 2022 20:26:58 +0100
Subject: [PATCH] Add persistent Redis data store and copy over Druid config to
 production

---
 docker-compose.yml             |  4 ++-
 docker/docker-compose.prod.yml |  2 ++
 docker/environment             | 54 +++++++++++++++++++++++++++-------
 3 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 621b96d..3305ff6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -320,6 +320,7 @@ services:
          hard: 65535
     volumes:
       - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
     volumes_from:
       - tmp
     healthcheck:
@@ -338,8 +339,9 @@ volumes:
     external: false
   db_home:
     external: false
-  redis:
+  redis_superset:
     external: false
+  redis_data: {}
   metadata_data: {}
   middle_var: {}
   historical_var: {}
diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml
index 4b0d476..fae70cc 100644
--- a/docker/docker-compose.prod.yml
+++ b/docker/docker-compose.prod.yml
@@ -324,6 +324,7 @@ services:
          hard: 65535
     volumes:
       - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
     volumes_from:
       - tmp
     healthcheck:
@@ -338,6 +339,7 @@ networks:
          name: pathogen
 
 volumes:
+  redis_data: {}
   superset_home:
     external: false
   db_home:
diff --git a/docker/environment b/docker/environment
index 73fc2e7..d5a6398 100644
--- a/docker/environment
+++ b/docker/environment
@@ -18,11 +18,46 @@
 #
 
 # Java tuning
-DRUID_XMX=1g
-DRUID_XMS=1g
-DRUID_MAXNEWSIZE=250m
-DRUID_NEWSIZE=250m
-DRUID_MAXDIRECTMEMORYSIZE=500m
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=1g
+
+#druid_emitter_logging_logLevel=debug
+
+#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+#druid_zk_service_host=zookeeper
+
+#druid_metadata_storage_host=
+#druid_metadata_storage_type=postgresql
+#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+#druid_metadata_storage_connector_user=druid
+#druid_metadata_storage_connector_password=FoolishPassword
+
+#druid_coordinator_balancer_strategy=cachingCost
+
+#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
+#druid_processing_buffer_sizeBytes=268435456 # 256MiB
+
+#druid_storage_type=local
+#druid_storage_storageDirectory=/opt/shared/segments
+#druid_indexer_logs_type=file
+#druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+#druid_processing_numThreads=1
+#druid_processing_numMergeBuffers=1
+
+#DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=6172m
+DRUID_SINGLE_NODE_CONF=micro-quickstart
 
 druid_emitter_logging_logLevel=debug
 
@@ -39,15 +74,14 @@ druid_metadata_storage_connector_password=FoolishPassword
 druid_coordinator_balancer_strategy=cachingCost
 
 druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
-druid_processing_buffer_sizeBytes=134217728 # 128MiB
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
 
 druid_storage_type=local
 druid_storage_storageDirectory=/opt/shared/segments
 druid_indexer_logs_type=file
 druid_indexer_logs_directory=/opt/shared/indexing-logs
 
-druid_processing_numThreads=1
-druid_processing_numMergeBuffers=1
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
 
-DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
\ No newline at end of file