Add persistent Redis data store and copy over Druid config to production

docker-compose.yml

@@ -320,6 +320,7 @@ services:
          hard: 65535
     volumes:
       - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
     volumes_from:
       - tmp
     healthcheck:
@@ -338,8 +339,9 @@ volumes:
     external: false
   db_home:
     external: false
-  redis:
+  redis_superset:
     external: false
+  redis_data: {}
   metadata_data: {}
   middle_var: {}
   historical_var: {}

docker/docker-compose.prod.yml

@@ -324,6 +324,7 @@ services:
          hard: 65535
     volumes:
       - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
     volumes_from:
       - tmp
     healthcheck:
@@ -338,6 +339,7 @@ networks:
          name: pathogen
 
 volumes:
+  redis_data: {}
   superset_home:
     external: false
   db_home:

docker/environment

@@ -18,11 +18,46 @@
 #
 
 # Java tuning
-DRUID_XMX=1g
+#DRUID_XMX=1g
-DRUID_XMS=1g
+#DRUID_XMS=1g
-DRUID_MAXNEWSIZE=250m
+#DRUID_MAXNEWSIZE=250m
-DRUID_NEWSIZE=250m
+#DRUID_NEWSIZE=250m
-DRUID_MAXDIRECTMEMORYSIZE=500m
+#DRUID_MAXDIRECTMEMORYSIZE=1g
+
+#druid_emitter_logging_logLevel=debug
+
+#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+#druid_zk_service_host=zookeeper
+
+#druid_metadata_storage_host=
+#druid_metadata_storage_type=postgresql
+#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+#druid_metadata_storage_connector_user=druid
+#druid_metadata_storage_connector_password=FoolishPassword
+
+#druid_coordinator_balancer_strategy=cachingCost
+
+#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
+#druid_processing_buffer_sizeBytes=268435456 # 256MiB
+
+#druid_storage_type=local
+#druid_storage_storageDirectory=/opt/shared/segments
+#druid_indexer_logs_type=file
+#druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+#druid_processing_numThreads=1
+#druid_processing_numMergeBuffers=1
+
+#DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=6172m
+DRUID_SINGLE_NODE_CONF=micro-quickstart
 
 druid_emitter_logging_logLevel=debug
 
@@ -39,15 +74,14 @@ druid_metadata_storage_connector_password=FoolishPassword
 druid_coordinator_balancer_strategy=cachingCost
 
 druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
-druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
-druid_processing_buffer_sizeBytes=134217728 # 128MiB
 
 druid_storage_type=local
 druid_storage_storageDirectory=/opt/shared/segments
 druid_indexer_logs_type=file
 druid_indexer_logs_directory=/opt/shared/indexing-logs
 
-druid_processing_numThreads=1
+druid_processing_numThreads=2
-druid_processing_numMergeBuffers=1
+druid_processing_numMergeBuffers=2
 
 DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>