From 79a430be04345a04bd818964b3190bf1f0516539 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 8 Sep 2022 07:20:30 +0100 Subject: [PATCH] Begin implementing Apache Druid --- db.py | 12 ++-- docker-compose.yml | 167 +++++++++++++++++++++++++++++++++++++-------- environment | 52 ++++++++++++++ 3 files changed, 198 insertions(+), 33 deletions(-) create mode 100644 environment diff --git a/db.py b/db.py index 58cb9a5..710e7d3 100644 --- a/db.py +++ b/db.py @@ -72,7 +72,8 @@ def store_message(msg): # print(body_post) try: # Bulk index operations - api_response = api_instance.bulk(body_post) # , async_req=True + print("FAKE POST") + #api_response = api_instance.bulk(body_post) # , async_req=True # print(api_response) except ApiException as e: print("Exception when calling IndexApi->bulk: %s\n" % e) @@ -124,8 +125,9 @@ def store_message_bulk(data): # print(body_post) try: # Bulk index operations - api_response = api_instance.bulk(body_post) # , async_req=True - print(api_response) + print("FAKE POST") + #api_response = api_instance.bulk(body_post) # , async_req=True + #print(api_response) except ApiException as e: print("Exception when calling IndexApi->bulk: %s\n" % e) print("ATTEMPT", body_post) @@ -152,5 +154,5 @@ def create_index(api_client): util_instance.sql(create_query) -create_index(api_client) -update_schema() +#create_index(api_client) +#update_schema() diff --git a/docker-compose.yml b/docker-compose.yml index d3b25f7..e73c970 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,14 @@ -version: "2" - +version: "2.2" + +volumes: + metadata_data: {} + middle_var: {} + historical_var: {} + broker_var: {} + coordinator_var: {} + router_var: {} + druid_shared: {} + services: app: image: pathogen/monolith:latest @@ -10,8 +19,8 @@ services: - .env volumes_from: - tmp - depends_on: - - db + # depends_on: + # - db threshold: image: pathogen/threshold:latest @@ -36,31 +45,133 @@ services: - tmp - redis - db: - #image: pathogen/manticore:kibana - image: manticoresearch/manticore:dev - #build: - # context: ./docker/manticore - # args: - # DEV: 1 - restart: always - ports: - - 9308 - - 9312 - - 9306 - ulimits: - nproc: 65535 - nofile: - soft: 65535 - hard: 65535 - memlock: - soft: -1 - hard: -1 - environment: - - MCL=1 + postgres: + container_name: postgres + image: postgres:latest volumes: - - ./docker/data:/var/lib/manticore - - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf + - metadata_data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=FoolishPassword + - POSTGRES_USER=druid + - POSTGRES_DB=druid + + # Need 3.5 or later for container nodes + zookeeper: + container_name: zookeeper + image: zookeeper:3.5 + ports: + - "2181:2181" + environment: + - ZOO_MY_ID=1 + + coordinator: + image: apache/druid:0.23.0 + container_name: coordinator + volumes: + - druid_shared:/opt/shared + - coordinator_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + ports: + - "8081:8081" + command: + - coordinator + env_file: + - environment + + broker: + image: apache/druid:0.23.0 + container_name: broker + volumes: + - broker_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8082:8082" + command: + - broker + env_file: + - environment + + historical: + image: apache/druid:0.23.0 + container_name: historical + volumes: + - druid_shared:/opt/shared + - historical_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8083:8083" + command: + - historical + env_file: + - environment + + middlemanager: + image: apache/druid:0.23.0 + container_name: middlemanager + volumes: + - druid_shared:/opt/shared + - middle_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8091:8091" + - "8100-8105:8100-8105" + command: + - middleManager + env_file: + - environment + + router: + image: apache/druid:0.23.0 + container_name: router + volumes: + - router_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8888:8888" + command: + - router + env_file: + - environment + + # db: + # #image: pathogen/manticore:kibana + # image: manticoresearch/manticore:dev + # #build: + # # context: ./docker/manticore + # # args: + # # DEV: 1 + # restart: always + # ports: + # - 9308 + # - 9312 + # - 9306 + # ulimits: + # nproc: 65535 + # nofile: + # soft: 65535 + # hard: 65535 + # memlock: + # soft: -1 + # hard: -1 + # environment: + # - MCL=1 + # volumes: + # - ./docker/data:/var/lib/manticore + # - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf tmp: image: busybox diff --git a/environment b/environment new file mode 100644 index 0000000..7bb9214 --- /dev/null +++ b/environment @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=1g +DRUID_XMS=1g +DRUID_MAXNEWSIZE=250m +DRUID_NEWSIZE=250m +DRUID_MAXDIRECTMEMORYSIZE=6172m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB + +druid_storage_type=local +druid_storage_storageDirectory=/opt/shared/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/shared/indexing-logs + +druid_processing_numThreads=2 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J=