Implement indexing into Apache Druid #1
12
db.py
12
db.py
|
|
@ -72,7 +72,8 @@ def store_message(msg):
|
|||
# print(body_post)
|
||||
try:
|
||||
# Bulk index operations
|
||||
api_response = api_instance.bulk(body_post) # , async_req=True
|
||||
print("FAKE POST")
|
||||
#api_response = api_instance.bulk(body_post) # , async_req=True
|
||||
# print(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling IndexApi->bulk: %s\n" % e)
|
||||
|
|
@ -124,8 +125,9 @@ def store_message_bulk(data):
|
|||
# print(body_post)
|
||||
try:
|
||||
# Bulk index operations
|
||||
api_response = api_instance.bulk(body_post) # , async_req=True
|
||||
print(api_response)
|
||||
print("FAKE POST")
|
||||
#api_response = api_instance.bulk(body_post) # , async_req=True
|
||||
#print(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling IndexApi->bulk: %s\n" % e)
|
||||
print("ATTEMPT", body_post)
|
||||
|
|
@ -152,5 +154,5 @@ def create_index(api_client):
|
|||
util_instance.sql(create_query)
|
||||
|
||||
|
||||
create_index(api_client)
|
||||
update_schema()
|
||||
#create_index(api_client)
|
||||
#update_schema()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,13 @@
|
|||
version: "2"
|
||||
version: "2.2"
|
||||
|
||||
volumes:
|
||||
metadata_data: {}
|
||||
middle_var: {}
|
||||
historical_var: {}
|
||||
broker_var: {}
|
||||
coordinator_var: {}
|
||||
router_var: {}
|
||||
druid_shared: {}
|
||||
|
||||
services:
|
||||
app:
|
||||
|
|
@ -10,8 +19,8 @@ services:
|
|||
- .env
|
||||
volumes_from:
|
||||
- tmp
|
||||
depends_on:
|
||||
- db
|
||||
# depends_on:
|
||||
# - db
|
||||
|
||||
threshold:
|
||||
image: pathogen/threshold:latest
|
||||
|
|
@ -36,31 +45,133 @@ services:
|
|||
- tmp
|
||||
- redis
|
||||
|
||||
db:
|
||||
#image: pathogen/manticore:kibana
|
||||
image: manticoresearch/manticore:dev
|
||||
#build:
|
||||
# context: ./docker/manticore
|
||||
# args:
|
||||
# DEV: 1
|
||||
restart: always
|
||||
ports:
|
||||
- 9308
|
||||
- 9312
|
||||
- 9306
|
||||
ulimits:
|
||||
nproc: 65535
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
environment:
|
||||
- MCL=1
|
||||
postgres:
|
||||
container_name: postgres
|
||||
image: postgres:latest
|
||||
volumes:
|
||||
- ./docker/data:/var/lib/manticore
|
||||
- ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
||||
- metadata_data:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_PASSWORD=FoolishPassword
|
||||
- POSTGRES_USER=druid
|
||||
- POSTGRES_DB=druid
|
||||
|
||||
# Need 3.5 or later for container nodes
|
||||
zookeeper:
|
||||
container_name: zookeeper
|
||||
image: zookeeper:3.5
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
- ZOO_MY_ID=1
|
||||
|
||||
coordinator:
|
||||
image: apache/druid:0.23.0
|
||||
container_name: coordinator
|
||||
volumes:
|
||||
- druid_shared:/opt/shared
|
||||
- coordinator_var:/opt/druid/var
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- postgres
|
||||
ports:
|
||||
- "8081:8081"
|
||||
command:
|
||||
- coordinator
|
||||
env_file:
|
||||
- environment
|
||||
|
||||
broker:
|
||||
image: apache/druid:0.23.0
|
||||
container_name: broker
|
||||
volumes:
|
||||
- broker_var:/opt/druid/var
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- postgres
|
||||
- coordinator
|
||||
ports:
|
||||
- "8082:8082"
|
||||
command:
|
||||
- broker
|
||||
env_file:
|
||||
- environment
|
||||
|
||||
historical:
|
||||
image: apache/druid:0.23.0
|
||||
container_name: historical
|
||||
volumes:
|
||||
- druid_shared:/opt/shared
|
||||
- historical_var:/opt/druid/var
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- postgres
|
||||
- coordinator
|
||||
ports:
|
||||
- "8083:8083"
|
||||
command:
|
||||
- historical
|
||||
env_file:
|
||||
- environment
|
||||
|
||||
middlemanager:
|
||||
image: apache/druid:0.23.0
|
||||
container_name: middlemanager
|
||||
volumes:
|
||||
- druid_shared:/opt/shared
|
||||
- middle_var:/opt/druid/var
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- postgres
|
||||
- coordinator
|
||||
ports:
|
||||
- "8091:8091"
|
||||
- "8100-8105:8100-8105"
|
||||
command:
|
||||
- middleManager
|
||||
env_file:
|
||||
- environment
|
||||
|
||||
router:
|
||||
image: apache/druid:0.23.0
|
||||
container_name: router
|
||||
volumes:
|
||||
- router_var:/opt/druid/var
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- postgres
|
||||
- coordinator
|
||||
ports:
|
||||
- "8888:8888"
|
||||
command:
|
||||
- router
|
||||
env_file:
|
||||
- environment
|
||||
|
||||
# db:
|
||||
# #image: pathogen/manticore:kibana
|
||||
# image: manticoresearch/manticore:dev
|
||||
# #build:
|
||||
# # context: ./docker/manticore
|
||||
# # args:
|
||||
# # DEV: 1
|
||||
# restart: always
|
||||
# ports:
|
||||
# - 9308
|
||||
# - 9312
|
||||
# - 9306
|
||||
# ulimits:
|
||||
# nproc: 65535
|
||||
# nofile:
|
||||
# soft: 65535
|
||||
# hard: 65535
|
||||
# memlock:
|
||||
# soft: -1
|
||||
# hard: -1
|
||||
# environment:
|
||||
# - MCL=1
|
||||
# volumes:
|
||||
# - ./docker/data:/var/lib/manticore
|
||||
# - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
||||
|
||||
tmp:
|
||||
image: busybox
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
# Java tuning
|
||||
DRUID_XMX=1g
|
||||
DRUID_XMS=1g
|
||||
DRUID_MAXNEWSIZE=250m
|
||||
DRUID_NEWSIZE=250m
|
||||
DRUID_MAXDIRECTMEMORYSIZE=6172m
|
||||
|
||||
druid_emitter_logging_logLevel=debug
|
||||
|
||||
druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage"]
|
||||
|
||||
druid_zk_service_host=zookeeper
|
||||
|
||||
druid_metadata_storage_host=
|
||||
druid_metadata_storage_type=postgresql
|
||||
druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
|
||||
druid_metadata_storage_connector_user=druid
|
||||
druid_metadata_storage_connector_password=FoolishPassword
|
||||
|
||||
druid_coordinator_balancer_strategy=cachingCost
|
||||
|
||||
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
|
||||
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
|
||||
|
||||
druid_storage_type=local
|
||||
druid_storage_storageDirectory=/opt/shared/segments
|
||||
druid_indexer_logs_type=file
|
||||
druid_indexer_logs_directory=/opt/shared/indexing-logs
|
||||
|
||||
druid_processing_numThreads=2
|
||||
druid_processing_numMergeBuffers=2
|
||||
|
||||
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
|
||||
Loading…
Reference in New Issue