Implement indexing into Apache Druid #1
6
db.py
6
db.py
|
@ -60,7 +60,7 @@ def store_message(msg):
|
||||||
del msg[key]
|
del msg[key]
|
||||||
if key in schema:
|
if key in schema:
|
||||||
if isinstance(value, int):
|
if isinstance(value, int):
|
||||||
if schema[key].startswith("string"):
|
if schema[key].startswith("string") or schema[key].startswith("text"):
|
||||||
msg[key] = str(value)
|
msg[key] = str(value)
|
||||||
|
|
||||||
body = [{"insert": {"index": index, "doc": msg}}]
|
body = [{"insert": {"index": index, "doc": msg}}]
|
||||||
|
@ -108,7 +108,9 @@ def store_message_bulk(data):
|
||||||
del msg[key]
|
del msg[key]
|
||||||
if key in schema:
|
if key in schema:
|
||||||
if isinstance(value, int):
|
if isinstance(value, int):
|
||||||
if schema[key].startswith("string"):
|
if schema[key].startswith("string") or schema[key].startswith(
|
||||||
|
"text"
|
||||||
|
):
|
||||||
msg[key] = str(value)
|
msg[key] = str(value)
|
||||||
|
|
||||||
body = {"insert": {"index": index, "doc": msg}}
|
body = {"insert": {"index": index, "doc": msg}}
|
||||||
|
|
|
@ -32,13 +32,21 @@ services:
|
||||||
- "host.docker.internal:host-gateway"
|
- "host.docker.internal:host-gateway"
|
||||||
volumes_from:
|
volumes_from:
|
||||||
- tmp
|
- tmp
|
||||||
|
depends_on:
|
||||||
|
- tmp
|
||||||
|
- redis
|
||||||
|
|
||||||
db:
|
db:
|
||||||
image: manticoresearch/manticore
|
image: pathogen/manticore:kibana
|
||||||
|
build:
|
||||||
|
context: ./docker/manticore
|
||||||
|
args:
|
||||||
|
DEV: 2
|
||||||
restart: always
|
restart: always
|
||||||
ports:
|
ports:
|
||||||
- 9308
|
- 9308
|
||||||
- 9312
|
- 9312
|
||||||
|
- 9306
|
||||||
ulimits:
|
ulimits:
|
||||||
nproc: 65535
|
nproc: 65535
|
||||||
nofile:
|
nofile:
|
||||||
|
@ -49,6 +57,7 @@ services:
|
||||||
hard: -1
|
hard: -1
|
||||||
environment:
|
environment:
|
||||||
- MCL=1
|
- MCL=1
|
||||||
|
- DEV=2
|
||||||
volumes:
|
volumes:
|
||||||
- ./docker/data:/var/lib/manticore
|
- ./docker/data:/var/lib/manticore
|
||||||
- ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
- ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
||||||
|
|
|
@ -9,9 +9,11 @@ RUN groupadd -r manticore && useradd -r -g manticore manticore
|
||||||
ENV GOSU_VERSION 1.11
|
ENV GOSU_VERSION 1.11
|
||||||
ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
|
ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
|
||||||
ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
|
ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
|
||||||
|
ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"}
|
||||||
|
|
||||||
|
|
||||||
RUN set -x \
|
RUN set -x \
|
||||||
&& apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr && rm -rf /var/lib/apt/lists/* \
|
&& apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \
|
||||||
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
|
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
|
||||||
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
|
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
|
||||||
&& export GNUPGHOME="$(mktemp -d)" \
|
&& export GNUPGHOME="$(mktemp -d)" \
|
||||||
|
@ -22,12 +24,20 @@ RUN set -x \
|
||||||
&& chmod +x /usr/local/bin/gosu \
|
&& chmod +x /usr/local/bin/gosu \
|
||||||
&& gosu nobody true && \
|
&& gosu nobody true && \
|
||||||
if [ "${DEV}" = "1" ]; then \
|
if [ "${DEV}" = "1" ]; then \
|
||||||
wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
|
echo "DEV IS ONE" && \
|
||||||
&& dpkg -i manticore-dev-repo.noarch.deb \
|
exit && \
|
||||||
&& apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
|
wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
|
||||||
&& apt-get update \
|
&& dpkg -i manticore-dev-repo.noarch.deb \
|
||||||
&& echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
|
&& apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
|
||||||
|
&& apt-get update \
|
||||||
|
&& echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
|
||||||
|
elif [ "${DEV}" = "2" ]; then \
|
||||||
|
echo "DEV IS TWO" && \
|
||||||
|
wget $BETA_URL && unzip focal.zip && rm focal.zip && \
|
||||||
|
dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\
|
||||||
else \
|
else \
|
||||||
|
echo "DEV NOT EITHER" && \
|
||||||
|
exit && \
|
||||||
wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
|
wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
|
||||||
dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
|
dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
|
||||||
fi \
|
fi \
|
||||||
|
@ -63,4 +73,4 @@ EXPOSE 9308
|
||||||
EXPOSE 9312
|
EXPOSE 9312
|
||||||
ENV LANG C.UTF-8
|
ENV LANG C.UTF-8
|
||||||
ENV LC_ALL C.UTF-8
|
ENV LC_ALL C.UTF-8
|
||||||
CMD ["searchd", "--nodetach"]
|
CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"]
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -eo pipefail
|
set -eo pipefail
|
||||||
|
echo "RUNNING ENTRYPOINT"
|
||||||
|
|
||||||
# check to see if this file is being run or sourced from another script
|
# check to see if this file is being run or sourced from another script
|
||||||
_is_sourced() {
|
_is_sourced() {
|
||||||
|
@ -25,7 +26,6 @@ docker_setup_env() {
|
||||||
export searchd_query_log=/var/log/manticore/query.log
|
export searchd_query_log=/var/log/manticore/query.log
|
||||||
[ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
|
[ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "${MCL}" == "1" ]]; then
|
if [[ "${MCL}" == "1" ]]; then
|
||||||
LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
|
LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
|
||||||
LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
|
LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
|
||||||
|
@ -61,7 +61,8 @@ _main() {
|
||||||
if [ "${1#-}" != "$1" ]; then
|
if [ "${1#-}" != "$1" ]; then
|
||||||
set -- searchd "$@"
|
set -- searchd "$@"
|
||||||
fi
|
fi
|
||||||
if [ "$1" = 'searchd' ] && ! _searchd_want_help "@"; then
|
# Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options
|
||||||
|
if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then
|
||||||
docker_setup_env "$@"
|
docker_setup_env "$@"
|
||||||
# allow the container to be started with `--user`
|
# allow the container to be started with `--user`
|
||||||
if [ "$(id -u)" = '0' ]; then
|
if [ "$(id -u)" = '0' ]; then
|
||||||
|
|
|
@ -68,6 +68,7 @@ def parsemeta(numName, c):
|
||||||
def queue_message(c):
|
def queue_message(c):
|
||||||
message = json.dumps(c)
|
message = json.dumps(c)
|
||||||
print("APPENDING MESSAGE", message)
|
print("APPENDING MESSAGE", message)
|
||||||
|
print("KEY", main.config["Ingest"]["Key"])
|
||||||
main.g.sadd(main.config["Ingest"]["Key"], message)
|
main.g.sadd(main.config["Ingest"]["Key"], message)
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue