diff --git a/db.py b/db.py index 7951ef4..58cb9a5 100644 --- a/db.py +++ b/db.py @@ -60,7 +60,7 @@ def store_message(msg): del msg[key] if key in schema: if isinstance(value, int): - if schema[key].startswith("string"): + if schema[key].startswith("string") or schema[key].startswith("text"): msg[key] = str(value) body = [{"insert": {"index": index, "doc": msg}}] @@ -108,7 +108,9 @@ def store_message_bulk(data): del msg[key] if key in schema: if isinstance(value, int): - if schema[key].startswith("string"): + if schema[key].startswith("string") or schema[key].startswith( + "text" + ): msg[key] = str(value) body = {"insert": {"index": index, "doc": msg}} diff --git a/docker-compose.yml b/docker-compose.yml index ebd9d34..8563f3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,13 +32,21 @@ services: - "host.docker.internal:host-gateway" volumes_from: - tmp + depends_on: + - tmp + - redis db: - image: manticoresearch/manticore + image: pathogen/manticore:kibana + build: + context: ./docker/manticore + args: + DEV: 2 restart: always ports: - 9308 - 9312 + - 9306 ulimits: nproc: 65535 nofile: @@ -49,6 +57,7 @@ services: hard: -1 environment: - MCL=1 + - DEV=2 volumes: - ./docker/data:/var/lib/manticore - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf diff --git a/docker/manticore/Dockerfile b/docker/manticore/Dockerfile index ef60f07..73bb16b 100644 --- a/docker/manticore/Dockerfile +++ b/docker/manticore/Dockerfile @@ -9,9 +9,11 @@ RUN groupadd -r manticore && useradd -r -g manticore manticore ENV GOSU_VERSION 1.11 ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"} ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"} +ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"} + RUN set -x \ - && apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr && rm -rf /var/lib/apt/lists/* \ + && apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ && export GNUPGHOME="$(mktemp -d)" \ @@ -22,12 +24,20 @@ RUN set -x \ && chmod +x /usr/local/bin/gosu \ && gosu nobody true && \ if [ "${DEV}" = "1" ]; then \ - wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \ - && dpkg -i manticore-dev-repo.noarch.deb \ - && apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \ - && apt-get update \ - && echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\ + echo "DEV IS ONE" && \ + exit && \ + wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \ + && dpkg -i manticore-dev-repo.noarch.deb \ + && apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \ + && apt-get update \ + && echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\ + elif [ "${DEV}" = "2" ]; then \ + echo "DEV IS TWO" && \ + wget $BETA_URL && unzip focal.zip && rm focal.zip && \ + dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\ else \ + echo "DEV NOT EITHER" && \ + exit && \ wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \ dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \ fi \ @@ -63,4 +73,4 @@ EXPOSE 9308 EXPOSE 9312 ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 -CMD ["searchd", "--nodetach"] +CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"] diff --git a/docker/manticore/docker-entrypoint.sh b/docker/manticore/docker-entrypoint.sh index 07e4e1a..5dbf0bc 100755 --- a/docker/manticore/docker-entrypoint.sh +++ b/docker/manticore/docker-entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +echo "RUNNING ENTRYPOINT" # check to see if this file is being run or sourced from another script _is_sourced() { @@ -25,7 +26,6 @@ docker_setup_env() { export searchd_query_log=/var/log/manticore/query.log [ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log fi - if [[ "${MCL}" == "1" ]]; then LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so" LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so" @@ -61,7 +61,8 @@ _main() { if [ "${1#-}" != "$1" ]; then set -- searchd "$@" fi - if [ "$1" = 'searchd' ] && ! _searchd_want_help "@"; then + # Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options + if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then docker_setup_env "$@" # allow the container to be started with `--user` if [ "$(id -u)" = '0' ]; then diff --git a/legacy/modules/monitor.py b/legacy/modules/monitor.py index dc2ac97..4c2e457 100644 --- a/legacy/modules/monitor.py +++ b/legacy/modules/monitor.py @@ -68,6 +68,7 @@ def parsemeta(numName, c): def queue_message(c): message = json.dumps(c) print("APPENDING MESSAGE", message) + print("KEY", main.config["Ingest"]["Key"]) main.g.sadd(main.config["Ingest"]["Key"], message)