Treat text fields as string and try beta Kibana image

db.py

@@ -60,7 +60,7 @@ def store_message(msg):
             del msg[key]
         if key in schema:
             if isinstance(value, int):
-                if schema[key].startswith("string"):
+                if schema[key].startswith("string") or schema[key].startswith("text"):
                     msg[key] = str(value)
 
     body = [{"insert": {"index": index, "doc": msg}}]
@@ -108,7 +108,9 @@ def store_message_bulk(data):
                     del msg[key]
                 if key in schema:
                     if isinstance(value, int):
-                        if schema[key].startswith("string"):
+                        if schema[key].startswith("string") or schema[key].startswith(
+                            "text"
+                        ):
                             msg[key] = str(value)
 
             body = {"insert": {"index": index, "doc": msg}}

docker-compose.yml

@@ -32,13 +32,21 @@ services:
       - "host.docker.internal:host-gateway"
     volumes_from:
         - tmp
+    depends_on:
+      - tmp
+      - redis
 
   db:
-    image: manticoresearch/manticore
+    image: pathogen/manticore:kibana
+    build: 
+      context: ./docker/manticore
+      args:
+        DEV: 2
     restart: always
     ports:
       - 9308
       - 9312
+      - 9306
     ulimits:
       nproc: 65535
       nofile:
@@ -49,6 +57,7 @@ services:
         hard: -1
     environment:
       - MCL=1
+      - DEV=2
     volumes:
       - ./docker/data:/var/lib/manticore
       - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf

docker/manticore/Dockerfile

@@ -9,9 +9,11 @@ RUN groupadd -r manticore && useradd -r -g manticore manticore
 ENV GOSU_VERSION 1.11
 ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
 ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
+ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"}
+
 
 RUN set -x \
-    && apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr && rm -rf /var/lib/apt/lists/* \
+    && apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \
     && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
     && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
     && export GNUPGHOME="$(mktemp -d)" \
@@ -22,12 +24,20 @@ RUN set -x \
     && chmod +x /usr/local/bin/gosu \
     && gosu nobody true && \
     if [ "${DEV}" = "1" ]; then \
-      wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
+    echo "DEV IS ONE" && \
-      && dpkg -i manticore-dev-repo.noarch.deb \
+     exit && \
-      && apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
+     wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
-      && apt-get update  \
+     && dpkg -i manticore-dev-repo.noarch.deb \
-      && echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
+     && apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
+     && apt-get update  \
+     && echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
+    elif [ "${DEV}" = "2" ]; then \
+      echo "DEV IS TWO" && \
+      wget $BETA_URL && unzip focal.zip && rm focal.zip && \
+      dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\
     else \
+      echo "DEV NOT EITHER" && \
+      exit && \
       wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
       dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
     fi \
@@ -63,4 +73,4 @@ EXPOSE 9308
 EXPOSE 9312
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
-CMD ["searchd", "--nodetach"]
+CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"]

docker/manticore/docker-entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eo pipefail
+echo "RUNNING ENTRYPOINT"
 
 # check to see if this file is being run or sourced from another script
 _is_sourced() {
@@ -25,7 +26,6 @@ docker_setup_env() {
     export searchd_query_log=/var/log/manticore/query.log
     [ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
   fi
-
   if [[ "${MCL}" == "1" ]]; then
       LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
       LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
@@ -61,7 +61,8 @@ _main() {
   if [ "${1#-}" != "$1" ]; then
     set -- searchd "$@"
   fi
-  if [ "$1" = 'searchd' ] && ! _searchd_want_help "@"; then
+  # Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options
+  if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then
     docker_setup_env "$@"
     # allow the container to be started with `--user`
     if [ "$(id -u)" = '0' ]; then

legacy/modules/monitor.py

@@ -68,6 +68,7 @@ def parsemeta(numName, c):
 def queue_message(c):
     message = json.dumps(c)
     print("APPENDING MESSAGE", message)
+    print("KEY", main.config["Ingest"]["Key"])
     main.g.sadd(main.config["Ingest"]["Key"], message)