Remove Manticore and Superset stuff

This commit is contained in:
Mark Veidemanis 2022-11-22 18:37:49 +00:00
parent 44d6d90325
commit 7edc231ea9
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
14 changed files with 2 additions and 1037 deletions

View File

@ -1,46 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
COMPOSE_PROJECT_NAME=superset
# database configurations (do not modify)
DATABASE_DB=superset
DATABASE_HOST=db
DATABASE_PASSWORD=superset
DATABASE_USER=superset
# database engine specific environment variables
# change the below if you prefers another database engine
DATABASE_PORT=5432
DATABASE_DIALECT=postgresql
POSTGRES_DB=superset
POSTGRES_USER=superset
POSTGRES_PASSWORD=superset
#MYSQL_DATABASE=superset
#MYSQL_USER=superset
#MYSQL_PASSWORD=superset
#MYSQL_RANDOM_ROOT_PASSWORD=yes
# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
REDIS_HOST=redis
REDIS_PORT=6379
FLASK_ENV=production
SUPERSET_ENV=production
SUPERSET_LOAD_EXAMPLES=no
CYPRESS_CONFIG=false
SUPERSET_PORT=8088

View File

@ -1,22 +0,0 @@
ARG DRUID_VER=0.23.0
FROM apache/druid:${DRUID_VER} AS druid
FROM ubuntu:bionic
RUN apt-get update && \
apt-get install --yes openjdk-8-jre-headless perl-modules && \
apt-get clean
RUN addgroup --system -gid 1000 druid \
&& adduser --system --uid 1000 --disabled-password --home /opt/druid --shell /bin/bash --group druid
COPY --from=druid --chown=druid:druid /opt/druid /opt/druid
WORKDIR /opt/druid
USER druid
EXPOSE 8888/tcp
EXPOSE 8081/tcp
CMD /opt/druid/bin/start-nano-quickstart

View File

@ -1,12 +0,0 @@
_HiStOrY_V2_
SELECT * FROM films WHERE MATCH('"shark monkey boy robot"/2') AND release_year IN(2006,2007) AND rental_rate BETWEEN 2.0 and 3.0;
SELECT title, HIGHLIGHT({},'description') FROM films WHERE MATCH('"shark monkey boy robot"/2');
SELECT * FROM films WHERE MATCH('" shark monkey boy robot "/2');
SELECT * FROM films WHERE MATCH('Emotional drama') FACET release_year FACET category_id;
SELECT * FROM films WHERE MATCH('Emotional drama') GROUP BY release_year;
SELECT * FROM films WHERE MATCH('Emotional drama -dog -shark');
SELECT * FROM films WHERE MATCH('Emotional drama');
SELECT * FROM films;
DESCRIBE films;
SHOW TABLES;
SOURCE /sandbox.sql

View File

@ -1,76 +0,0 @@
FROM ubuntu:focal
ARG DEV
ARG DAEMON_URL
ARG MCL_URL
RUN groupadd -r manticore && useradd -r -g manticore manticore
ENV GOSU_VERSION 1.11
ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"}
RUN set -x \
&& apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
&& export GNUPGHOME="$(mktemp -d)" \
&& gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
&& rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc \
&& chmod +x /usr/local/bin/gosu \
&& gosu nobody true && \
if [ "${DEV}" = "1" ]; then \
echo "DEV IS ONE" && \
exit && \
wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
&& dpkg -i manticore-dev-repo.noarch.deb \
&& apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
&& apt-get update \
&& echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
elif [ "${DEV}" = "2" ]; then \
echo "DEV IS TWO" && \
wget $BETA_URL && unzip focal.zip && rm focal.zip && \
dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\
else \
echo "DEV NOT EITHER" && \
exit && \
wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
fi \
&& mkdir -p /var/run/manticore && mkdir -p /var/lib/manticore/replication \
&& apt-get update && apt-get -y install libexpat1 libodbc1 libpq5 openssl libcrypto++6 libmysqlclient21 mysql-client \
&& apt-get -y purge --auto-remove \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /usr/bin/mariabackup /usr/bin/mysqldump /usr/bin/mysqlslap /usr/bin/mysqladmin /usr/bin/mysqlimport \
/usr/bin/mysqlshow /usr/bin/mbstream /usr/bin/mysql_waitpid /usr/bin/innotop /usr/bin/mysqlaccess /usr/bin/mytop \
/usr/bin/mysqlreport /usr/bin/mysqldumpslow /usr/bin/mysql_find_rows /usr/bin/mysql_fix_extensions \
/usr/bin/mysql_embedded /usr/bin/mysqlcheck \
&& rm -f /usr/bin/spelldump /usr/bin/wordbreaker \
&& mkdir -p /var/run/mysqld/ && chown manticore:manticore /var/run/mysqld/ \
&& echo "\n[mysql]\nsilent\nwait\ntable\n" >> /etc/mysql/my.cnf && \
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/en.pak.tgz && \
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/de.pak.tgz && \
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/ru.pak.tgz && \
tar -xf /tmp/en.pak.tgz -C /usr/share/manticore/ && \
tar -xf /tmp/de.pak.tgz -C /usr/share/manticore/ && \
tar -xf /tmp/ru.pak.tgz -C /usr/share/manticore/
COPY manticore.conf /etc/manticoresearch/
COPY sandbox.sql /sandbox.sql
COPY .mysql_history /root/.mysql_history
COPY docker-entrypoint.sh /usr/local/bin/
RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh
WORKDIR /var/lib/manticore
ENTRYPOINT ["docker-entrypoint.sh"]
EXPOSE 9306
EXPOSE 9308
EXPOSE 9312
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"]

View File

@ -1,278 +0,0 @@
# Manticore Search Docker image
This is the git repo of official [Docker image](https://hub.docker.com/r/manticoresearch/manticore/) for [Manticore Search](https://github.com/manticoresoftware/manticoresearch).
Manticore Search is an easy to use open source fast database for search. It helps thousands of companies from small to large, such as Craigslist, to search and filter petabytes of text data on a single or hundreds of nodes, do stream full-text filtering, add auto-complete, spell correction, more-like-this, faceting and other search-related technologies to their websites and applications.
The default configuration includes a sample Real-Time index and listens on the default ports:
* `9306` for connections from a MySQL client
* `9308` for connections via HTTP
* `9312` for connections via a binary protocol (e.g. in case you run a cluster)
The image comes with libraries for easy indexing data from MySQL, PostgreSQL XML and CSV files.
# How to run Manticore Search Docker image
## Quick usage
The below is the simplest way to start Manticore in a container and log in to it via mysql client:
```bash
docker run --name manticore --rm -d manticoresearch/manticore && sleep 3 && docker exec -it manticore mysql && docker stop manticore
```
When you exit from the mysql client it stops and removes the container, so **use it only for testing / sandboxing purposes**. See below how to use it in production.
The image comes with a sample index which can be loaded like this:
```mysql
mysql> source /sandbox.sql
```
Also the mysql client has in history several sample queries that you can run on the above index, just use Up/Down keys in the client to see and run them.
## Production use
### Ports and mounting points
For data persistence `/var/lib/manticore/` should be mounted to local storage or other desired storage engine.
```bash
docker run --name manticore -v $(pwd)/data:/var/lib/manticore -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
```
Configuration file inside the instance is located at `/etc/manticoresearch/manticore.conf`. For custom settings, this file should be mounted to your own configuration file.
The ports are 9306/9308/9312 for SQL/HTTP/Binary, expose them depending on how you are going to use Manticore. For example:
```bash
docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data:/var/lib/manticore/ -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
```
Make sure to remove `127.0.0.1:` if you want the ports to be available for external hosts.
### Manticore Columnar Library
The docker image doesn't include [Manticore Columnar Library](https://github.com/manticoresoftware/columnar) which has to be used if you need:
* columnar storage
* secondary indexes
but you can easily enable it in runtime by using environment variable `MCL=1`, i.e. `docker run -e MCL=1 ... manticoresearch/manticore`. It will then download and install the library and put it to the data dir (which is normally mapped as a volume in production). Next time you run the container the library will be already there, hence it won't be downloaded again unless you change the Manticore Search version.
### Docker-compose
In many cases you might want to use Manticore together with other images specified in a docker-compose YAML file. Here is the minimal recommended specification for Manticore Search in docker-compose.yml:
```yaml
version: '2.2'
services:
manticore:
container_name: manticore
image: manticoresearch/manticore
restart: always
ports:
- 127.0.0.1:9306:9306
- 127.0.0.1:9308:9308
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
volumes:
- ./data:/var/lib/manticore
# - ./manticore.conf:/etc/manticoresearch/manticore.conf # uncommment if you use a custom config
```
Besides using the exposed ports 9306 and 9308 you can log into the instance by running `docker-compose exec manticore mysql`.
### HTTP protocol
Manticore is accessible via HTTP on ports 9308 and 9312. You can map either of them locally and connect with curl:
```bash
docker run --name manticore -p 9308:9308 -d manticoresearch/manticore
```
Create a table:
```bash
curl -X POST 'http://127.0.0.1:9308/sql' -d 'mode=raw&query=CREATE TABLE testrt ( title text, content text, gid integer)'
```
Insert a document:
```bash
curl -X POST 'http://127.0.0.1:9308/json/insert' -d'{"index":"testrt","id":1,"doc":{"title":"Hello","content":"world","gid":1}}'
```
Perform a simple search:
```bash
curl -X POST 'http://127.0.0.1:9308/json/search' -d '{"index":"testrt","query":{"match":{"*":"hello world"}}}'
```
### Logging
By default, Manticore logs to `/dev/stdout`, so you can watch the log on the host with:
```bash
docker logs manticore
```
If you want to get log of your queries the same way you can do it by passing environment variable `QUERY_LOG_TO_STDOUT=true`.
### Multi-node cluster with replication
Here is a simple `docker-compose.yml` for defining a two node cluster:
```yaml
version: '2.2'
services:
manticore-1:
image: manticoresearch/manticore
restart: always
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
networks:
- manticore
manticore-2:
image: manticoresearch/manticore
restart: always
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
networks:
- manticore
networks:
manticore:
driver: bridge
```
* Start it: `docker-compose up`
* Create a cluster with a table:
```mysql
$ docker-compose exec manticore-1 mysql
mysql> CREATE TABLE testrt ( title text, content text, gid integer);
mysql> CREATE CLUSTER posts;
Query OK, 0 rows affected (0.24 sec)
mysql> ALTER CLUSTER posts ADD testrt;
Query OK, 0 rows affected (0.07 sec)
MySQL [(none)]> exit
Bye
```
* Join to the the cluster on the 2nd instance and insert smth to the table:
```mysql
$ docker-compose exec manticore-2 mysql
mysql> JOIN CLUSTER posts AT 'manticore-1:9312';
mysql> INSERT INTO posts:testrt(title,content,gid) VALUES('hello','world',1);
Query OK, 1 row affected (0.00 sec)
MySQL [(none)]> exit
Bye
```
* If you now go back to the first instance you'll see the new record:
```mysql
$ docker-compose exec manticore-1 mysql
MySQL [(none)]> select * from testrt;
+---------------------+------+-------+---------+
| id | gid | title | content |
+---------------------+------+-------+---------+
| 3891565839006040065 | 1 | hello | world |
+---------------------+------+-------+---------+
1 row in set (0.00 sec)
MySQL [(none)]> exit
Bye
```
## Memory locking and limits
It's recommended to overwrite the default ulimits of docker for the Manticore instance:
```bash
--ulimit nofile=65536:65536
```
For the best performance, Manticore tables' components can be locked into memory. When Manticore is run under Docker, the instance requires additional privileges to allow memory locking. The following options must be added when running the instance:
```bash
--cap-add=IPC_LOCK --ulimit memlock=-1:-1
```
## Configuring Manticore Search with Docker
If you want to run Manticore with your custom config containing indexes definition you will need to mount the configuration to the instance:
```bash
docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data/:/var/lib/manticore -p 127.0.0.1:9306:9306 -d manticoresearch/manticore
```
Take into account that Manticore search inside the container is run under user `manticore`. Performing operations with tables (like creating or rotating plain indexes) should be also done under `manticore`. Otherwise the files will be created under `root` and the search daemon won't have rights to open them. For example here is how you can rotate all plain indexes:
```bash
docker exec -it manticore gosu manticore indexer --all --rotate
```
### Environment variables
You can also set individual `searchd` and `common` configuration settings using Docker environment variables.
The settings must be prefixed with their section name, for example to change value of setting `mysql_version_string` in section `searchd` the variable must be named `searchd_mysql_version_string`:
```bash
docker run --name manticore -p 127.0.0.1:9306:9306 -e searchd_mysql_version_string='5.5.0' -d manticoresearch/manticore
```
In case of `listen` directive, you can pass using Docker variable `searchd_listen` new listening interfaces in addition to the default ones. Multiple interfaces can be declared separated by semi-colon ("|").
For listening only on network address, the `$ip` (retrieved internally from `hostname -i`) can be used as address alias.
For example `-e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip'` will add an additional SQL interface on port 9307, a SQL VIP on 5443 running only on the instance IP and HTTP on port 9316, beside the defaults on 9306 and 9308, respectively.
```bash
$ docker run --rm -p 1188:9307 -e searchd_mysql_version_string='5.5.0' -e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip' manticore
[Mon Aug 17 07:31:58.719 2020] [1] using config file '/etc/manticoresearch/manticore.conf' (9130 chars)...
listening on all interfaces for http, port=9316
listening on all interfaces for mysql, port=9307
listening on 172.17.0.17:5443 for VIP mysql
listening on all interfaces for mysql, port=9306
listening on UNIX socket /var/run/mysqld/mysqld.sock
listening on 172.17.0.17:9312 for sphinx
listening on all interfaces for http, port=9308
prereading 0 indexes
prereaded 0 indexes in 0.000 sec
accepting connections
```
# Issues
For reporting issues, please use the [issue tracker](https://github.com/manticoresoftware/docker/issues).

View File

@ -1,118 +0,0 @@
#!/bin/bash
set -eo pipefail
echo "RUNNING ENTRYPOINT"
# check to see if this file is being run or sourced from another script
_is_sourced() {
# https://unix.stackexchange.com/a/215279
[ "${#FUNCNAME[@]}" -ge 2 ] &&
[ "${FUNCNAME[0]}" = '_is_sourced' ] &&
[ "${FUNCNAME[1]}" = 'source' ]
}
_searchd_want_help() {
local arg
for arg; do
case "$arg" in
-'?' | --help | -h | -v)
return 0
;;
esac
done
return 1
}
docker_setup_env() {
if [ -n "$QUERY_LOG_TO_STDOUT" ]; then
export searchd_query_log=/var/log/manticore/query.log
[ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
fi
if [[ "${MCL}" == "1" ]]; then
LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
[ -L /usr/share/manticore/modules/lib_manticore_columnar.so ] || ln -s $LIB_MANTICORE_COLUMNAR /usr/share/manticore/modules/lib_manticore_columnar.so
[ -L /usr/share/manticore/modules/lib_manticore_secondary.so ] || ln -s $LIB_MANTICORE_SECONDARY /usr/share/manticore/modules/lib_manticore_secondary.so
searchd -v|grep -i error|egrep "trying to load" \
&& rm $LIB_MANTICORE_COLUMNAR $LIB_MANTICORE_SECONDARY \
&& echo "WARNING: wrong MCL version was removed, installing the correct one"
if [[ ! -f "$LIB_MANTICORE_COLUMNAR" || ! -f "$LIB_MANTICORE_SECONDARY" ]]; then
if ! mkdir -p /var/lib/manticore/.mcl/ ; then
echo "ERROR: Manticore Columnar Library is inaccessible: couldn't create /var/lib/manticore/.mcl/."
exit
fi
MCL_URL=$(cat /mcl.url)
wget -P /tmp $MCL_URL
LAST_PATH=$(pwd)
cd /tmp
PACKAGE_NAME=$(ls | grep manticore-columnar | head -n 1)
ar -x $PACKAGE_NAME
tar -xf data.tar.gz
find . -name '*.so' -exec cp {} /var/lib/manticore/.mcl/ \;
cd $LAST_PATH
fi
fi
}
_main() {
# first arg is `h` or some `--option`
if [ "${1#-}" != "$1" ]; then
set -- searchd "$@"
fi
# Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options
if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then
docker_setup_env "$@"
# allow the container to be started with `--user`
if [ "$(id -u)" = '0' ]; then
find /var/lib/manticore /var/log/manticore /var/run/manticore /etc/manticoresearch \! -user manticore -exec chown manticore '{}' +
exec gosu manticore "$0" "$@"
fi
fi
_replace_conf_from_env
exec "$@"
}
_replace_conf_from_env() {
sed_query=""
while IFS='=' read -r oldname value; do
if [[ $oldname == 'searchd_'* || $oldname == 'common_'* ]]; then
value=$(echo ${!oldname} | sed 's/\//\\\//g')
oldname=$(echo $oldname | sed "s/searchd_//g;s/common_//g;")
newname=$oldname
if [[ $newname == 'listen' ]]; then
oldname="listen_env"
IFS='|' read -ra ADDR <<<"$value"
count=0
for i in "${ADDR[@]}"; do
if [[ $count == 0 ]]; then
value=$i
else
value="$value\n listen = $i"
fi
count=$((count + 1))
done
fi
if [[ -z $sed_query ]]; then
sed_query="s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
else
sed_query="$sed_query;s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
fi
fi
done < <(env)
if [[ ! -z $sed_query ]]; then
sed -i -E "$sed_query" /etc/manticoresearch/manticore.conf
fi
}
# If we are sourced from elsewhere, don't perform any further actions
if ! _is_sourced; then
_main "$@"
fi

View File

@ -1,259 +0,0 @@
#!/bin/sh
ip=`hostname -i|rev|cut -d\ -f 1|rev`
cat << EOF
searchd {
# https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
# access_plain_attrs = mmap_preread
# https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
# access_blob_attrs = mmap_preread
# https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
# access_doclists = file
# https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
# access_hitlists = file
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
# agent_connect_timeout =
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
# agent_query_timeout =
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
# agent_retry_count = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
# agent_retry_delay = 500
# https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
# attr_flush_period = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
# binlog_flush = 2
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
# binlog_max_log_size = 268435456
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
# binlog_path =
# https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
# client_timeout = 300
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
# collation_libc_locale = C
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
# collation_server = libc_ci
# https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
data_dir = /var/lib/manticore
# https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
# docstore_cache_size = 16m
# https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
# expansion_limit = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
# grouping_in_utc = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
# ha_period_karma = 60
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
# ha_ping_interval = 1000
# https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
# hostname_lookup =
# https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
# jobs_queue_size =
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
# listen_backlog = 5
# https://manual.manticoresearch.com/Server_settings/Searchd#listen
# listen_env = this directive allows to append listeners from environment variables
listen = 9306:mysql41
listen = /var/run/mysqld/mysqld.sock:mysql41
listen = $ip:9312
listen = 9308:http
listen = $ip:9315-9325:replication
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
# listen_tfo = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#log
log = /var/log/manticore/searchd.log
# https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
# max_batch_queries = 32
# https://manual.manticoresearch.com/Server_settings/Searchd#threads
# threads =
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
# max_filters = 256
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
# max_filter_values = 4096
# https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
# max_open_files =
# https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
max_packet_size = 128M
# https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
# mysql_version_string =
# https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
# net_workers = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
# net_wait_tm = -1
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
# net_throttle_accept = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
# net_throttle_action = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#node_address
# node_address =
# https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
# ondisk_attrs_default = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
# persistent_connections_limit =
# https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
pid_file = /var/run/manticore/searchd.pid
# https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
# predicted_time_costs = doc=64, hit=48, skip=2048, match=64
# https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
# preopen_indexes = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
# qcache_max_bytes = 16Mb
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
# qcache_thresh_msec = 3000
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
# qcache_ttl_sec = 60
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
query_log_format = sphinxql
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
query_log_min_msec = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log
query_log = /var/log/manticore/query.log
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
# query_log_mode = 600
# https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
# max_connections =
# https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
# network_timeout = 5
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
# read_buffer = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
# read_buffer_docs = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
# read_buffer_hits = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
# read_unhinted 32K
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
# rt_flush_period =
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
# rt_merge_iops = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
# rt_merge_maxiosize = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
# seamless_rotate = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#server_id
# server_id =
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
# shutdown_timeout = 3
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
# shutdown_token =
# https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
# snippets_file_prefix =
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
# sphinxql_state =
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
# sphinxql_timeout = 900
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
# ssl_ca =
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
# ssl_cert =
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
# ssl_key =
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
# subtree_docs_cache = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
# subtree_hits_cache = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
# thread_stack =
# https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
# unlink_old = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
# watchdog = 1
}
common {
# https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
# lemmatizer_base = /usr/local/share
# https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
# progressive_merge =
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
# json_autoconv_keynames =
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
# json_autoconv_numbers = 0
# https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
# on_json_attr_error = ignore_attr
# https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
# plugin_dir =
}
EOF

File diff suppressed because one or more lines are too long

View File

@ -7,7 +7,7 @@ python-dotenv
#manticoresearch #manticoresearch
numpy numpy
aioredis[hiredis] aioredis[hiredis]
aiokafka #aiokafka
vaderSentiment vaderSentiment
polyglot polyglot
pyicu pyicu
@ -21,3 +21,4 @@ python-Levenshtein
orjson orjson
uvloop uvloop
numba numba
elasticsearch[async]

View File

@ -1,50 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
# If Cypress run overwrite the password for admin and export env variables
if [ "$CYPRESS_CONFIG" == "true" ]; then
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
export SUPERSET_TESTENV=true
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
fi
#
# Make sure we have dev requirements installed
#
if [ -f "${REQUIREMENTS_LOCAL}" ]; then
echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
pip install -r "${REQUIREMENTS_LOCAL}"
else
echo "Skipping local overrides"
fi
if [[ "${1}" == "worker" ]]; then
echo "Starting Celery worker..."
celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO
elif [[ "${1}" == "beat" ]]; then
echo "Starting Celery beat..."
celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
elif [[ "${1}" == "app" ]]; then
echo "Starting web app..."
flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
elif [[ "${1}" == "app-gunicorn" ]]; then
echo "Starting web app..."
/usr/bin/run-server.sh
fi

View File

@ -1,78 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
#
# Always install local overrides first
#
/app/docker/docker-bootstrap.sh
STEP_CNT=4
echo_step() {
cat <<EOF
######################################################################
Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
######################################################################
EOF
}
ADMIN_PASSWORD="admin"
# If Cypress run overwrite the password for admin and export env variables
if [ "$CYPRESS_CONFIG" == "true" ]; then
ADMIN_PASSWORD="general"
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
export SUPERSET_TESTENV=true
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
fi
# Initialize the database
echo_step "1" "Starting" "Applying DB migrations"
superset db upgrade
echo_step "1" "Complete" "Applying DB migrations"
# Create an admin user
echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
superset fab create-admin \
--username admin \
--firstname Superset \
--lastname Admin \
--email admin@superset.com \
--password $ADMIN_PASSWORD
echo_step "2" "Complete" "Setting up admin user"
# Create default roles and permissions
echo_step "3" "Starting" "Setting up roles and perms"
superset init
echo_step "3" "Complete" "Setting up roles and perms"
if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
# Load some data to play with
echo_step "4" "Starting" "Loading examples"
# If Cypress run which consumes superset_test_config load required data for tests
if [ "$CYPRESS_CONFIG" == "true" ]; then
superset load_test_users
superset load_examples --load-test-data
else
superset load_examples
fi
echo_step "4" "Complete" "Loading examples"
fi

View File

@ -1 +0,0 @@
pydruid

View File

@ -1,87 +0,0 @@
{
"type": "kafka",
"spec": {
"ioConfig": {
"type": "kafka",
"consumerProperties": {
"bootstrap.servers": "kafka:9092"
},
"topic": "main",
"inputFormat": {
"type": "json"
},
"useEarliestOffset": true
},
"tuningConfig": {
"type": "kafka",
"maxTotalRows": 250000
},
"dataSchema": {
"dataSource": "main",
"timestampSpec": {
"column": "ts",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"id",
"host",
"ident",
"nick",
"nick_id",
"user_id",
"msg",
"msg_id",
"net",
"net_id",
"num",
"src",
"ts",
"date",
"time",
"type",
"bot",
"channel",
"channel_category",
"channel_category_id",
"channel_category_nsfw",
"channel_id",
"channel_nsfw",
{
"type": "long",
"name": "guild_member_count"
},
"guild",
"guild_id",
"mode",
"modearg",
{
"type": "double",
"name": "sentiment"
},
"status",
"user",
"exemption",
"num_chans",
"num_users",
"online",
"realname",
"server",
"mtype",
"tokens",
"lang_code",
"lang_name",
"filename",
"file_md5",
"file_ext",
"file_size"
]
},
"granularitySpec": {
"queryGranularity": "none",
"rollup": false,
"segmentGranularity": "year"
}
}
}
}

View File