Make performance settings configurable
This commit is contained in:
parent
24929a5fbb
commit
2c5133a546
|
@ -79,7 +79,7 @@ services:
|
||||||
container_name: metabase
|
container_name: metabase
|
||||||
image: metabase/metabase:latest
|
image: metabase/metabase:latest
|
||||||
ports:
|
ports:
|
||||||
- 3001:3000
|
- 3096:3000
|
||||||
depends_on:
|
depends_on:
|
||||||
- broker
|
- broker
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
PORTAINER_GIT_DIR=.
|
||||||
|
MODULES_ENABLED="dis"
|
||||||
|
DISCORD_TOKEN="xx"
|
||||||
|
THRESHOLD_LISTENER_HOST=0.0.0.0
|
||||||
|
THRESHOLD_LISTENER_PORT=13867
|
||||||
|
THRESHOLD_LISTENER_SSL=1
|
||||||
|
|
||||||
|
THRESHOLD_RELAY_ENABLED=0
|
||||||
|
THRESHOLD_RELAY_HOST=0.0.0.0
|
||||||
|
THRESHOLD_RELAY_PORT=13868
|
||||||
|
THRESHOLD_RELAY_SSL=1
|
||||||
|
|
||||||
|
THRESHOLD_API_ENABLED=1
|
||||||
|
THRESHOLD_API_HOST=0.0.0.0
|
||||||
|
THRESHOLD_API_PORT=13869
|
||||||
|
PORTAINER_GIT_DIR=.
|
||||||
|
|
||||||
|
THRESHOLD_CONFIG_DIR=./legacy/conf/live/
|
||||||
|
THRESHOLD_CERT_DIR=./legacy/conf/cert/
|
||||||
|
|
||||||
|
# How many messages to ingest at once from Redis
|
||||||
|
MONOLITH_INGEST_CHUNK_SIZE=900
|
||||||
|
|
||||||
|
# Time to wait between polling Redis again
|
||||||
|
MONOLITH_INGEST_ITER_DELAY=0.5
|
||||||
|
|
||||||
|
# Number of 4chan threads to request at once
|
||||||
|
MONOLITH_CH4_THREADS_CONCURRENT=1000
|
||||||
|
|
||||||
|
# Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
|
||||||
|
MONOLITH_CH4_THREADS_DELAY=0.1
|
||||||
|
|
||||||
|
# Time to wait after finishing a crawl before starting again
|
||||||
|
MONOLITH_CH4_CRAWL_DELAY=30
|
||||||
|
|
||||||
|
# Semaphore value
|
||||||
|
MONOLITH_CH4_THREADS_SEMAPHORE=1000
|
|
@ -10,19 +10,21 @@ from numpy import array_split
|
||||||
import db
|
import db
|
||||||
import util
|
import util
|
||||||
|
|
||||||
|
from os import getenv
|
||||||
|
|
||||||
# CONFIGURATION #
|
# CONFIGURATION #
|
||||||
|
|
||||||
# Number of 4chan threads to request at once
|
# Number of 4chan threads to request at once
|
||||||
THREADS_CONCURRENT = 1000
|
THREADS_CONCURRENT = int(getenv("MONOLITH_CH4_THREADS_CONCURRENT", 1000))
|
||||||
|
|
||||||
# Seconds to wait between every THREADS_CONCURRENT requests
|
# Seconds to wait between every THREADS_CONCURRENT requests
|
||||||
THREADS_DELAY = 0.1
|
THREADS_DELAY = float(getenv("MONOLITH_CH4_THREADS_DELAY", 0.1))
|
||||||
|
|
||||||
# Seconds to wait between crawls
|
# Seconds to wait between crawls
|
||||||
CRAWL_DELAY = 5
|
CRAWL_DELAY = int(getenv("MONOLITH_CH4_CRAWL_DELAY", 5))
|
||||||
|
|
||||||
# Semaphore value ?
|
# Semaphore value ?
|
||||||
THREADS_SEMAPHORE = 1000
|
THREADS_SEMAPHORE = int(getenv("MONOLITH_CH4_THREADS_SEMAPHORE", 1000))
|
||||||
|
|
||||||
# CONFIGURATION END #
|
# CONFIGURATION END #
|
||||||
|
|
||||||
|
|
|
@ -5,13 +5,14 @@ import orjson
|
||||||
import db
|
import db
|
||||||
import util
|
import util
|
||||||
from processing import process
|
from processing import process
|
||||||
|
from os import getenv
|
||||||
|
|
||||||
SOURCES = ["4ch", "irc", "dis"]
|
SOURCES = ["4ch", "irc", "dis"]
|
||||||
KEYPREFIX = "queue."
|
KEYPREFIX = "queue."
|
||||||
|
|
||||||
# Chunk size per source (divide by len(SOURCES) for total)
|
# Chunk size per source (divide by len(SOURCES) for total)
|
||||||
CHUNK_SIZE = 9000
|
CHUNK_SIZE = int(getenv("MONOLITH_INGEST_CHUNK_SIZE", "900"))
|
||||||
ITER_DELAY = 0.5
|
ITER_DELAY = float(getenv("MONOLITH_INGEST_ITER_DELAY", "0.5"))
|
||||||
|
|
||||||
log = util.get_logger("ingest")
|
log = util.get_logger("ingest")
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue