Make performance settings configurable

This commit is contained in:
Mark Veidemanis 2022-09-20 18:13:46 +01:00
parent 24929a5fbb
commit 2c5133a546
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
4 changed files with 47 additions and 7 deletions

View File

@ -79,7 +79,7 @@ services:
container_name: metabase container_name: metabase
image: metabase/metabase:latest image: metabase/metabase:latest
ports: ports:
- 3001:3000 - 3096:3000
depends_on: depends_on:
- broker - broker

37
env.example Normal file
View File

@ -0,0 +1,37 @@
PORTAINER_GIT_DIR=.
MODULES_ENABLED="dis"
DISCORD_TOKEN="xx"
THRESHOLD_LISTENER_HOST=0.0.0.0
THRESHOLD_LISTENER_PORT=13867
THRESHOLD_LISTENER_SSL=1
THRESHOLD_RELAY_ENABLED=0
THRESHOLD_RELAY_HOST=0.0.0.0
THRESHOLD_RELAY_PORT=13868
THRESHOLD_RELAY_SSL=1
THRESHOLD_API_ENABLED=1
THRESHOLD_API_HOST=0.0.0.0
THRESHOLD_API_PORT=13869
PORTAINER_GIT_DIR=.
THRESHOLD_CONFIG_DIR=./legacy/conf/live/
THRESHOLD_CERT_DIR=./legacy/conf/cert/
# How many messages to ingest at once from Redis
MONOLITH_INGEST_CHUNK_SIZE=900
# Time to wait between polling Redis again
MONOLITH_INGEST_ITER_DELAY=0.5
# Number of 4chan threads to request at once
MONOLITH_CH4_THREADS_CONCURRENT=1000
# Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
MONOLITH_CH4_THREADS_DELAY=0.1
# Time to wait after finishing a crawl before starting again
MONOLITH_CH4_CRAWL_DELAY=30
# Semaphore value
MONOLITH_CH4_THREADS_SEMAPHORE=1000

View File

@ -10,19 +10,21 @@ from numpy import array_split
import db import db
import util import util
from os import getenv
# CONFIGURATION # # CONFIGURATION #
# Number of 4chan threads to request at once # Number of 4chan threads to request at once
THREADS_CONCURRENT = 1000 THREADS_CONCURRENT = int(getenv("MONOLITH_CH4_THREADS_CONCURRENT", 1000))
# Seconds to wait between every THREADS_CONCURRENT requests # Seconds to wait between every THREADS_CONCURRENT requests
THREADS_DELAY = 0.1 THREADS_DELAY = float(getenv("MONOLITH_CH4_THREADS_DELAY", 0.1))
# Seconds to wait between crawls # Seconds to wait between crawls
CRAWL_DELAY = 5 CRAWL_DELAY = int(getenv("MONOLITH_CH4_CRAWL_DELAY", 5))
# Semaphore value ? # Semaphore value ?
THREADS_SEMAPHORE = 1000 THREADS_SEMAPHORE = int(getenv("MONOLITH_CH4_THREADS_SEMAPHORE", 1000))
# CONFIGURATION END # # CONFIGURATION END #

View File

@ -5,13 +5,14 @@ import orjson
import db import db
import util import util
from processing import process from processing import process
from os import getenv
SOURCES = ["4ch", "irc", "dis"] SOURCES = ["4ch", "irc", "dis"]
KEYPREFIX = "queue." KEYPREFIX = "queue."
# Chunk size per source (divide by len(SOURCES) for total) # Chunk size per source (divide by len(SOURCES) for total)
CHUNK_SIZE = 9000 CHUNK_SIZE = int(getenv("MONOLITH_INGEST_CHUNK_SIZE", "900"))
ITER_DELAY = 0.5 ITER_DELAY = float(getenv("MONOLITH_INGEST_ITER_DELAY", "0.5"))
log = util.get_logger("ingest") log = util.get_logger("ingest")