Begin switching away from Redis

This reverts commit 64e3e1160aa76d191740342ab3edc68807f890fb.

.gitignore

@@ -1,13 +1,161 @@
-*.pyc
-*.pem
-*.swp
+# ---> Python
+# Byte-compiled / optimized / DLL files
 __pycache__/
-conf/config.json
-conf/wholist.json
-conf/counters.json
-conf/monitor.json
-conf/tokens.json
-conf/network.dat
-conf/alias.json
-conf/dist.sh
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+# lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
 env/
+venv/
+env-glibc/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+.bash_history
+.vscode/
+docker/data
+*.pem
+legacy/conf/live/
+legacy/conf/cert/

.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+    - id: black
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: [--max-line-length=88]

Makefile

@@ -0,0 +1,20 @@
+run:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env up -d
+
+build:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env build
+
+stop:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env down
+
+log:
+	docker-compose -f docker/docker-compose.prod.yml --env-file=stack.env logs -f
+
+run-infra:
+	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env up -d
+
+stop-infra:
+	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env down
+
+log-infra:
+	docker-compose -f docker/docker-compose.infra.yml --env-file=stack.env logs -f

commands/list.py

@@ -1,26 +0,0 @@
-import main
-
-class ListCommand:
-    def __init__(self, *args):
-        self.list(*args)
-
-    def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
-        if authed:
-            if length == 2:
-                if not spl[1] in main.network.keys():
-                    failure("No such network: %s" % spl[1])
-                    return
-                if not 1 in main.network[spl[1]].relays.keys():
-                    failure("Network has no first instance")
-                    return
-                if not spl[1]+"1" in main.IRCPool.keys():
-                    failure("No IRC instance: %s - 1" % spl[1])
-                    return
-                main.IRCPool[spl[1]+"1"].list()
-                success("Requested list with first instance of %s" % spl[1])
-                return
-            else:
-                incUsage("list")
-                return
-        else:
-            incUsage(None)

commands/mon.py

@@ -1,280 +0,0 @@
-import main
-import argparse
-import sys
-from io import StringIO
-from yaml import dump
-
-class MonCommand:
-    def __init__(self, *args):
-        self.mon(*args)
-
-    def setup_arguments(self, ArgumentParser):
-        self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.")
-        group1 = self.parser.add_mutually_exclusive_group(required=True)
-        group1.add_argument("-a", "--add", metavar="entry", dest="addEntry",  help="Add an entry")
-        group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry")
-        group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries")
-        group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry")
-
-        group2 = self.parser.add_mutually_exclusive_group()
-        group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing")
-        group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing")
-
-        self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values")
-
-        self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who")
-        self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching")
-        self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching")
-        self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching")
-        self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching")
-        self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who")
-
-        self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)")
-        self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action")
-        self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick")
-        self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode")
-        self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode")
-        self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who")
-        self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who")
-        self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)")
-
-    def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
-        # We need to override the ArgumentParser class because
-            # it's only meant for CLI applications and exits
-            # after running, so we override the exit function
-            # to do nothing. We also need to do it here in order
-            # to catch the error messages and show them to the user.
-            # It sucks. I know.
-        if authed:
-            class ArgumentParser(argparse.ArgumentParser):
-                def exit(self, status=0, message=None):
-                    if message:
-                        failure(message)
-            self.setup_arguments(ArgumentParser)
-            if length == 1:
-                info("Incorrect usage, use mon -h for help")
-                return
-            old_stdout = sys.stdout
-            old_stderr = sys.stderr
-            my_stdout = sys.stdout = StringIO()
-            my_stderr = sys.stderr = StringIO()
-            try:
-                parsed = self.parser.parse_args(spl[1:])
-            except:
-                return
-            sys.stdout = old_stdout
-            sys.stderr = old_stderr
-            stdout = my_stdout.getvalue()
-            stderr = my_stdout.getvalue()
-            if not stdout == "":
-                info(stdout)
-            elif not stderr == "":
-                failure(my_stdout.getvalue())
-                return
-            my_stdout.close()
-            my_stderr.close()
-
-            if parsed.addEntry:
-                if parsed.addEntry in main.monitor.keys():
-                    failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry)
-                    return
-                cast = self.makeCast(parsed, failure, info)
-                if cast == False:
-                    return
-                main.monitor[parsed.addEntry] = cast
-                main.saveConf("monitor")
-                success("Successfully created monitor group %s" % parsed.addEntry)
-                return
-            elif parsed.delEntry:
-                if not parsed.delEntry in main.monitor.keys():
-                    failure("No such monitor group: %s" % parsed.delEntry)
-                    return
-                del main.monitor[parsed.delEntry]
-                main.saveConf("monitor")
-                success("Successfully removed monitor group: %s" % parsed.delEntry)
-                return
-            elif parsed.modEntry:
-                if not parsed.doAppend and not parsed.doRemove:
-                    failure("Specify --append or --remove with --mod")
-                    return
-                if not parsed.modEntry in main.monitor.keys():
-                    failure("No such monitor group: %s" % parsed.modEntry)
-                    return
-                cast = self.makeCast(parsed, failure, info)
-                if cast == False:
-                    return
-                if parsed.doAppend:
-                    merged = self.addCast(main.monitor[parsed.modEntry], cast, info)
-                    main.monitor[parsed.modEntry] = merged
-                elif parsed.doRemove:
-                    merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info)
-                    if merged == {}:
-                        del main.monitor[parsed.modEntry]
-                        info("Group %s deleted due to having no attributes" % parsed.modEntry)
-                        main.saveConf("monitor")
-                        return
-                    main.monitor[parsed.modEntry] = merged
-                main.saveConf("monitor")
-                success("Successfully updated entry %s" % parsed.modEntry)
-                return
-            elif parsed.listEntry:
-                info(dump(main.monitor))
-                return
-
-        else:
-            incUsage(None)
-
-    def dedup(self, data):
-        if not isinstance(data, bool):
-            return list(set(data))
-        return data
-
-    def parseNetworkFormat(self, lst, failure, info):
-        cast = {}
-        if lst == None:
-            return "nil"
-        if len(lst) == 0:
-            failure("List has no entries")
-            return False
-        elif len(lst) > 0:
-            if len(lst[0]) == 0:
-                failure("Nested list has no entries")
-                return False
-        for i in lst:
-            if not i[0] in main.pool.keys():
-                failure("Name does not exist: %s" % i[0])
-                return False
-            if i[0] in main.IRCPool.keys():
-                for x in i[1:]:
-                    if not x in main.IRCPool[i[0]].channels:
-                        info("%s: Bot not on channel: %s" % (i[0], x))
-            if len(i) == 1:
-                cast[i[0]] = True
-            else:
-                if i[0] in cast.keys():
-                    if not cast[i[0]] == True:
-                        for x in self.dedup(i[1:]):
-                            cast[i[0]].append(x)
-                    else:
-                        cast[i[0]] = self.dedup(i[1:])
-                else:
-                    cast[i[0]] = self.dedup(i[1:])
-        for i in cast.keys():
-            deduped = self.dedup(cast[i])
-            cast[i] = deduped
-        return cast
-
-    # Create or modify a monitor group magically
-    def makeCast(self, obj, failure, info):
-        validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"]
-        cast = {}
-        if not obj.specType == None:
-            types = self.dedup(obj.specType)
-            for i in types:
-                if not i in validTypes:
-                    failure("Invalid type: %s" % i)
-                    info("Available types: %s" % ", ".join(validTypes))
-                    return False
-            cast["type"] = types
-        if not obj.source == None:
-            sourceParse = self.parseNetworkFormat(obj.source, failure, info)
-            if not sourceParse:
-                return False
-            if not sourceParse == "nil":
-                cast["sources"] = sourceParse
-
-        if not obj.send == None:
-            sendParse = self.parseNetworkFormat(obj.send, failure, info)
-            if not sendParse:
-                return False
-            if not sendParse == "nil":
-                cast["send"] = sendParse
-
-        if not obj.message == None:
-            cast["message"] = []
-            for i in obj.message:
-                cast["message"].append(" ".join(i))
-
-        if not obj.user == None:
-            cast["user"] = obj.user
-        if not obj.modes == None:
-            cast["modes"] = obj.modes
-        if not obj.modeargs == None:
-            cast["modeargs"] = obj.modeargs
-        if not obj.server == None:
-            cast["server"] = obj.server
-        if not obj.status == None:
-            cast["status"] = obj.status
-        if not obj.free == None:
-            cast["free"] = obj.free
-        if not obj.nick == None:
-            cast["nick"] = obj.nick
-        if not obj.ident == None:
-            cast["ident"] = obj.ident
-        if not obj.host == None:
-            cast["host"] = obj.host
-        if not obj.real == None:
-            cast["real"] = []
-            for i in obj.real:
-                cast["real"].append(" ".join(i))
-        if not obj.inside == None:
-            if obj.inside.lower() in ["yes", "true", "on"]:
-                cast["inside"] = True
-            elif obj.inside.lower() in ["no", "false", "off"]:
-                cast["inside"] = False
-            else:
-                failure("inside: Unknown operand: %s" % obj.inside)
-                return
-        return cast
-
-    def subtractCast(self, source, patch, info):
-        for i in patch.keys():
-            if i in source.keys():
-                if isinstance(source[i], dict):
-                    result = self.subtractCast(source[i], patch[i], info)
-                    if result == {}:
-                        info("Removing upper element: %s" % i)
-                        del source[i]
-                        continue
-                    source[i] = result
-                    continue
-                if isinstance(patch[i], bool):
-                    del source[i]
-                    info("Removing entire element: %s" % i)
-                    continue
-                for x in patch[i]:
-                    if isinstance(source[i], bool):
-                        info("Attempt to remove %s from network-wide definition" % x)
-                        continue
-                    if x in source[i]:
-                        source[i].remove(x)
-                        if source[i] == []:
-                            del source[i]
-                    else:
-                        info("Element %s not in source %s" % (x, i))
-            else:
-                info("Non-matched key: %s" % i)
-        return source
-
-    def addCast(self, source, patch, info):
-        for i in patch.keys():
-            if i in source.keys():
-                if isinstance(source[i], dict):
-                    result = self.addCast(source[i], patch[i], info)
-                    source[i] = result
-                    continue
-                if isinstance(patch[i], bool):
-                    source[i] = patch[i]
-                    info("Overriding local element %s with network-wide definition" % i)
-                    continue
-                for x in patch[i]:
-                    if isinstance(source[i], bool):
-                        source[i] = []
-                        info("Overriding element %s, previously set network-wide" % i)
-                    if x in source[i]:
-                        info("Element %s already in source %s" % (x, i))
-                    else:
-                        source[i].append(x)
-            else:
-                source[i] = patch[i]
-        return source

conf/example/alias.json

@@ -1 +0,0 @@
-{}

conf/example/counters.json

@@ -1 +0,0 @@
-{}

conf/example/masterbuf.json

@@ -1 +0,0 @@
-[]

conf/example/monitor.json

@@ -1 +0,0 @@
-{}

conf/example/tokens.json

@@ -1 +0,0 @@
-{}

core/bot.py

@@ -1,706 +0,0 @@
-from twisted.internet.protocol import ReconnectingClientFactory
-from twisted.words.protocols.irc import IRCClient
-from twisted.internet.defer import Deferred
-from twisted.internet.task import LoopingCall
-from twisted.internet import reactor, task
-from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage
-
-import sys
-from string import digits
-from random import randint
-from copy import deepcopy
-
-from modules import userinfo
-from modules import counters
-from modules import monitor
-from modules import chankeep
-
-from core.relay import sendRelayNotification
-from utils.dedup import dedup
-from utils.get import getRelay
-
-import main
-from utils.logging.log import *
-from utils.logging.debug import *
-from utils.logging.send import *
-from utils.parsing import parsen
-
-from twisted.internet.ssl import DefaultOpenSSLContextFactory
-
-def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
-    keyFN = main.certPath+main.config["Key"]
-    certFN = main.certPath+main.config["Certificate"]
-    contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"),
-            certFN.encode("utf-8", "replace"))
-    bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
-    host, port = getRelay(num)
-    rct = reactor.connectSSL(host,
-            port,
-            bot, contextFactory)
-
-def parsemsg(s):
-    """
-    Breaks a message from an IRC server into its prefix, command, and
-    arguments.
-    @param s: The message to break.
-    @type s: L{bytes}
-    @return: A tuple of (prefix, command, args).
-    @rtype: L{tuple}
-    """
-    prefix = ''
-    trailing = []
-    if not s:
-        raise IRCBadMessage("Empty line.")
-    if s[0:1] == ':':
-        prefix, s = s[1:].split(' ', 1)
-    if s.find(' :') != -1:
-        s, trailing = s.split(' :', 1)
-        args = s.split(' ')
-        args.append(trailing)
-    else:
-        args = s.split(' ')
-    command = args.pop(0)
-    return prefix, command, args
-
-class IRCRelay(IRCClient):
-    def __init__(self, num, relayCommands, user, stage2):
-        self.isconnected = False
-        self.buffer = ""
-        if user == None:
-            self.user = main.config["Relay"]["User"]
-        else:
-            self.user = user
-        password = main.config["Relay"]["Password"]
-        self.nickname = "relay"
-        self.realname = "relay"
-        self.username = self.user
-        self.password = self.user+":"+password
-
-        self.relayCommands = relayCommands
-        self.num = num
-        self.stage2 = stage2
-        self.loop = None
-
-    def privmsg(self, user, channel, msg):
-        nick, ident, host = parsen(user)
-        for i in main.ZNCErrors:
-            if i in msg:
-                error("ZNC issue:", msg)
-        if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
-            nick = nick[1:]
-            if nick in self.relayCommands.keys():
-                sendAll("[%s] %s -> %s" % (self.num, nick, msg))
-
-    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
-        log("%s: relay password mismatch" % self.num)
-        sendAll("%s: relay password mismatch" % self.num)
-
-    def sendStage2(self):
-        if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []]
-            if not len(self.stage2) == 0:
-                user = self.stage2[0].pop(0)
-                commands = self.stage2[0].pop(0)
-                del self.stage2[0]
-                deliverRelayCommands(self.num, commands, user, self.stage2)
-
-    def signedOn(self):
-        if not self.isconnected:
-            self.isconnected = True
-            log("signed on as a relay: %s" % self.num)
-            #sendRelayNotification("Relay", {"type": "conn", "status": "connected"})  nobody actually cares
-            sleeptime = 0
-            increment = 0.8
-            for i in self.relayCommands.keys():
-                for x in self.relayCommands[i]:
-                    reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x)
-                sleeptime += increment
-                increment += 0.8
-            reactor.callLater(sleeptime, self.sendStage2)
-            reactor.callLater(sleeptime+5, self.transport.loseConnection)
-            return
-
-class IRCBot(IRCClient):
-    def __init__(self, net, num):
-        self.isconnected = False
-        self.channels = []
-        self.net = net
-        self.num = num
-        self.buffer = ""
-        self.name = net + str(num)
-        alias = main.alias[num]
-        relay = main.network[self.net].relays[num]
-        self.nickname = alias["nick"]
-        self.realname = alias["realname"]
-        self.username = alias["nick"]+"/"+relay["net"]
-        self.password = main.config["Relay"]["Password"]
-        self.userinfo = None      #
-        self.fingerReply = None   #
-        self.versionName = None   # don't tell anyone information about us
-        self.versionNum = None    #
-        self.versionEnv = None    #
-        self.sourceURL = None     #
-
-        self._getWho = {} # LoopingCall objects -- needed to be able to stop them
-
-        self._tempWho = {} # temporary storage for gathering WHO info
-        self._tempNames = {} # temporary storage for gathering NAMES info
-        self._tempList = ([], []) # temporary storage for gathering LIST info
-        self.listOngoing = False   # we are currently receiving a LIST
-        self.listRetried = False   # we asked and got nothing so asked again
-        self.listAttempted = False # we asked for a list
-        self.listSimple = False    # after asking again we got the list, so use the simple
-                                   #  syntax from now on
-        self.wantList = False      # we want to send a LIST, but not all relays are active yet
-        self.chanlimit = 0
-        self.servername = None
-
-    def lineReceived(self, line):
-        if bytes != str and isinstance(line, bytes):
-            # decode bytes from transport to unicode
-            line = line.decode("utf-8", "replace")
-
-        line = lowDequote(line)
-        try:
-            prefix, command, params = parsemsg(line)
-            if command in numeric_to_symbolic:
-                command = numeric_to_symbolic[command]
-            self.handleCommand(command, prefix, params)
-        except IRCBadMessage:
-            self.badMessage(line, *sys.exc_info())
-
-    def joinChannels(self, channels):
-        sleeptime = 0.0
-        increment = 0.8
-        for i in channels:
-            if not i in self.channels:
-                debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds")
-                reactor.callLater(sleeptime, self.join, i)
-                sleeptime += increment
-                if sleeptime == 10:
-                    sleeptime = 0.0
-                    increment = 0.7
-                increment += 0.1
-            else:
-                error("%s - %i - Cannot join channel we are already on - %s" % (self.net, self.num, i))
-
-    def checkChannels(self):
-        if self.net in main.TempChan.keys():
-            if self.num in main.TempChan[self.net].keys():
-                self.joinChannels(main.TempChan[self.net][self.num])
-                del main.TempChan[self.net][self.num]
-                if not main.TempChan[self.net]:
-                    del main.TempChan[self.net]
-
-    def event(self, **cast):
-        for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over
-            if cast[i] == "":       #  a dictionary that changes length with each iteration
-                del cast[i]
-        if "muser" in cast.keys():
-            if cast["muser"] == self.servername:
-                return
-        if "channel" in cast.keys():
-            if cast["channel"] == "*":
-                return
-        if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
-            cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
-        if set(["nick", "ident", "host", "msg"]).issubset(set(cast)):
-            if "msg" in cast.keys():
-                if cast["ident"] == "znc" and cast["host"] == "znc.in":
-                    cast["type"] = "znc"
-                    cast["num"] = self.num
-                    del cast["nick"]
-                    del cast["ident"]
-                    del cast["host"]
-                    del cast["channel"]
-                    if "Disconnected from IRC" in cast["msg"]:
-                        log("ZNC disconnected on %s - %i" % (self.net, self.num))
-                        self.isconnected = False
-                    if "Connected!" in cast["msg"]:
-                        log("ZNC connected on %s - %i" % (self.net, self.num))
-                        self.isconnected = True
-        if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
-            if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here
-                if cast["channel"].lower() == self.nickname.lower():    #  as they are channel == nickname
-                    #castDup = deepcopy(cast)                          #  however modes are not queries!
-                    cast["mtype"] = cast["type"]
-                    cast["type"] = "query"
-                    cast["num"] = self.num
-                    #self.event(**castDup)
-                    # Don't call self.event for this one because queries are not events on a
-                    #  channel, but we still want to see them
-            if "user" in cast.keys():
-                if cast["user"].lower() == self.nickname.lower():
-                    castDup = deepcopy(cast)
-                    castDup["mtype"] = cast["type"]
-                    castDup["type"] = "self"
-                    cast["num"] = self.num
-                    self.event(**castDup)
-            if "nick" in cast.keys():
-                if cast["nick"].lower() == self.nickname.lower():
-                    castDup = deepcopy(cast)
-                    castDup["mtype"] = cast["type"]
-                    castDup["type"] = "self"
-                    cast["num"] = self.num
-                    if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly
-                        self.event(**castDup)                               #  don't tell anyone else
-            if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
-                if not cast["msg"] == None:
-                    if self.nickname.lower() in cast["msg"].lower():
-                        castDup = deepcopy(cast)
-                        castDup["mtype"] = cast["type"]
-                        castDup["type"] = "highlight"
-                        cast["num"] = self.num
-                        self.event(**castDup)
-
-        if not "net" in cast.keys():
-            cast["net"] = self.net
-        counters.event(self.net, cast["type"])
-        monitor.event(self.net, cast)
-
-    def privmsg(self, user, channel, msg):
-        self.event(type="msg", muser=user, channel=channel, msg=msg)
-
-    def noticed(self, user, channel, msg):
-        self.event(type="notice", muser=user, channel=channel, msg=msg)
-
-    def action(self, user, channel, msg):
-        self.event(type="action", muser=user, channel=channel, msg=msg)
-
-    def get(self, var):
-        try:
-            result = getattr(self, var)
-        except AttributeError:
-            result = None
-        return result
-
-    def setNick(self, nickname):
-        self._attemptedNick = nickname
-        self.sendLine("NICK %s" % nickname)
-        self.nickname = nickname
-
-    def alterCollidedNick(self, nickname):
-        newnick = nickname + "_"
-        return newnick
-
-    def nickChanged(self, olduser, newnick):
-        self.nickname = newnick
-        self.event(type="self", mtype="nick", muser=olduser, user=newnick)
-
-    def irc_ERR_NICKNAMEINUSE(self, prefix, params):
-        self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
-        self.setNick(self._attemptedNick)
-
-    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
-        log("%s - %i: password mismatch" % (self.net, self.num))
-        sendAll("%s - %i: password mismatch" % (self.net, self.num))
-
-    def _who(self, channel):
-        d = Deferred()
-        if channel not in self._tempWho:
-            self._tempWho[channel] = ([], [])
-        self._tempWho[channel][0].append(d)
-        self.sendLine("WHO %s" % channel)
-        return d
-
-    def who(self, channel):
-        self._who(channel).addCallback(self.got_who)
-
-    def irc_RPL_WHOREPLY(self, prefix, params):
-        channel = params[1]
-        ident = params[2]
-        host = params[3]
-        server = params[4]
-        nick = params[5]
-        status = params[6]
-        realname = params[7]
-        if channel not in self._tempWho:
-            return
-        n = self._tempWho[channel][1]
-        n.append([nick, nick, host, server, status, realname])
-        self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status)
-
-    def irc_RPL_ENDOFWHO(self, prefix, params):
-        channel = params[1]
-        if channel not in self._tempWho:
-            return
-        callbacks, info = self._tempWho[channel]
-        for cb in callbacks:
-            cb.callback((channel, info))
-        del self._tempWho[channel]
-
-    def got_who(self, whoinfo):
-        userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
-
-    def sanit(self, data):
-        if len(data) >= 1:
-            if data[0] in ["!", "~", "&", "@", "%", "+"]:
-                data = data[1:]
-                return data
-            return data
-        else:
-            return False
-
-    def names(self, channel):
-        d = Deferred()
-        if channel not in self._tempNames:
-            self._tempNames[channel] = ([], [])
-        self._tempNames[channel][0].append(d)
-        self.sendLine("NAMES %s" % channel)
-        return d
-
-    def irc_RPL_NAMREPLY(self, prefix, params):
-        channel = params[2]
-        nicklist = params[3].split(' ')
-        if channel not in self._tempNames:
-            return
-        n = self._tempNames[channel][1]
-        n.append(nicklist)
-
-    def irc_RPL_ENDOFNAMES(self, prefix, params):
-        channel = params[1]
-        if channel not in self._tempNames:
-            return
-        callbacks, namelist = self._tempNames[channel]
-        for cb in callbacks:
-            cb.callback((channel, namelist))
-        del self._tempNames[channel]
-
-    def got_names(self, nicklist):
-        newNicklist = []
-        for i in nicklist[1]:
-            for x in i:
-                f = self.sanit(x) # need to store this as well, or potentially just do not remove it...
-                if f:
-                    newNicklist.append(f)
-        userinfo.initialNames(self.net, nicklist[0], newNicklist)
-
-    def myInfo(self, servername, version, umodes, cmodes):
-        self.servername = servername
-
-    def _list(self, noargs):
-        d = Deferred()
-        self._tempList = ([], [])
-        self._tempList[0].append(d)
-        if self.listSimple:
-            self.sendLine("LIST")
-            return d            # return early if we know what to do
-
-        if noargs:
-            self.sendLine("LIST")
-        else:
-            self.sendLine("LIST >0")
-        return d
-
-    def list(self, noargs=False, nocheck=False):
-        if self.listAttempted:
-            debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num))
-            return
-        else:
-            self.listAttempted = True
-        if self.listOngoing:
-            debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num))
-            return
-        else:
-            if nocheck:
-                allRelays = True # override the system - if this is
-            else:                #  specified, we already did this
-                allRelays = chankeep.allRelaysActive(self.net)
-            if not allRelays:
-                self.wantList = True
-                debug("Not all relays were active for LIST request")
-                return
-            self._list(noargs).addCallback(self.got_list)
-
-    def irc_RPL_LISTSTART(self, prefix, params):
-        self.listAttempted = False
-        self.listOngoing = True
-        self.wantList = False
-
-    def irc_RPL_LIST(self, prefix, params):
-        channel = params[1]
-        users = params[2]
-        topic = params[3]
-        self._tempList[1].append([channel, users, topic])
-
-    def irc_RPL_LISTEND(self, prefix, params):
-        if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there
-            debug("We didn't ask for this LIST, discarding")
-            self._tempList[0].clear()
-            self._tempList[1].clear()
-            return
-        callbacks, info = self._tempList
-        self.listOngoing = False
-        for cb in callbacks:
-            cb.callback((info))
-        noResults = False
-        if len(self._tempList[1]) == 0:
-            noResults = True
-        self._tempList[0].clear()
-        self._tempList[1].clear()
-        if noResults:
-            if self.listRetried:
-                warn("LIST still empty after retry: %s - %i" % (net, num))
-                self.listRetried = False
-                return
-            else:
-                self.list(True)
-                self.listRetried = True
-        else:
-            if self.listRetried:
-                self.listRetried = False
-                debug("List received after retry - defaulting to simple list syntax")
-                self.listSimple = True
-
-    def got_list(self, listinfo):
-        if len(listinfo) == 0: # probably ngircd not supporting LIST >0
-            return
-        chankeep.initialList(self.net, self.num, listinfo, self.chanlimit)
-
-    def isupport(self, options):
-        interested = ("CHANLIMIT", "MAXCHANNELS")
-        if not any((x for x in options if any(y in x for y in interested))):
-            return          # check if any of interested is in any of options, some networks
-        chanlimit = None    #  call isupport() more than once, so discard swiftly anything
-        if not self.isconnected: #  we don't care about
-            log("endpoint connected: %s - %i" % (self.net, self.num))
-            self.isconnected = True
-        for i in options:
-            if i.startswith("CHANLIMIT"):
-                if ":" in i:
-                    split = i.split(":")
-                    if len(split) >= 2:
-                        chanlimit = split[1]
-                        break
-            elif i.startswith("MAXCHANNELS"):
-                if "=" in i:
-                    split = i.split("=")
-                    if len(split) == 2:
-                        chanlimit = split[1]
-                        break
-        print("chanlimit", chanlimit)
-        try:
-            self.chanlimit = int(chanlimit)
-        except TypeError:
-            warn("Invalid chanlimit: %s" % i)
-        if self.chanlimit == 0:
-            self.chanlimit = 200 # don't take the piss if it's unlimited
-        allRelays = chankeep.allRelaysActive(self.net)
-        print(self.net, self.num, allRelays)
-        if allRelays:
-            for i in main.network.keys():
-                for x in main.network[i].relays.keys():
-                    name = i+str(x)
-                    if main.IRCPool[name].wantList == True:
-                        main.IRCPool[name].list(nocheck=True)
-                        debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
-        if self.num == 1: # Only one instance should do a list
-            if self.chanlimit:
-                if allRelays:
-                    self.list()
-                else:
-                    self.wantList = True
-            else:
-                debug("Aborting LIST due to bad chanlimit")
-        self.checkChannels()
-
-    #twisted sucks so i have to do this to actually get the user info
-    def irc_JOIN(self, prefix, params):
-        """
-        Called when a user joins a channel.
-        """
-        nick = prefix.split('!')[0]
-        channel = params[-1]
-        if nick == self.nickname:
-            self.joined(channel)
-        else:
-            self.userJoined(prefix, channel)
-
-    def irc_PART(self, prefix, params):
-        """
-        Called when a user leaves a channel.
-        """
-        nick = prefix.split('!')[0]
-        channel = params[0]
-        if len(params) >= 2:
-            message = params[1]
-        else:
-            message = None
-        if nick == self.nickname:
-            self.left(prefix, channel, message)
-        else:
-            self.userLeft(prefix, channel, message)
-
-    def irc_QUIT(self, prefix, params):
-        """
-        Called when a user has quit.
-        """
-        nick = prefix.split('!')[0]
-        #if nick == self.nickname:
-        #self.botQuit(prefix, params[0])
-        #else:
-        self.userQuit(prefix, params[0])
-
-    def irc_NICK(self, prefix, params):
-        """
-        Called when a user changes their nickname.
-        """
-        nick = prefix.split('!', 1)[0]
-        if nick == self.nickname:
-            self.nickChanged(prefix, params[0])
-        else:
-            self.userRenamed(prefix, params[0])
-
-    def irc_KICK(self, prefix, params):
-        """
-        Called when a user is kicked from a channel.
-        """
-        #kicker = prefix.split('!')[0]
-        channel = params[0]
-        kicked = params[1]
-        message = params[-1]
-        # Checks on whether it was us that was kicked are done in userKicked
-        self.userKicked(kicked, channel, prefix, message)
-
-    def irc_TOPIC(self, prefix, params):
-        """
-        Someone in the channel set the topic.
-        """
-        #user = prefix.split('!')[0]
-        channel = params[0]
-        newtopic = params[1]
-        self.topicUpdated(prefix, channel, newtopic)
-    #END hacks
-
-    def signedOn(self):
-        log("signed on: %s - %i" % (self.net, self.num))
-        #self.event(type="conn", status="connected")
-        sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"})
-
-    def joined(self, channel):
-        if not channel in self.channels:
-            self.channels.append(channel)
-        self.names(channel).addCallback(self.got_names)
-        if main.config["Toggles"]["Who"]:
-            #self.who(channel).addCallback(self.got_who)
-            #self.who(channel)
-            lc = LoopingCall(self.who, channel)
-            self._getWho[channel] = lc
-            intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
-            minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
-            interval = randint(minint, minint+intrange)
-            lc.start(interval)
-
-    def botLeft(self, channel):
-        if channel in self.channels:
-            self.channels.remove(channel)
-        if channel in self._getWho.keys():
-            lc = self._getWho[channel]
-            lc.stop()
-            del self._getWho[channel]
-        userinfo.delChannels(self.net, [channel])                    # < we do not need to deduplicate this
-        #log("Can no longer cover %s, removing records" % channel)#  as it will only be matched once --
-                                                                  #  other bots have different nicknames so
-    def left(self, user, channel, message):                       #  even if they saw it, they wouldn't react
-        self.event(type="part", muser=user, channel=channel, message=message)
-        self.botLeft(channel)
-
-    def userJoined(self, user, channel):
-        self.event(type="join", muser=user, channel=channel)
-
-    def userLeft(self, user, channel, message):
-        self.event(type="part", muser=user, channel=channel, message=message)
-
-    def userQuit(self, user, quitMessage):
-        self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage})
-
-    def userKicked(self, kickee, channel, kicker, message):
-        if kickee.lower() == self.nickname.lower():
-            self.botLeft(channel)
-        self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee)
-
-    def chanlessEvent(self, cast):
-        cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
-        if dedup(self.name, cast): # Needs to be kept self.name until the dedup
-                                   #  function is converted to the new net, num
-                                   #  format
-            return # stop right there sir!
-        chans = userinfo.getChanList(self.net, cast["nick"])
-        if chans == None:
-            error("No channels returned for chanless event: %s" % cast)
-        #    self.event(**cast) -- no, should NEVER happen
-            return
-        # getChansSingle returns all channels of the user, we only want to use
-        #  ones we have common with them
-        realChans = set(chans).intersection(set(self.channels))
-        for i in realChans:
-            cast["channel"] = i
-            self.event(**cast)
-
-    def userRenamed(self, oldname, newname):
-        self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
-
-    def topicUpdated(self, user, channel, newTopic):
-        self.event(type="topic", muser=user, channel=channel, message= newTopic)
-
-    def modeChanged(self, user, channel, toset, modes, args):
-        argList = list(args)
-        modeList = [i for i in modes]
-        for a, m in zip(argList, modeList):
-            self.event(type="mode", muser=user, channel=channel, modes=m, status=toset, modeargs=a)
-
-class IRCBotFactory(ReconnectingClientFactory):
-    def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
-        if net == None:
-            self.num = num
-            self.net = None
-            self.name = "relay - %i" % num
-            self.relay = True
-        else:
-            self.name = net+str(num)
-            self.num = num
-            self.net = net
-            self.relay = False
-        self.client = None
-        self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
-        self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
-        self.factor = main.config["Tweaks"]["Delays"]["Factor"]
-        self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
-
-        self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
-
-    def buildProtocol(self, addr):
-        if self.relay == False:
-            entry = IRCBot(self.net, self.num)
-            main.IRCPool[self.name] = entry
-        else:
-            entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
-
-        self.client = entry
-        return entry
-
-    def clientConnectionLost(self, connector, reason):
-        if not self.relay:
-            userinfo.delChannels(self.net, self.client.channels)
-        if not self.client == None:
-            self.client.isconnected = False
-            self.client.channels = []
-        error = reason.getErrorMessage()
-        log("%s - %i: connection lost: %s" % (self.net, self.num, error))
-        if not self.relay:
-            sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
-            sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error})
-            self.retry(connector)
-        #ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
-
-    def clientConnectionFailed(self, connector, reason):
-        if not self.client == None:
-            self.client.isconnected = False
-            self.client.channels = []
-        error = reason.getErrorMessage()
-        log("%s - %i: connection failed: %s" % (self.net, self.num, error))
-        if not self.relay:
-            sendAll("%s -%s: connection failed: %s" % (self.net, self.num, error))
-            sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error})
-            self.retry(connector)
-        #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
-

core/parser.py

@@ -1,29 +0,0 @@
-import main
-from utils.logging.log import *
-from utils.logging.send import *
-
-def parseCommand(addr, authed, data):
-    #call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
-    spl = data.split()
-    if addr in main.connections.keys():
-        obj = main.connections[addr]
-    else:
-        warn("Got connection object with no instance in the address pool")
-        return
-
-    success = lambda data: sendSuccess(addr, data)
-    failure = lambda data: sendFailure(addr, data)
-    info = lambda data: sendInfo(addr, data)
-
-    incUsage = lambda mode: incorrectUsage(addr, mode)
-    length = len(spl)
-    if len(spl) > 0:
-        cmd = spl[0]
-    else:
-        failure("No text was sent")
-        return
-    if spl[0] in main.CommandMap.keys():
-        main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length)
-        return
-    incUsage(None)
-    return

db.py

@@ -0,0 +1,128 @@
+import random
+
+import aioredis
+import orjson
+
+# Kafka
+from aiokafka import AIOKafkaProducer
+import redis
+
+import util
+
+# KAFKA_TOPIC = "msg"
+
+log = util.get_logger("db")
+
+# Redis (legacy)
+r = redis.from_url("redis://ssdb:1289", db=0)
+
+# AIORedis
+ar = aioredis.from_url("redis://ssdb:1289", db=0)
+
+TYPES_MAIN = [
+    "msg",
+    "notice",
+    "action",
+    "part",
+    "join",
+    "kick",
+    "quit",
+    "nick",
+    "mode",
+    "topic",
+    "update",
+]
+MAIN_SRC_MAP = {
+    "dis": "main",
+    "irc": "restricted",
+    "4ch": "main",
+}
+
+TYPES_META = ["who"]
+TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
+KEYNAME = "queue"
+
+
+async def store_kafka_batch(data):
+    print("FAKE STORE KAFKA BATCH")
+    return
+    # log.debug(f"Storing Kafka batch of {len(data)} messages")
+    producer = AIOKafkaProducer(bootstrap_servers="kafka:9092")
+    await producer.start()
+    topicmap = {}
+    for msg in data:
+        if msg["type"] in TYPES_MAIN:
+            # index = "main"
+            index = MAIN_SRC_MAP[msg["src"]]
+            # schema = mc_s.schema_main
+        elif msg["type"] in TYPES_META:
+            index = "meta"
+            # schema = mc_s.schema_meta
+        elif msg["type"] in TYPES_INT:
+            index = "internal"
+            # schema = mc_s.schema_int
+
+        KAFKA_TOPIC = index
+
+        # if key in schema:
+        #     if isinstance(value, int):
+        #         if schema[key].startswith("string") or schema[key].startswith(
+        #             "text"
+        #         ):
+        #             msg[key] = str(value)
+        body = orjson.dumps(msg)
+        if "ts" not in msg:
+            raise Exception("No TS in msg")
+        if KAFKA_TOPIC not in topicmap:
+            topicmap[KAFKA_TOPIC] = [body]
+        else:
+            topicmap[KAFKA_TOPIC].append(body)
+
+    for topic, messages in topicmap.items():
+        batch = producer.create_batch()
+        for body in messages:
+            metadata = batch.append(key=None, value=body, timestamp=msg["ts"])
+            if metadata is None:
+                partitions = await producer.partitions_for(topic)
+                partition = random.choice(tuple(partitions))
+                await producer.send_batch(batch, topic, partition=partition)
+                # log.debug(
+                #     (
+                #         f"{batch.record_count()} messages sent to topic "
+                #         f"{topic} partition {partition}"
+                #     )
+                # )
+                batch = producer.create_batch()
+                continue
+
+        partitions = await producer.partitions_for(topic)
+        partition = random.choice(tuple(partitions))
+        await producer.send_batch(batch, topic, partition=partition)
+        # log.debug(
+        #     (
+        #         f"{batch.record_count()} messages sent to topic "
+        #         f"{topic} partition {partition}"
+        #     )
+        # )
+    log.debug(
+        "Kafka batches sent: "
+        + ", ".join([topic + ": " + str(len(topicmap[topic])) for topic in topicmap])
+    )
+    await producer.stop()
+
+
+async def queue_message(msg):
+    """
+    Queue a message on the Redis buffer.
+    """
+    message = orjson.dumps(msg)
+    await ar.zadd(KEYNAME, message)
+
+
+async def queue_message_bulk(data):
+    """
+    Queue multiple messages on the Redis buffer.
+    """
+    for msg in data:
+        message = orjson.dumps(msg)
+        await ar.zadd(KEYNAME, message)

docker-compose.yml

@@ -0,0 +1,379 @@
+version: "2.2"
+
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis_superset
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
+  - superset_home:/app/superset_home
+
+services:
+  app:
+    image: pathogen/monolith:latest
+    container_name: monolith
+    build: ./docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+    env_file:
+      - .env
+    volumes_from:
+      - tmp
+    depends_on:
+      druid:
+        condition: service_started
+      kafka:
+        condition: service_healthy
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
+    #   - db
+
+  threshold:
+    image: pathogen/threshold:latest
+    container_name: threshold
+    build: ./legacy/docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      - ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
+    ports:
+      - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
+      - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
+      - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
+    env_file:
+      - .env
+    # for development
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes_from:
+        - tmp
+    depends_on:
+      tmp:
+        condition: service_started
+      redis:
+        condition: service_healthy
+
+  # db:
+    #image: pathogen/manticore:kibana
+    # image: manticoresearch/manticore:latest
+    #build: 
+    #  context: ./docker/manticore
+    #  args:
+    #    DEV: 1
+    # restart: always
+
+  # turnilo:
+  #   container_name: turnilo
+  #   image: uchhatre/turnilo:latest
+  #   ports:
+  #     - 9093:9090
+  #   environment:
+  #     - DRUID_BROKER_URL=http://broker:8082
+  #     - CONFIG_FILE=/config.yaml
+  #   volumes:
+  #     - ${PORTAINER_GIT_DIR}/docker/turnilo.yaml:/config.yaml
+  #   depends_on:
+  #     - broker
+
+  # metabase:
+  #   container_name: metabase
+  #   image: metabase/metabase:latest
+  #   ports:
+  #     - 3096:3000
+  #   environment:
+  #     JAVA_OPTS: -Xmx1g
+  #     MB_DB_TYPE: postgres
+  #     MB_DB_DBNAME: metabase
+  #     MB_DB_PORT: 5432
+  #     MB_DB_USER: druid
+  #     MB_DB_PASS: FoolishPassword
+  #     MB_DB_HOST: postgres
+  #   depends_on:
+  #     - broker
+
+  redis_superset:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis_superset:/data
+  db:
+    env_file: docker/.env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: docker/.env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: docker/.env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: docker/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: docker/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - metadata_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: FoolishPassword
+      POSTGRES_USER: druid
+      POSTGRES_DB: druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOO_MY_ID=1
+
+  kafka:
+    image: wurstmeister/kafka:latest
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_MESSAGE_MAX_BYTES: 2000000
+      #KAFKA_HEAP_OPTS: -Xmx2g
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
+      start_period: 15s
+      interval: 30s
+      timeout: 30s
+      retries: 45
+
+  druid:
+    image: pathogen/druid:0.23.0
+    build: ./docker/druid/
+    container_name: druid
+    volumes:
+      - druid_shared:/opt/shared
+      - druid_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+      - "8082:8082"
+      - "8083:8083"
+      - "8888:8888"
+    env_file:
+      - environment
+
+  # coordinator:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: coordinator
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - coordinator_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #   ports:
+  #     - "8081:8081"
+  #   command:
+  #     - coordinator
+  #   env_file:
+  #     - environment
+
+  # broker:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: broker
+  #   volumes:
+  #     - broker_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8082:8082"
+  #   command:
+  #     - broker
+  #   env_file:
+  #     - environment
+
+  # historical:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: historical
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - historical_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8083:8083"
+  #   command:
+  #     - historical
+  #   env_file:
+  #     - environment
+
+  # middlemanager:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: middlemanager
+  #   volumes:
+  #     - druid_shared:/opt/shared
+  #     - middle_var:/opt/druid/var
+  #   depends_on: 
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8091:8091"
+  #     - "8100-8105:8100-8105"
+  #   command:
+  #     - middleManager
+  #   env_file:
+  #     - environment
+
+  # router:
+  #   #image: apache/druid:0.23.0
+  #   image: pathogen/druid:0.23.0
+  #   build: ./docker/druid/
+  #   container_name: router
+  #   volumes:
+  #     - router_var:/opt/druid/var
+  #   depends_on:
+  #     - zookeeper
+  #     - postgres
+  #     - coordinator
+  #   ports:
+  #     - "8888:8888"
+  #   command:
+  #     - router
+  #   env_file:
+  #     - environment
+
+  # db:
+  #   #image: pathogen/manticore:kibana
+  #   image: manticoresearch/manticore:dev
+  #   #build: 
+  #   #  context: ./docker/manticore
+  #   #  args:
+  #   #    DEV: 1
+  #   restart: always
+  #   ports:
+  #     - 9308
+  #     - 9312
+  #     - 9306
+  #   ulimits:
+  #     nproc: 65535
+  #     nofile:
+  #        soft: 65535
+  #        hard: 65535
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   environment:
+  #     - MCL=1
+  #   volumes:
+  #     - ./docker/data:/var/lib/manticore
+  #     - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+        - /var/run/redis
+
+  redis:
+    image: redis
+    command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - redis_data:/data
+    volumes_from:
+      - tmp
+    healthcheck:
+      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
+
+networks:
+   default:
+      external:
+         name: pathogen
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis_superset:
+    external: false
+  redis_data: {}
+  metadata_data: {}
+  # middle_var: {}
+  # historical_var: {}
+  # broker_var: {}
+  # coordinator_var: {}
+  druid_var: {}
+  druid_shared: {}

docker/.env-non-dev

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+COMPOSE_PROJECT_NAME=superset
+
+# database configurations (do not modify)
+DATABASE_DB=superset
+DATABASE_HOST=db
+DATABASE_PASSWORD=superset
+DATABASE_USER=superset
+
+# database engine specific environment variables
+# change the below if you prefers another database engine
+DATABASE_PORT=5432
+DATABASE_DIALECT=postgresql
+POSTGRES_DB=superset
+POSTGRES_USER=superset
+POSTGRES_PASSWORD=superset
+#MYSQL_DATABASE=superset
+#MYSQL_USER=superset
+#MYSQL_PASSWORD=superset
+#MYSQL_RANDOM_ROOT_PASSWORD=yes
+
+# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
+PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+FLASK_ENV=production
+SUPERSET_ENV=production
+SUPERSET_LOAD_EXAMPLES=no
+CYPRESS_CONFIG=false
+SUPERSET_PORT=8088

docker/Dockerfile

@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+
+RUN useradd -d /code pathogen
+RUN mkdir /code
+RUN chown pathogen:pathogen /code
+
+RUN mkdir /venv
+RUN chown pathogen:pathogen /venv
+
+USER pathogen
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY requirements.txt /code/
+COPY discord-patched.tgz /code/
+
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r requirements.txt
+# && python -m spacy download en_core_web_sm
+
+RUN tar xf /code/discord-patched.tgz -C /venv/lib/python3.10/site-packages
+
+CMD . /venv/bin/activate && exec python monolith.py

docker/docker-compose.infra.yml

@@ -0,0 +1,142 @@
+version: "2.2"
+
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis_superset
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ${PORTAINER_GIT_DIR}/docker/superset:/app/docker
+  - superset_home:/app/superset_home
+
+services:
+  redis_superset:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis:/data
+  db:
+    env_file: .env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: .env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: .env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: .env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - /block/store/metadata_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOO_MY_ID=1
+
+  kafka:
+    image: wurstmeister/kafka:latest
+    container_name: kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_MESSAGE_MAX_BYTES: 2000000
+      #KAFKA_HEAP_OPTS: -Xmx2g
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
+      start_period: 15s
+      interval: 30s
+      timeout: 30s
+      retries: 45
+
+  druid:
+    image: pathogen/druid:0.23.0
+    build: ./druid/
+    container_name: druid
+    volumes:
+      - /block/store/druid_shared:/opt/shared
+      - /block/store/druid_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+      - "8082:8082"
+      - "8083:8083"
+      - "8888:8888"
+    env_file:
+      - environment
+
+networks:
+   default:
+      external:
+         name: pathogen
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker/docker-compose.prod.yml

@@ -0,0 +1,59 @@
+version: "2.2"
+
+services:
+  app:
+    image: pathogen/monolith:latest
+    container_name: monolith
+    build: ${PORTAINER_GIT_DIR}/docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+    env_file:
+      - ../stack.env
+    # volumes_from:
+    #   - tmp
+    # depends_on:
+      # broker:
+      #   condition: service_started
+      # kafka:
+      #   condition: service_healthy
+      # tmp:
+      #   condition: service_started
+      # redis:
+      #   condition: service_healthy
+
+  threshold:
+    image: pathogen/threshold:latest
+    container_name: threshold
+    build: ./legacy/docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      - ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
+    ports:
+      - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
+      - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
+      - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
+    env_file:
+      - ../stack.env
+    # for development
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  ssdb:
+    image: tsl0922/ssdb
+    container_name: ssdb_monolith
+    volumes:
+      - ssdb_data:/ssdb/var
+    ports:
+      - "1289:1289"
+    environment:
+      - SSDB_PORT=1289
+
+networks:
+   default:
+      external:
+         name: pathogen
+
+volumes:
+  ssdb_data: {}

docker/druid/Dockerfile

@@ -0,0 +1,22 @@
+ARG DRUID_VER=0.23.0
+
+FROM apache/druid:${DRUID_VER} AS druid
+
+FROM ubuntu:bionic
+RUN apt-get update && \
+    apt-get install --yes openjdk-8-jre-headless perl-modules && \
+    apt-get clean
+
+RUN addgroup --system -gid 1000 druid \
+ && adduser --system --uid 1000 --disabled-password --home /opt/druid --shell /bin/bash --group druid
+
+COPY --from=druid --chown=druid:druid /opt/druid /opt/druid
+
+WORKDIR /opt/druid
+
+USER druid
+
+EXPOSE 8888/tcp
+EXPOSE 8081/tcp
+
+CMD /opt/druid/bin/start-nano-quickstart

docker/environment

@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=1g
+
+#druid_emitter_logging_logLevel=debug
+
+#druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+#druid_zk_service_host=zookeeper
+
+#druid_metadata_storage_host=
+#druid_metadata_storage_type=postgresql
+#druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+#druid_metadata_storage_connector_user=druid
+#druid_metadata_storage_connector_password=FoolishPassword
+
+#druid_coordinator_balancer_strategy=cachingCost
+
+#druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+#druid_indexer_fork_property_druid_processing_buffer_sizeBytes=128MiB
+#druid_processing_buffer_sizeBytes=268435456 # 256MiB
+
+#druid_storage_type=local
+#druid_storage_storageDirectory=/opt/shared/segments
+#druid_indexer_logs_type=file
+#druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+#druid_processing_numThreads=1
+#druid_processing_numMergeBuffers=1
+
+#DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=6172m
+DRUID_SINGLE_NODE_CONF=nano-quickstart
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/shared/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

docker/manticore.conf

@@ -0,0 +1,265 @@
+#!/bin/sh
+ip=`hostname -i|rev|cut -d\  -f 1|rev`
+cat << EOF
+searchd {
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
+    # access_plain_attrs = mmap_preread
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
+    # access_blob_attrs = mmap_preread
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
+    # access_doclists = file
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
+    # access_hitlists = file
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
+    # agent_connect_timeout =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
+    # agent_query_timeout =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
+    # agent_retry_count = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
+    # agent_retry_delay = 500
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
+    # attr_flush_period = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
+    # binlog_flush = 2
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
+    # binlog_max_log_size = 268435456
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
+    # binlog_path =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
+    # client_timeout = 300
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
+    # collation_libc_locale = C
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
+    # collation_server = libc_ci
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
+    data_dir = /var/lib/manticore
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
+    # docstore_cache_size = 16m
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
+    # expansion_limit = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
+    # grouping_in_utc = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
+    # ha_period_karma = 60
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
+    # ha_ping_interval = 1000
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
+    # hostname_lookup =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
+    # jobs_queue_size =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
+    # listen_backlog = 5
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#listen
+    # listen_env = this directive allows to append listeners from environment variables
+
+    listen = 9306:mysql41
+    listen = /var/run/mysqld/mysqld.sock:mysql41
+    listen = $ip:9312
+    listen = 9308:http
+    listen = $ip:9315-9325:replication
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
+    # listen_tfo = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#log
+    log = /var/log/manticore/searchd.log
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
+    # max_batch_queries = 32
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#threads
+    # threads =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
+    # max_filters = 256
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
+    # max_filter_values = 4096
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
+    # max_open_files = max
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
+    max_packet_size = 128M
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
+    # mysql_version_string =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
+    # net_workers = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
+    # net_wait_tm = -1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
+    # net_throttle_accept = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
+    # net_throttle_action = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#node_address
+    # node_address =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
+    # ondisk_attrs_default = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
+    # persistent_connections_limit =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
+    pid_file = /var/run/manticore/searchd.pid
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
+    # predicted_time_costs = doc=64, hit=48, skip=2048, match=64
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
+    # preopen_indexes = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
+    qcache_max_bytes = 128Mb
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
+    qcache_thresh_msec = 150
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
+    qcache_ttl_sec = 120
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
+    query_log_format = sphinxql
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
+    # query_log_min_msec = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log
+    # query_log = /var/log/manticore/query.log
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
+    # query_log_mode = 600
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
+    # max_connections =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
+    # network_timeout = 5
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
+    # read_buffer = 256K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
+    # read_buffer_docs = 256K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
+    # read_buffer_hits = 256K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
+    # read_unhinted 32K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
+    # rt_flush_period =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
+    # rt_merge_iops = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
+    # rt_merge_maxiosize = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
+    # seamless_rotate = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#server_id
+    # server_id =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
+    # shutdown_timeout = 3
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
+    # shutdown_token =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
+    # snippets_file_prefix =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
+    # sphinxql_state =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
+    # sphinxql_timeout = 900
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
+    # ssl_ca =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
+    # ssl_cert =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
+    # ssl_key =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
+    # subtree_docs_cache = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
+    # subtree_hits_cache = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
+    # thread_stack =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
+    # unlink_old = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
+    # watchdog = 1
+}
+
+common {
+
+    # https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
+    # lemmatizer_base = /usr/local/share
+
+    # https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
+    # progressive_merge =
+
+    # https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
+    # json_autoconv_keynames =
+
+    # https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
+    # json_autoconv_numbers = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
+    # on_json_attr_error = ignore_attr
+
+    # https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
+    # plugin_dir =
+}
+
+# indexer {
+#     lemmatizer_cache = 1024M
+#     max_iops = 0
+#     max_iosize = 0
+#     mem_limit = 1024M
+# }
+
+EOF

docker/manticore/.mysql_history

@@ -0,0 +1,12 @@
+_HiStOrY_V2_
+SELECT * FROM films WHERE MATCH('"shark monkey boy robot"/2') AND release_year IN(2006,2007) AND rental_rate BETWEEN 2.0 and 3.0;
+SELECT title, HIGHLIGHT({},'description') FROM films WHERE MATCH('"shark monkey boy robot"/2');
+SELECT * FROM films WHERE MATCH('" shark  monkey  boy robot "/2');
+SELECT * FROM films WHERE MATCH('Emotional drama') FACET release_year FACET category_id;
+SELECT * FROM films WHERE MATCH('Emotional drama') GROUP BY release_year;
+SELECT * FROM films WHERE MATCH('Emotional drama -dog -shark');
+SELECT * FROM films WHERE MATCH('Emotional drama');
+SELECT * FROM films;
+DESCRIBE films;
+SHOW TABLES;
+SOURCE /sandbox.sql

docker/manticore/Dockerfile

@@ -0,0 +1,76 @@
+FROM ubuntu:focal
+
+ARG DEV
+ARG DAEMON_URL
+ARG MCL_URL
+
+RUN groupadd -r manticore && useradd -r -g manticore manticore
+
+ENV GOSU_VERSION 1.11
+ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
+ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
+ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"}
+
+
+RUN set -x \
+    && apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \
+    && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
+    && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
+    && export GNUPGHOME="$(mktemp -d)" \
+    && gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+    && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+    && { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
+    && rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc \
+    && chmod +x /usr/local/bin/gosu \
+    && gosu nobody true && \
+    if [ "${DEV}" = "1" ]; then \
+    echo "DEV IS ONE" && \
+     exit && \
+     wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
+     && dpkg -i manticore-dev-repo.noarch.deb \
+     && apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
+     && apt-get update  \
+     && echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
+    elif [ "${DEV}" = "2" ]; then \
+      echo "DEV IS TWO" && \
+      wget $BETA_URL && unzip focal.zip && rm focal.zip && \
+      dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\
+    else \
+      echo "DEV NOT EITHER" && \
+      exit && \
+      wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
+      dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
+    fi \
+    && mkdir -p /var/run/manticore && mkdir -p /var/lib/manticore/replication \
+    && apt-get update && apt-get -y install  libexpat1 libodbc1 libpq5 openssl libcrypto++6 libmysqlclient21 mysql-client \
+    && apt-get -y purge --auto-remove \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /usr/bin/mariabackup /usr/bin/mysqldump /usr/bin/mysqlslap /usr/bin/mysqladmin /usr/bin/mysqlimport  \
+    /usr/bin/mysqlshow /usr/bin/mbstream /usr/bin/mysql_waitpid /usr/bin/innotop /usr/bin/mysqlaccess /usr/bin/mytop  \
+    /usr/bin/mysqlreport /usr/bin/mysqldumpslow /usr/bin/mysql_find_rows /usr/bin/mysql_fix_extensions  \
+    /usr/bin/mysql_embedded /usr/bin/mysqlcheck \
+    && rm -f /usr/bin/spelldump /usr/bin/wordbreaker \
+    && mkdir -p /var/run/mysqld/ && chown manticore:manticore /var/run/mysqld/ \
+    && echo "\n[mysql]\nsilent\nwait\ntable\n" >> /etc/mysql/my.cnf && \
+    wget -P /tmp https://repo.manticoresearch.com/repository/morphology/en.pak.tgz && \
+    wget -P /tmp https://repo.manticoresearch.com/repository/morphology/de.pak.tgz && \
+    wget -P /tmp https://repo.manticoresearch.com/repository/morphology/ru.pak.tgz && \
+    tar -xf /tmp/en.pak.tgz -C /usr/share/manticore/ &&  \
+    tar -xf /tmp/de.pak.tgz -C /usr/share/manticore/ &&  \
+    tar -xf /tmp/ru.pak.tgz -C /usr/share/manticore/
+
+
+COPY manticore.conf /etc/manticoresearch/
+COPY sandbox.sql /sandbox.sql
+COPY .mysql_history /root/.mysql_history
+
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh
+WORKDIR /var/lib/manticore
+ENTRYPOINT ["docker-entrypoint.sh"]
+EXPOSE 9306
+EXPOSE 9308
+EXPOSE 9312
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"]

docker/manticore/README.md

@@ -0,0 +1,278 @@
+# Manticore Search Docker image
+
+This is the git repo of official [Docker image](https://hub.docker.com/r/manticoresearch/manticore/) for [Manticore Search](https://github.com/manticoresoftware/manticoresearch).
+
+Manticore Search is an easy to use open source fast database for search. It helps thousands of companies from small to large, such as Craigslist, to search and filter petabytes of text data on a single or hundreds of nodes, do stream full-text filtering, add auto-complete, spell correction, more-like-this, faceting and other search-related technologies to their websites and applications.
+
+The default configuration includes a sample Real-Time index and listens on the default ports:
+  * `9306` for connections from a MySQL client
+  * `9308` for connections via HTTP
+  * `9312` for connections via a binary protocol (e.g. in case you run a cluster)
+
+The image comes with libraries for easy indexing data from MySQL, PostgreSQL XML and CSV files.
+
+# How to run Manticore Search Docker image
+
+## Quick usage
+
+The below is the simplest way to start Manticore in a container and log in to it via mysql client:
+
+```bash
+docker run --name manticore --rm -d manticoresearch/manticore && sleep 3 && docker exec -it manticore mysql && docker stop manticore
+```
+
+When you exit from the mysql client it stops and removes the container, so **use it only for testing / sandboxing purposes**. See below how to use it in production.
+
+The image comes with a sample index which can be loaded like this:
+
+```mysql
+mysql> source /sandbox.sql
+```
+
+Also the mysql client has in history several sample queries that you can run on the above index, just use Up/Down keys in the client to see and run them.
+
+## Production use
+
+
+### Ports and mounting points
+
+For data persistence `/var/lib/manticore/` should be mounted to local storage or other desired storage engine.
+
+```bash
+docker run --name manticore -v $(pwd)/data:/var/lib/manticore -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
+```
+
+Configuration file inside the instance is located at `/etc/manticoresearch/manticore.conf`. For custom settings, this file should be mounted to your own configuration file.
+
+The ports are 9306/9308/9312 for SQL/HTTP/Binary, expose them depending on how you are going to use Manticore. For example:
+
+```bash
+docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data:/var/lib/manticore/ -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
+```
+
+Make sure to remove `127.0.0.1:` if you want the ports to be available for external hosts.
+
+### Manticore Columnar Library
+
+The docker image doesn't include [Manticore Columnar Library](https://github.com/manticoresoftware/columnar) which has to be used if you need:
+* columnar storage
+* secondary indexes
+
+but you can easily enable it in runtime by using environment variable `MCL=1`, i.e. `docker run -e MCL=1 ... manticoresearch/manticore`. It will then download and install the library and put it to the data dir (which is normally mapped as a volume in production). Next time you run the container the library will be already there, hence it won't be downloaded again unless you change the Manticore Search version.
+
+### Docker-compose
+
+In many cases you might want to use Manticore together with other images specified in a docker-compose YAML file. Here is the minimal recommended specification for Manticore Search in docker-compose.yml:
+
+```yaml
+version: '2.2'
+
+services:
+  manticore:
+    container_name: manticore
+    image: manticoresearch/manticore
+    restart: always
+    ports:
+      - 127.0.0.1:9306:9306
+      - 127.0.0.1:9308:9308
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      - MCL=1
+    volumes:
+      - ./data:/var/lib/manticore
+#      - ./manticore.conf:/etc/manticoresearch/manticore.conf # uncommment if you use a custom config
+```
+
+Besides using the exposed ports 9306 and 9308 you can log into the instance by running `docker-compose exec manticore mysql`.
+
+### HTTP protocol
+
+Manticore is accessible via HTTP on ports 9308 and 9312. You can map either of them locally and connect with curl:
+
+```bash
+docker run --name manticore -p 9308:9308 -d manticoresearch/manticore
+```
+
+Create a table:
+```bash
+curl -X POST 'http://127.0.0.1:9308/sql' -d 'mode=raw&query=CREATE TABLE testrt ( title text, content text, gid integer)'
+```
+Insert a document:
+
+```bash
+curl -X POST 'http://127.0.0.1:9308/json/insert' -d'{"index":"testrt","id":1,"doc":{"title":"Hello","content":"world","gid":1}}'
+```
+
+Perform a simple search:
+
+```bash
+curl -X POST 'http://127.0.0.1:9308/json/search' -d '{"index":"testrt","query":{"match":{"*":"hello world"}}}'
+```
+
+### Logging
+
+By default, Manticore logs to `/dev/stdout`, so you can watch the log on the host with:
+
+```bash
+docker logs manticore
+```
+
+If you want to get log of your queries the same way you can do it by passing environment variable `QUERY_LOG_TO_STDOUT=true`.
+
+### Multi-node cluster with replication
+
+Here is a simple `docker-compose.yml` for defining a two node cluster:
+
+```yaml
+version: '2.2'
+
+services:
+
+  manticore-1:
+    image: manticoresearch/manticore
+    restart: always
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      - MCL=1
+    networks:
+      - manticore
+  manticore-2:
+    image: manticoresearch/manticore
+    restart: always
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      - MCL=1
+    networks:
+      - manticore
+networks:
+  manticore:
+    driver: bridge
+```
+* Start it: `docker-compose up`
+* Create a cluster with a table:
+  ```mysql
+  $ docker-compose exec manticore-1 mysql
+
+  mysql> CREATE TABLE testrt ( title text, content text, gid integer);
+
+  mysql> CREATE CLUSTER posts;
+  Query OK, 0 rows affected (0.24 sec)
+
+  mysql> ALTER CLUSTER posts ADD testrt;
+  Query OK, 0 rows affected (0.07 sec)
+
+  MySQL [(none)]> exit
+  Bye
+  ```
+* Join to the the cluster on the 2nd instance and insert smth to the table:
+  ```mysql
+  $ docker-compose exec manticore-2 mysql
+
+  mysql> JOIN CLUSTER posts AT 'manticore-1:9312';
+  mysql> INSERT INTO posts:testrt(title,content,gid)  VALUES('hello','world',1);
+  Query OK, 1 row affected (0.00 sec)
+
+  MySQL [(none)]> exit
+  Bye
+  ```
+
+* If you now go back to the first instance you'll see the new record:
+  ```mysql
+  $ docker-compose exec manticore-1 mysql
+
+  MySQL [(none)]> select * from testrt;
+  +---------------------+------+-------+---------+
+  | id                  | gid  | title | content |
+  +---------------------+------+-------+---------+
+  | 3891565839006040065 |    1 | hello | world   |
+  +---------------------+------+-------+---------+
+  1 row in set (0.00 sec)
+
+  MySQL [(none)]> exit
+  Bye
+  ```
+
+## Memory locking and limits
+
+It's recommended to overwrite the default ulimits of docker for the Manticore instance:
+
+```bash
+ --ulimit nofile=65536:65536
+```
+
+For the best performance, Manticore tables' components can be locked into memory. When Manticore is run under Docker, the instance requires additional privileges to allow memory locking. The following options must be added when running the instance:
+
+```bash
+  --cap-add=IPC_LOCK --ulimit memlock=-1:-1
+```
+
+## Configuring Manticore Search with Docker
+
+If you want to run Manticore with your custom config containing indexes definition you will need to mount the configuration to the instance:
+
+```bash
+docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data/:/var/lib/manticore -p 127.0.0.1:9306:9306 -d manticoresearch/manticore
+```
+
+Take into account that Manticore search inside the container is run under user `manticore`. Performing operations with tables (like creating or rotating plain indexes) should be also done under `manticore`. Otherwise the files will be created under `root` and the search daemon won't have rights to open them. For example here is how you can rotate all plain indexes:
+
+```bash
+docker exec -it manticore gosu manticore indexer --all --rotate
+```
+
+### Environment variables
+
+You can also set individual `searchd` and `common` configuration settings using Docker environment variables.  
+
+The settings must be prefixed with their section name, for example to change value of setting `mysql_version_string` in section `searchd` the variable must be named `searchd_mysql_version_string`:
+
+
+```bash
+docker run --name manticore  -p 127.0.0.1:9306:9306  -e searchd_mysql_version_string='5.5.0' -d manticoresearch/manticore
+```
+
+In case of `listen` directive, you can pass using Docker variable `searchd_listen` new listening interfaces in addition to the default ones. Multiple interfaces can be declared separated by semi-colon ("|").
+For listening only on network address, the `$ip` (retrieved internally from `hostname -i`) can be used as address alias.
+
+For example `-e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip'` will add an additional SQL interface on port 9307, a SQL VIP on 5443 running only on the instance IP  and HTTP on port 9316, beside the defaults on 9306 and 9308, respectively.
+
+```bash
+$ docker run --rm -p 1188:9307  -e searchd_mysql_version_string='5.5.0' -e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip'  manticore
+[Mon Aug 17 07:31:58.719 2020] [1] using config file '/etc/manticoresearch/manticore.conf' (9130 chars)...
+listening on all interfaces for http, port=9316
+listening on all interfaces for mysql, port=9307
+listening on 172.17.0.17:5443 for VIP mysql
+listening on all interfaces for mysql, port=9306
+listening on UNIX socket /var/run/mysqld/mysqld.sock
+listening on 172.17.0.17:9312 for sphinx
+listening on all interfaces for http, port=9308
+prereading 0 indexes
+prereaded 0 indexes in 0.000 sec
+accepting connections
+```
+
+
+# Issues
+
+For reporting issues, please use the [issue tracker](https://github.com/manticoresoftware/docker/issues).

docker/manticore/docker-entrypoint.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+set -eo pipefail
+echo "RUNNING ENTRYPOINT"
+
+# check to see if this file is being run or sourced from another script
+_is_sourced() {
+  # https://unix.stackexchange.com/a/215279
+  [ "${#FUNCNAME[@]}" -ge 2 ] &&
+    [ "${FUNCNAME[0]}" = '_is_sourced' ] &&
+    [ "${FUNCNAME[1]}" = 'source' ]
+}
+_searchd_want_help() {
+  local arg
+  for arg; do
+    case "$arg" in
+    -'?' | --help | -h | -v)
+      return 0
+      ;;
+    esac
+  done
+  return 1
+}
+
+docker_setup_env() {
+  if [ -n "$QUERY_LOG_TO_STDOUT" ]; then
+    export searchd_query_log=/var/log/manticore/query.log
+    [ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
+  fi
+  if [[ "${MCL}" == "1" ]]; then
+      LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
+      LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
+
+      [ -L /usr/share/manticore/modules/lib_manticore_columnar.so ] || ln -s $LIB_MANTICORE_COLUMNAR /usr/share/manticore/modules/lib_manticore_columnar.so
+      [ -L /usr/share/manticore/modules/lib_manticore_secondary.so ] || ln -s $LIB_MANTICORE_SECONDARY /usr/share/manticore/modules/lib_manticore_secondary.so
+
+      searchd -v|grep -i error|egrep "trying to load" \
+      && rm $LIB_MANTICORE_COLUMNAR $LIB_MANTICORE_SECONDARY \
+      && echo "WARNING: wrong MCL version was removed, installing the correct one"
+
+      if [[ ! -f "$LIB_MANTICORE_COLUMNAR" || ! -f "$LIB_MANTICORE_SECONDARY" ]]; then
+        if ! mkdir -p /var/lib/manticore/.mcl/ ; then
+          echo "ERROR: Manticore Columnar Library is inaccessible: couldn't create /var/lib/manticore/.mcl/."
+          exit
+        fi
+
+        MCL_URL=$(cat /mcl.url)
+        wget -P /tmp $MCL_URL
+
+        LAST_PATH=$(pwd)
+        cd /tmp
+        PACKAGE_NAME=$(ls | grep manticore-columnar | head -n 1)
+        ar -x $PACKAGE_NAME
+        tar -xf data.tar.gz
+        find . -name '*.so' -exec cp {} /var/lib/manticore/.mcl/ \;
+        cd $LAST_PATH
+      fi
+  fi
+}
+_main() {
+  # first arg is `h` or some `--option`
+  if [ "${1#-}" != "$1" ]; then
+    set -- searchd "$@"
+  fi
+  # Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options
+  if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then
+    docker_setup_env "$@"
+    # allow the container to be started with `--user`
+    if [ "$(id -u)" = '0' ]; then
+      find /var/lib/manticore /var/log/manticore /var/run/manticore /etc/manticoresearch \! -user manticore -exec chown manticore '{}' +
+      exec gosu manticore "$0" "$@"
+    fi
+  fi
+  _replace_conf_from_env
+  exec "$@"
+}
+
+_replace_conf_from_env() {
+
+  sed_query=""
+
+  while IFS='=' read -r oldname value; do
+    if [[ $oldname == 'searchd_'* || $oldname == 'common_'* ]]; then
+      value=$(echo ${!oldname} | sed 's/\//\\\//g')
+      oldname=$(echo $oldname | sed "s/searchd_//g;s/common_//g;")
+      newname=$oldname
+
+      if [[ $newname == 'listen' ]]; then
+        oldname="listen_env"
+        IFS='|' read -ra ADDR <<<"$value"
+        count=0
+
+        for i in "${ADDR[@]}"; do
+          if [[ $count == 0 ]]; then
+            value=$i
+          else
+            value="$value\n    listen = $i"
+          fi
+          count=$((count + 1))
+        done
+      fi
+
+      if [[ -z $sed_query ]]; then
+        sed_query="s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
+      else
+        sed_query="$sed_query;s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
+      fi
+
+    fi
+  done < <(env)
+
+  if [[ ! -z $sed_query ]]; then
+    sed -i -E "$sed_query" /etc/manticoresearch/manticore.conf
+  fi
+}
+# If we are sourced from elsewhere, don't perform any further actions
+if ! _is_sourced; then
+  _main "$@"
+fi

docker/manticore/manticore.conf

@@ -0,0 +1,259 @@
+#!/bin/sh
+ip=`hostname -i|rev|cut -d\  -f 1|rev`
+cat << EOF
+searchd {
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
+    # access_plain_attrs = mmap_preread
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
+    # access_blob_attrs = mmap_preread
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
+    # access_doclists = file
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
+    # access_hitlists = file
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
+    # agent_connect_timeout =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
+    # agent_query_timeout =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
+    # agent_retry_count = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
+    # agent_retry_delay = 500
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
+    # attr_flush_period = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
+    # binlog_flush = 2
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
+    # binlog_max_log_size = 268435456
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
+    # binlog_path =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
+    # client_timeout = 300
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
+    # collation_libc_locale = C
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
+    # collation_server = libc_ci
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
+    data_dir = /var/lib/manticore
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
+    # docstore_cache_size = 16m
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
+    # expansion_limit = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
+    # grouping_in_utc = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
+    # ha_period_karma = 60
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
+    # ha_ping_interval = 1000
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
+    # hostname_lookup =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
+    # jobs_queue_size =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
+    # listen_backlog = 5
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#listen
+    # listen_env = this directive allows to append listeners from environment variables
+
+    listen = 9306:mysql41
+    listen = /var/run/mysqld/mysqld.sock:mysql41
+    listen = $ip:9312
+    listen = 9308:http
+    listen = $ip:9315-9325:replication
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
+    # listen_tfo = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#log
+    log = /var/log/manticore/searchd.log
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
+    # max_batch_queries = 32
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#threads
+    # threads =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
+    # max_filters = 256
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
+    # max_filter_values = 4096
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
+    # max_open_files =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
+    max_packet_size = 128M
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
+    # mysql_version_string =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
+    # net_workers = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
+    # net_wait_tm = -1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
+    # net_throttle_accept = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
+    # net_throttle_action = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#node_address
+    # node_address =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
+    # ondisk_attrs_default = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
+    # persistent_connections_limit =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
+    pid_file = /var/run/manticore/searchd.pid
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
+    # predicted_time_costs = doc=64, hit=48, skip=2048, match=64
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
+    # preopen_indexes = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
+    # qcache_max_bytes = 16Mb
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
+    # qcache_thresh_msec = 3000
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
+    # qcache_ttl_sec = 60
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
+    query_log_format = sphinxql
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
+    query_log_min_msec = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log
+    query_log = /var/log/manticore/query.log
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
+    # query_log_mode = 600
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
+    # max_connections =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
+    # network_timeout = 5
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
+    # read_buffer = 256K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
+    # read_buffer_docs = 256K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
+    # read_buffer_hits = 256K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
+    # read_unhinted 32K
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
+    # rt_flush_period =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
+    # rt_merge_iops = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
+    # rt_merge_maxiosize = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
+    # seamless_rotate = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#server_id
+    # server_id =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
+    # shutdown_timeout = 3
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
+    # shutdown_token =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
+    # snippets_file_prefix =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
+    # sphinxql_state =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
+    # sphinxql_timeout = 900
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
+    # ssl_ca =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
+    # ssl_cert =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
+    # ssl_key =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
+    # subtree_docs_cache = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
+    # subtree_hits_cache = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
+    # thread_stack =
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
+    # unlink_old = 1
+
+    # https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
+    # watchdog = 1
+}
+
+common {
+
+    # https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
+    # lemmatizer_base = /usr/local/share
+
+    # https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
+    # progressive_merge =
+
+    # https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
+    # json_autoconv_keynames =
+
+    # https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
+    # json_autoconv_numbers = 0
+
+    # https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
+    # on_json_attr_error = ignore_attr
+
+    # https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
+    # plugin_dir =
+
+}
+
+EOF

docker/redis.conf

@@ -0,0 +1,2 @@
+unixsocket /var/run/redis/redis.sock
+unixsocketperm 777

docker/requirements.txt

@@ -0,0 +1,23 @@
+wheel
+beautifulsoup4
+redis
+siphashc
+aiohttp[speedups]
+python-dotenv
+#manticoresearch
+numpy
+aioredis[hiredis]
+aiokafka
+vaderSentiment
+polyglot
+pyicu
+pycld2
+morfessor
+six
+nltk
+#spacy
+gensim
+python-Levenshtein
+orjson
+uvloop
+numba

docker/superset/docker-bootstrap.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+fi
+
+if [[ "${1}" == "worker" ]]; then
+  echo "Starting Celery worker..."
+  celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO
+elif [[ "${1}" == "beat" ]]; then
+  echo "Starting Celery beat..."
+  celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+elif [[ "${1}" == "app" ]]; then
+  echo "Starting web app..."
+  flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+elif [[ "${1}" == "app-gunicorn" ]]; then
+  echo "Starting web app..."
+  /usr/bin/run-server.sh
+fi

docker/superset/docker-init.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+#
+# Always install local overrides first
+#
+/app/docker/docker-bootstrap.sh
+
+STEP_CNT=4
+
+echo_step() {
+cat <<EOF
+
+######################################################################
+
+
+Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
+
+
+######################################################################
+
+EOF
+}
+ADMIN_PASSWORD="admin"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    ADMIN_PASSWORD="general"
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+# Initialize the database
+echo_step "1" "Starting" "Applying DB migrations"
+superset db upgrade
+echo_step "1" "Complete" "Applying DB migrations"
+
+# Create an admin user
+echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
+superset fab create-admin \
+              --username admin \
+              --firstname Superset \
+              --lastname Admin \
+              --email admin@superset.com \
+              --password $ADMIN_PASSWORD
+echo_step "2" "Complete" "Setting up admin user"
+# Create default roles and permissions
+echo_step "3" "Starting" "Setting up roles and perms"
+superset init
+echo_step "3" "Complete" "Setting up roles and perms"
+
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+    # Load some data to play with
+    echo_step "4" "Starting" "Loading examples"
+    # If Cypress run which consumes superset_test_config – load required data for tests
+    if [ "$CYPRESS_CONFIG" == "true" ]; then
+        superset load_test_users
+        superset load_examples --load-test-data
+    else
+        superset load_examples
+    fi
+    echo_step "4" "Complete" "Loading examples"
+fi

docker/superset/requirements-local.txt

@@ -0,0 +1 @@
+pydruid

docker/turnilo.yaml

@@ -0,0 +1,3 @@
+clusters:
+  - name: druid
+    guardDataCubes: true

env.example

@@ -0,0 +1,41 @@
+PORTAINER_GIT_DIR=.
+MODULES_ENABLED="dis"
+DISCORD_TOKEN="xx"
+THRESHOLD_LISTENER_HOST=0.0.0.0
+THRESHOLD_LISTENER_PORT=13867
+THRESHOLD_LISTENER_SSL=1
+
+THRESHOLD_RELAY_ENABLED=0
+THRESHOLD_RELAY_HOST=0.0.0.0
+THRESHOLD_RELAY_PORT=13868
+THRESHOLD_RELAY_SSL=1
+
+THRESHOLD_API_ENABLED=1
+THRESHOLD_API_HOST=0.0.0.0
+THRESHOLD_API_PORT=13869
+PORTAINER_GIT_DIR=.
+
+THRESHOLD_CONFIG_DIR=./legacy/conf/live/
+THRESHOLD_CERT_DIR=./legacy/conf/cert/
+
+# How many messages to ingest at once from Redis
+MONOLITH_INGEST_CHUNK_SIZE=900
+
+# Time to wait between polling Redis again
+MONOLITH_INGEST_ITER_DELAY=0.5
+
+# Number of 4chan threads to request at once
+MONOLITH_CH4_THREADS_CONCURRENT=1000
+
+# Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
+MONOLITH_CH4_THREADS_DELAY=0.1
+
+# Time to wait after finishing a crawl before starting again
+MONOLITH_CH4_CRAWL_DELAY=30
+
+# Semaphore value
+MONOLITH_CH4_THREADS_SEMAPHORE=1000
+
+# Threads to use for data processing
+# Leave uncommented to use all available threads
+# MONOLITH_PROCESS_THREADS=4

legacy/.env.example

@@ -0,0 +1,13 @@
+THRESHOLD_LISTENER_HOST=0.0.0.0
+THRESHOLD_LISTENER_PORT=13867
+THRESHOLD_LISTENER_SSL=1
+
+THRESHOLD_RELAY_ENABLED=1
+THRESHOLD_RELAY_HOST=0.0.0.0
+THRESHOLD_RELAY_PORT=13868
+THRESHOLD_RELAY_SSL=1
+
+THRESHOLD_API_ENABLED=1
+THRESHOLD_API_HOST=0.0.0.0
+THRESHOLD_API_PORT=13869
+PORTAINER_GIT_DIR=.

legacy/.gitignore

@@ -0,0 +1,11 @@
+*.pyc
+*.pem
+*.swp
+__pycache__/
+conf/live/
+conf/cert/
+env/
+venv/
+.idea/
+.env
+.bash_history

legacy/.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+    - id: black
+      args:
+        - --line-length=120
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: [--max-line-length=120]

legacy/api/views.py

@@ -0,0 +1,746 @@
+import functools
+from json import JSONDecodeError, dumps, loads
+from string import digits
+
+import main
+from klein import Klein
+from modules import chankeep, helpers, provision, regproc, userinfo
+from modules.network import Network
+from twisted.web.server import Request
+from utils.logging.log import warn
+
+
+def login_required(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if isinstance(args[0], Request):
+            request = args[0]
+            apikey = request.getHeader("ApiKey")
+            token = request.getHeader("Token")
+            if not apikey:
+                return "No API key provided"
+            if not token:
+                return "No token provided"
+            if apikey not in main.tokens:
+                return "No such API key"
+            config_token = main.tokens[apikey]
+            if not token == config_token["hello"]:
+                return "Invalid token"
+            counter = config_token["counter"]
+            request.setHeader("Counter", counter)
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class API(object):
+    """
+    Our API webapp.
+    """
+
+    app = Klein()
+
+    @app.route("/", methods=["GET"])
+    @login_required
+    def hello(self, request):
+        return "Hello"
+
+    @app.route("/who/<query>/", methods=["POST"])
+    @login_required
+    def who(self, request, query):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "query" not in data:
+            return "No query provided"
+        result = userinfo.getWho(data["query"])
+        # Expand the generator
+        return dumps({k: [x for x in v] for k, v in result.items()})
+
+    @app.route("/chans/", methods=["POST"])
+    @login_required
+    def chans(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No query provided"
+        if not data["query"]:
+            warn(f"No query provided: for chans {data}")
+            return dumps({})
+        result = userinfo.getChansSingle(data["net"], data["query"])
+        if not result:
+            return dumps({})
+        return dumps({"chans": [x for x in result]})
+
+    @app.route("/users/", methods=["POST"])
+    @login_required
+    def users(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No query provided"
+        if not data["query"]:
+            warn(f"No query provided for users: {data}")
+            return dumps({})
+        result = userinfo.getUsersSingle(data["net"], data["query"])
+        if not result:
+            return dumps({})
+        return dumps({"users": [x for x in result]})
+
+    @app.route("/online/", methods=["POST"])
+    @login_required
+    def online(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No users provided"
+        if not data["query"]:
+            warn(f"No users provided: for online {data}")
+            return dumps({})
+        net = data["net"]
+        usermap = {}
+        for user in data["query"]:
+            channels = userinfo.getChansSingle(net, [user])
+            if channels:
+                usermap[user] = True
+            else:
+                usermap[user] = False
+
+        return dumps(usermap)
+
+    @app.route("/num_users/", methods=["POST"])
+    @login_required
+    def num_users(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No users provided"
+        if not data["query"]:
+            warn(f"No chans provided: for num_users {data}")
+            return dumps({})
+        net = data["net"]
+        results = userinfo.getUserNum(net, data["query"])
+
+        return dumps(results)
+
+    @app.route("/num_chans/", methods=["POST"])
+    @login_required
+    def num_chans(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No users provided"
+        if not data["query"]:
+            warn(f"No users provided: for num_chans {data}")
+            return dumps({})
+        net = data["net"]
+        results = userinfo.getChanNum(net, data["query"])
+
+        return dumps(results)
+
+    @app.route("/irc/stats/", methods=["POST"])
+    @login_required
+    def irc_stats(self, request):
+        stats = {}
+        numChannels = 0
+        numWhoEntries = 0
+        for i in main.IRCPool.keys():
+            numChannels += len(main.IRCPool[i].channels)
+        numWhoEntries += userinfo.getNumTotalWhoEntries()
+        numRelays = 0
+        for net in main.network.keys():
+            numRelays += len(main.network[net].relays)
+        stats["servers_total_total"] = numRelays
+        stats["servers_total_unique"] = len(main.network.keys())
+        stats["servers_online_total"] = len(main.IRCPool.keys())
+        stats["servers_online_unique"] = len(main.liveNets())
+        stats["channels"] = numChannels
+        stats["records"] = numWhoEntries
+        stats["eventrate"] = main.lastMinuteSample
+        return dumps(stats)
+
+    @app.route("/irc/networks/", methods=["POST"])
+    @login_required
+    def irc_networks(self, request):
+        networks = {}
+        for net in main.network.keys():
+            networks[net] = {
+                "active": chankeep.allRelaysActive(net),
+                "relays": len(main.network[net].relays),
+                "channels": userinfo.getTotalChanNum(net),
+                "records": userinfo.getNumWhoEntries(net),
+            }
+        return dumps(networks)
+
+    @app.route("/irc/auth/", methods=["POST"])
+    @login_required
+    def irc_network_recheckauth(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "func" not in data:
+            return dumps({"success": False, "reason": "no function specified."})
+        func = data["func"]
+
+        if "net" not in data:
+            return dumps({"success": False, "reason": "no net specified."})
+        net = data["net"]
+        if not net or net == "None":
+            nets = main.network.keys()
+        else:
+            if net not in main.network.keys():
+                return dumps({"success": False, "reason": "no such net."})
+            nets = [net]
+        for net_name in nets:
+            conns = helpers.get_connected_relays(net_name)
+            if not conns:
+                return dumps(
+                    {
+                        "success": False,
+                        "reason": f"failed to get instances for {net_name}.",
+                    }
+                )
+            if func == "recheckauth":
+                for conn in conns:
+                    conn.regPing(reset=True)
+            elif func == "resetauth":
+                for conn in conns:
+                    conn.authenticated = False
+                    conn.regPing(reset=True)
+            elif func == "register":
+                for conn in conns:
+                    regproc.registerAccount(conn.net, conn.num)
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/", methods=["POST"])
+    @login_required
+    def irc_network(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        inst = main.network[net]
+        network = {}
+        network["net"] = inst.net
+        network["auth"] = inst.auth
+        network["host"] = inst.host
+        network["last"] = inst.last
+        network["port"] = inst.port
+        network["security"] = inst.security
+        network["relays"] = len(inst.relays)
+        network["chanlimit"] = inst.chanlimit
+        network["channels"] = userinfo.getTotalChanNum(net)
+        network["records"] = userinfo.getNumWhoEntries(net)
+        return dumps(network)
+
+    @app.route("/irc/network/<net>/", methods=["DELETE"])
+    @login_required
+    def irc_network_delete(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        main.network[net].seppuku()
+        del main.network[net]
+        main.saveConf("network")
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/edit/", methods=["POST"])
+    @login_required
+    def irc_network_edit(self, request, net):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        inst = main.network[net]
+        for item in data:
+            if item == "auth":
+                auth = data[item][0]
+                if auth not in ["sasl", "ns", "none"]:
+                    return dumps({"success": False, "reason": "invalid auth."})
+            elif item == "host":
+                host = data[item][0]
+            elif item == "last":
+                last = data[item][0]
+                if not last.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid last: not a number."}
+                    )
+            elif item == "port":
+                port = data[item][0]
+                if not port.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid port: not a number."}
+                    )
+                port = int(port)
+            elif item == "chanlimit":
+                chanlimit = data[item][0]
+                if chanlimit == "None":
+                    chanlimit = None
+                elif not chanlimit.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid chanlimit: not a number."}
+                    )
+                else:
+                    chanlimit = int(chanlimit)
+                    online_relays = helpers.get_connected_relays(net)
+                    for r in online_relays:
+                        r.chanlimit = chanlimit
+            elif item == "security":
+                security = data[item][0]
+                if security not in ["ssl", "plain"]:
+                    return dumps({"success": False, "reason": "invalid security."})
+        inst.auth = auth
+        inst.host = host
+        inst.last = last
+        inst.port = port
+        inst.security = security
+        inst.chanlimit = chanlimit
+        main.saveConf("network")
+
+        return dumps({"success": True})
+
+    @app.route("/irc/network/create/", methods=["PUT"])
+    @login_required
+    def irc_network_create(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        fields = ["net", "auth", "host", "port", "security"]
+        if not set(fields).issubset(set(data)):
+            return dumps({"success": False, "reason": "not enough fields."})
+        for item in data:
+            if item == "net":
+                net = data[item]
+                if net in main.network.keys():
+                    return dumps(
+                        {"success": False, "reason": "network already exists."}
+                    )
+                if set(net).intersection(set(digits)):
+                    return dumps(
+                        {
+                            "success": False,
+                            "reason": "network name cannot contain numbers.",
+                        }
+                    )
+            elif item == "auth":
+                auth = data[item]
+                if auth not in ["sasl", "ns", "none"]:
+                    return dumps({"success": False, "reason": "invalid auth."})
+            elif item == "host":
+                host = data[item]
+            elif item == "port":
+                port = data[item]
+                if not port.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid port: not a number."}
+                    )
+                port = int(port)
+            elif item == "security":
+                security = data[item]
+                if security not in ["ssl", "plain"]:
+                    return dumps({"success": False, "reason": "invalid security."})
+        main.network[net] = Network(net, host, int(port), security, auth)
+        main.saveConf("network")
+
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/relays/", methods=["POST"])
+    @login_required
+    def irc_network_relays(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        relays_inst = main.network[net].relays
+        relays = []
+        for num in relays_inst.keys():
+            to_append = relays_inst[num]
+            name = f"{net}{num}"
+            if name in main.IRCPool.keys():
+                to_append["chans"] = len(main.IRCPool[name].channels)
+                to_append["nick"] = main.IRCPool[name].nickname
+                to_append["conn"] = main.IRCPool[name].isconnected
+                to_append["limit"] = main.IRCPool[name].chanlimit
+                to_append["authed"] = main.IRCPool[name].authenticated
+            else:
+                to_append["chans"] = 0
+                to_append["nick"] = None
+                to_append["conn"] = False
+                to_append["limit"] = None
+                to_append["authed"] = None
+            relays.append(to_append)
+        return dumps({"relays": relays})
+
+    @app.route("/irc/network/<net>/<num>/", methods=["POST"])
+    @login_required
+    def irc_network_relay(self, request, net, num):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num not in net_inst.relays:
+            return dumps({"success": False, "reason": "network has no such relay."})
+        if "status" in data:
+            if not type(data["status"]) == int:
+                return dumps({"success": False, "reason": "invalid type for enabled."})
+            enabled = data["status"]
+            if enabled:
+                net_inst.enable_relay(num)
+            else:
+                net_inst.disable_relay(num)
+        main.saveConf("network")
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/<num>/", methods=["PUT"])
+    @login_required
+    def irc_network_relay_add(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num in net_inst.relays:
+            return dumps(
+                {"success": False, "reason": "network already has this relay."}
+            )
+        id, alias = net_inst.add_relay(num)
+        main.saveConf("network")
+        return dumps({"success": True, "id": id, "alias": alias})
+
+    @app.route("/irc/network/<net>/", methods=["PUT"])
+    @login_required
+    def irc_network_relay_add_next(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        net_inst = main.network[net]
+        id, alias = net_inst.add_relay()
+        main.saveConf("network")
+        return dumps({"success": True, "id": id, "alias": alias})
+
+    @app.route("/irc/network/<net>/<num>/", methods=["DELETE"])
+    @login_required
+    def irc_network_relay_del(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num not in net_inst.relays:
+            return dumps(
+                {"success": False, "reason": "network does not have this relay."}
+            )
+        net_inst.delete_relay(num)
+        main.saveConf("network")
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/<num>/provision/", methods=["POST"])
+    @login_required
+    def irc_network_relay_provision(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num not in net_inst.relays:
+            return dumps(
+                {"success": False, "reason": "network does not have this relay."}
+            )
+        provision.provisionRelay(num, net)
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/channels/", methods=["POST"])
+    @login_required
+    def irc_network_channels(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        relays_inst = main.network[net].relays
+        channels = []
+        for num in relays_inst.keys():
+            name = f"{net}{num}"
+            if name in main.IRCPool.keys():
+                net_chans = main.IRCPool[name].channels
+                channels_annotated = userinfo.getUserNum(net, net_chans)
+                for channel in net_chans:
+                    channel_inst = {
+                        "name": channel,
+                        "users": channels_annotated[channel],
+                        "num": num,
+                    }
+                    channels.append(channel_inst)
+                    # channels[channel] = channels_annotated[channel]
+
+        return dumps({"channels": channels})
+
+    # API version
+    @app.route("/irc/network/<net>/channel/", methods=["DELETE"])
+    @login_required
+    def irc_network_channel_part_api(self, request, net):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "channel" not in data:
+            return dumps({"success": False, "reason": "no channel specified."})
+        channel = data["channel"]
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        parted = chankeep.partSingle(net, channel)
+        if not parted:
+            dumps({"success": False, "reason": "no channels matched."})
+        return dumps({"success": True, "relays": parted})
+
+    # API version
+    @app.route("/irc/network/<net>/channel/", methods=["PUT"])
+    @login_required
+    def irc_network_channel_join_api(self, request, net):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "channel" not in data:
+            return dumps({"success": False, "reason": "no channel specified."})
+        channel = data["channel"]
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        joined = chankeep.joinSingle(net, channel)
+        if not joined:
+            return dumps(
+                {"success": False, "reason": "could not allocate channel to relay."}
+            )
+        return dumps({"success": True, "relays": joined})
+
+    @app.route("/irc/network/<net>/channel/<channel>/", methods=["PUT"])
+    @login_required
+    def irc_network_channel_join(self, request, net, channel):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        joined = chankeep.joinSingle(net, channel)
+        if not joined:
+            return dumps(
+                {"success": False, "reason": "could not allocate channel to relay."}
+            )
+        return dumps({"success": True, "relays": joined})
+
+    @app.route("/aliases/", methods=["GET"])
+    @login_required
+    def aliases(self, request):
+        alias_list = []
+        for num, alias in main.alias.items():
+            alias_dup = dict(alias)
+            alias_dup["num"] = num
+            alias_list.append(alias_dup)
+
+        return dumps({"aliases": alias_list})
+
+    @app.route("/aliases/", methods=["POST"])
+    @login_required
+    def aliases_update(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        for alias, fields in data.items():
+            if not alias.isdigit():
+                return dumps({"success": False, "reason": "alias not a number."})
+            alias = int(alias)
+            if alias not in main.alias.keys():
+                return dumps({"success": False, "reason": "alias does not exist."})
+            if fields:
+                for field in fields:
+                    if field in main.alias[alias]:
+                        main.alias[alias][field] = fields[field]
+            if "emails" in fields:
+                if not fields["emails"]:
+                    main.alias[alias]["emails"] = []
+        main.saveConf("alias")
+
+        return dumps({"success": True})
+
+    @app.route("/irc/auto/<net>/", methods=["POST"])
+    @login_required
+    def irc_auto_network(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if 1 in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"first relay exists on {net}"})
+        num, alias = main.network[net].add_relay(1)
+        provision.provisionRelay(num, net)
+        main.saveConf("network")
+
+        return dumps(
+            {
+                "success": True,
+                "message": f"created relay {num} with alias {alias} on {net}",
+            }
+        )
+
+    @app.route("/irc/list/<net>/", methods=["POST"])
+    @login_required
+    def irc_list_network(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        first_relay = helpers.get_first_relay(net)
+        if not first_relay:
+            return dumps(
+                {"success": False, "reason": f"could not get first relay for {net}"}
+            )
+        first_relay.list()
+        return dumps(
+            {
+                "success": True,
+                "message": f"requested list with instance {first_relay.num} of {net}",
+            }
+        )
+
+    @app.route("/irc/list/<net>/", methods=["GET"])
+    @login_required
+    def get_irc_list_info(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        listinfo = chankeep.getListInfo(net)
+        return dumps({"success": True, "listinfo": listinfo})
+
+    @app.route("/irc/msg/<net>/<num>/", methods=["PUT"])
+    @login_required
+    def irc_send_message(self, request, net, num):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "msg" not in data:
+            return dumps({"success": False, "reason": "no msg."})
+        if "channel" not in data:
+            return dumps({"success": False, "reason": "no msg."})
+        msg = data["msg"]
+        channel = data["channel"]
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        if num not in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"no relay {num} on {net}"})
+        name = f"{net}{num}"
+        if name not in main.IRCPool.keys():
+            return dumps({"success": False, "reason": f"relay {num} not on {net}"})
+        # We are in a query
+        in_query = False
+        if "nick" in data:
+            nick = data["nick"]
+            if nick == channel:
+                in_query = True
+        else:
+            nick = None
+        if channel == main.IRCPool[name].nickname or nick == channel:
+            in_query = True
+            if not nick:
+                return dumps({"success": False, "reason": "no nick specified to query"})
+            else:
+                main.IRCPool[name].sendmsg(nick, msg, in_query=in_query)
+        else:
+            main.IRCPool[name].sendmsg(channel, msg, in_query=in_query)
+        return dumps(
+            {"success": True, "message": f"sent message to {channel} on {name}"}
+        )
+
+    @app.route("/irc/nick/<net>/<num>/", methods=["GET"])
+    @login_required
+    def irc_get_nick(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        if num not in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"no relay {num} on {net}"})
+        name = f"{net}{num}"
+        if name not in main.IRCPool.keys():
+            return dumps({"success": False, "reason": f"relay {num} not on {net}"})
+        return dumps({"nickname": main.IRCPool[name].nickname})
+
+    @app.route("/irc/reg/<net>/", methods=["GET"])
+    @login_required
+    def irc_get_unreg_net(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        unreg = regproc.get_unregistered_relays(net)
+        return dumps({"success": True, "unreg": unreg})
+
+    @app.route("/irc/reg/", methods=["GET"])
+    @login_required
+    def irc_get_unreg(self, request):
+        unreg = regproc.get_unregistered_relays()
+        return dumps({"success": True, "unreg": unreg})
+
+    @app.route("/irc/reg/", methods=["PUT"])
+    @login_required
+    def irc_confirm_accounts(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        for item, token in data.items():
+            if "|" not in item:
+                return dumps({"success": False, "reason": f"malformed item: {item}"})
+            spl = item.split("|")
+            if not len(spl) == 2:
+                return dumps({"success": False, "reason": f"malformed item: {item}"})
+            net, num = spl
+            if not num.isdigit():
+                return dumps({"success": False, "reason": "invalid num: not a number."})
+            num = int(num)
+            if token:
+                regproc.confirmAccount(net, num, token)
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/<num>/auth/", methods=["POST"])
+    @login_required
+    def irc_enable_auth(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        if num not in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"no relay {num} on {net}"})
+        regproc.enableAuthentication(net, num, jump=False, run_now=True)
+        return dumps({"success": True})
+
+    @app.route("/irc/sinst/<net>/", methods=["GET"])
+    @login_required
+    def irc_get_authentity(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        auth = regproc.selectInst(net)
+        if not auth:
+            return dumps({"success": False, "reason": "error getting results."})
+        return dumps({"success": True, "sinst": auth})

legacy/commands/admall.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class AdmallCommand:
     def __init__(self, *args):
         self.admall(*args)
 
-    def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def admall(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 2:
                 for i in main.network.keys():

legacy/commands/alias.py

@@ -1,12 +1,15 @@
 import main
-from yaml import dump
 from modules import alias
+from yaml import dump
+
 
 class AliasCommand:
     def __init__(self, *args):
         self.alias(*args)
 
-    def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def alias(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 1:
                 info(dump(main.alias))
@@ -21,7 +24,7 @@ class AliasCommand:
                         if len(main.alias.keys()) == 0:
                             nextNum = 1
                         else:
-                            nextNum = max(main.alias.keys())+1
+                            nextNum = max(main.alias.keys()) + 1
                         main.alias[nextNum] = alias.generate_alias()
                         success("Generated new alias: %i" % nextNum)
                     main.saveConf("alias")
@@ -31,7 +34,7 @@ class AliasCommand:
                         failure("Must be a number, not %s" % spl[2])
                         return
                     num = int(spl[2])
-                    if not num in main.alias.keys():
+                    if num not in main.alias.keys():
                         failure("No such alias: %i" % num)
                         return
                     failed = False

legacy/commands/all.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class AllCommand:
     def __init__(self, *args):
         self.all(*args)
 
-    def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def all(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 2:
                 for i in main.network.keys():
@@ -14,8 +17,11 @@ class AllCommand:
                         net = main.network[i].relays[x]["net"]
                         alias = main.alias[x]["nick"]
                         commands = {spl[1]: [" ".join(spl[2:])]}
-                        success("Sending commands to relay %s as user %s" % (num, alias+"/"+net))
-                        deliverRelayCommands(num, commands, user=alias+"/"+net)
+                        success(
+                            "Sending commands to relay %s as user %s"
+                            % (num, alias + "/" + net)
+                        )
+                        deliverRelayCommands(num, commands, user=alias + "/" + net)
                 return
             else:
                 incUsage("all")

legacy/commands/allc.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class AllcCommand:
     def __init__(self, *args):
         self.allc(*args)
 
-    def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def allc(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 4:
                 targets = []
@@ -16,9 +19,12 @@ class AllcCommand:
                                 targets.append((i, x))
                 elif spl[1] == "alias":
                     for i in main.network.keys():
-                        [targets.append((i, x)) for x in main.alias.keys() if
-                                main.alias[x]["nick"] == spl[2] and
-                                x in main.network[i].aliases.keys()]
+                        [
+                            targets.append((i, x))
+                            for x in main.alias.keys()
+                            if main.alias[x]["nick"] == spl[2]
+                            and x in main.network[i].aliases.keys()
+                        ]
                 else:
                     incUsage("allc")
                     return
@@ -30,8 +36,11 @@ class AllcCommand:
                     num = i[1]
                     alias = main.alias[num]["nick"]
                     commands = {spl[3]: [" ".join(spl[4:])]}
-                    success("Sending commands to relay %i as user %s" % (num, alias+"/"+net))
-                    deliverRelayCommands(num, commands, user=alias+"/"+net)
+                    success(
+                        "Sending commands to relay %i as user %s"
+                        % (num, alias + "/" + net)
+                    )
+                    deliverRelayCommands(num, commands, user=alias + "/" + net)
                 return
             else:
                 incUsage("allc")

legacy/commands/authcheck.py

@@ -0,0 +1,43 @@
+import main
+
+
+class AuthcheckCommand:
+    def __init__(self, *args):
+        self.authcheck(*args)
+
+    def authcheck(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                results = []
+                for i in main.IRCPool.keys():
+                    num = main.IRCPool[i].num
+                    net = main.IRCPool[i].net
+                    if not main.IRCPool[i].authenticated:
+                        results.append(
+                            "%s - %s: %s" % (net, num, main.alias[num]["nick"])
+                        )
+                info("\n".join(results))
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                results = []
+                for i in main.IRCPool.keys():
+                    num = main.IRCPool[i].num
+                    net = main.IRCPool[i].net
+                    if not net == spl[1]:
+                        continue
+                    if not main.IRCPool[i].authenticated:
+                        results.append(
+                            "%s - %s: %s" % (net, num, main.alias[num]["nick"])
+                        )
+                info("\n".join(results))
+                return
+            else:
+                incUsage("authcheck")
+                return
+        else:
+            incUsage(None)

legacy/commands/auto.py

@@ -1,11 +1,14 @@
 import main
 from modules import provision
 
+
 class AutoCommand:
     def __init__(self, *args):
         self.auto(*args)
 
-    def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def auto(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 1:
                 for i in main.network.keys():
@@ -13,9 +16,15 @@ class AutoCommand:
                         info("Skipping %s - first relay exists" % i)
                     else:
                         num, alias = main.network[i].add_relay(1)
-                        success("Successfully created first relay on network %s with alias %s" % (i, alias))
+                        success(
+                            "Successfully created first relay on network %s with alias %s"
+                            % (i, alias)
+                        )
                         provision.provisionRelay(num, i)
-                        success("Started provisioning network %s on first relay for alias %s" % (i, alias))
+                        success(
+                            "Started provisioning network %s on first relay for alias %s"
+                            % (i, alias)
+                        )
                 main.saveConf("network")
                 return
             elif length == 2:
@@ -26,9 +35,15 @@ class AutoCommand:
                     failure("First relay exists on %s" % spl[1])
                     return
                 num, alias = main.network[spl[1]].add_relay(1)
-                success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias))
+                success(
+                    "Successfully created relay %i on network %s with alias %s"
+                    % (num, spl[1], alias)
+                )
                 provision.provisionRelay(num, spl[1])
-                success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias))
+                success(
+                    "Started provisioning network %s on relay %s for alias %s"
+                    % (spl[1], num, alias)
+                )
                 main.saveConf("network")
                 return
             else:

legacy/commands/blacklist.py

@@ -0,0 +1,44 @@
+import main
+from yaml import dump
+
+
+class BlacklistCommand:
+    def __init__(self, *args):
+        self.blacklist(*args)
+
+    def blacklist(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                info(dump(main.blacklist))
+                return
+            elif length == 4:
+                if spl[1] == "add":
+                    if spl[2] in main.blacklist.keys():
+                        main.blacklist[spl[2]].append(spl[3])
+                    else:
+                        main.blacklist[spl[2]] = [spl[3]]
+                    success("Blacklisted %s on %s" % (spl[3], spl[2]))
+                    main.saveConf("blacklist")
+                    return
+                elif spl[1] == "del":
+                    if spl[2] in main.blacklist.keys():
+                        if spl[3] in main.blacklist[spl[2]]:
+                            main.blacklist[spl[2]].remove(spl[3])
+                            if len(main.blacklist[spl[2]]) == 0:
+                                del main.blacklist[spl[2]]
+                        else:
+                            failure("Not in list: %s" % spl[3])
+                            return
+                    else:
+                        failure("No such entry: %s" % spl[2])
+                        return
+                    success("Removed blacklist %s on %s" % (spl[3], spl[2]))
+                    main.saveConf("blacklist")
+                    return
+            else:
+                incUsage("blacklist")
+                return
+        else:
+            incUsage(None)

legacy/commands/chans.py

@@ -1,11 +1,13 @@
-import main
 import modules.userinfo as userinfo
 
+
 class ChansCommand:
     def __init__(self, *args):
         self.chans(*args)
 
-    def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def chans(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if len(spl) < 2:
                 incUsage("chans")
@@ -16,7 +18,7 @@ class ChansCommand:
                 rtrn += "Matches from: %s" % i
                 rtrn += "\n"
                 for x in result[i]:
-                    rtrn += (x)
+                    rtrn += x
                     rtrn += "\n"
                 rtrn += "\n"
             info(rtrn)

legacy/commands/cmd.py

@@ -1,11 +1,13 @@
-import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class CmdCommand:
     def __init__(self, *args):
         self.cmd(*args)
 
-    def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def cmd(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 4:
                 if not spl[1].isdigit():

legacy/commands/confirm.py

@@ -0,0 +1,33 @@
+import main
+from modules import regproc
+
+
+class ConfirmCommand:
+    def __init__(self, *args):
+        self.confirm(*args)
+
+    def confirm(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 4:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                if not spl[2].isdigit():
+                    failure("Must be a number, not %s" % spl[2])
+                    return
+                if not int(spl[2]) in main.network[spl[1]].relays.keys():
+                    failure("No such relay on %s: %s" % (spl[1], spl[2]))
+                    return
+                regproc.confirmAccount(spl[1], int(spl[2]), spl[3])
+                success(
+                    "Requested confirmation on %s - %s with token %s"
+                    % (spl[1], spl[2], spl[3])
+                )
+                return
+            else:
+                incUsage("confirm")
+                return
+        else:
+            incUsage(None)

legacy/commands/dedup.py

@@ -0,0 +1,27 @@
+from json import dumps
+
+import main
+from modules import chankeep
+
+
+class DedupCommand:
+    def __init__(self, *args):
+        self.dedup(*args)
+
+    def dedup(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                dupes = chankeep.getDuplicateChannels()
+                chankeep.partChannels(dupes)
+                info(dumps(dupes))
+                return
+            elif length == 2:
+                if spl[1] not in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                dupes = chankeep.getDuplicateChannels(spl[1])
+                chankeep.partChannels(dupes)
+                info(dumps(dupes))
+                return

legacy/commands/disable.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class DisableCommand:
     def __init__(self, *args):
         self.disable(*args)
 
-    def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def disable(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 3:
                 if not spl[1] in main.network.keys():
@@ -15,18 +18,18 @@ class DisableCommand:
                     failure("Must be a number, not %s" % spl[2])
                     return
                 relayNum = int(spl[2])
-                name = spl[1]+spl[2]
+                name = spl[1] + spl[2]
                 if not spl[1] in main.IRCPool.keys():
                     info("Note - instance not running, proceeding anyway")
-                if not relayNum in main.network[spl[1]].relays.keys():
+                if relayNum not in main.network[spl[1]].relays.keys():
                     failure("No such relay: %s in network %s" % (spl[2], spl[1]))
                     return
                 main.network[spl[1]].relays[relayNum]["enabled"] = False
                 user = main.alias[relayNum]["nick"]
                 network = spl[1]
-                relay = main.network[spl[1]].relays[relayNum]
+                # relay = main.network[spl[1]].relays[relayNum]
                 commands = {"status": ["Disconnect"]}
-                deliverRelayCommands(relayNum, commands, user=user+"/"+network)
+                deliverRelayCommands(relayNum, commands, user=user + "/" + network)
                 main.saveConf("network")
                 if name in main.ReactorPool.keys():
                     if name in main.FactoryPool.keys():

legacy/commands/dist.py

@@ -1,11 +1,15 @@
+from subprocess import PIPE, run
+
 import main
-from subprocess import run, PIPE
+
 
 class DistCommand:
     def __init__(self, *args):
         self.dist(*args)
 
-    def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def dist(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if main.config["Dist"]["Enabled"]:
                 rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE)

legacy/commands/email.py

@@ -0,0 +1,119 @@
+import main
+from yaml import dump
+
+
+class EmailCommand:
+    def __init__(self, *args):
+        self.email(*args)
+
+    def email(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 4:
+                if spl[1] == "add":
+                    if spl[2].isdigit():
+                        num = int(spl[2])
+                        if num not in main.alias.keys():
+                            failure("No such alias: %i" % num)
+                            return
+                        if not spl[3] in main.alias[num]["emails"]:
+                            main.alias[num]["emails"].append(spl[3])
+                            main.saveConf("alias")
+                            success(
+                                "Successfully added email %s to alias %i"
+                                % (spl[3], num)
+                            )
+                            return
+                        else:
+                            failure(
+                                "Email already exists in alias %i: %s" % (num, spl[3])
+                            )
+                            return
+                    else:
+                        failure("Must be a number, not %s" % spl[2])
+                        if spl[2] == "domain":
+                            domain = spl[3]
+                            if "@" in domain:
+                                failure("Not a domain: %s" % domain)
+                                return
+                            if domain not in main.irc["_"]["domains"]:
+                                main.irc["_"]["domains"].append(domain)
+                                success(
+                                    "Successfully added domain %s to default config"
+                                    % domain
+                                )
+                                main.saveConf("irc")
+                            else:
+                                failure(
+                                    "Domain already exists in default config: %s"
+                                    % domain
+                                )
+                                return
+
+                elif spl[1] == "del":
+                    if not spl[2].isdigit():
+                        # failure("Must be a number, not %s" % spl[2])
+                        if spl[2] == "domain":
+                            domain = spl[3]
+
+                            if domain in main.irc["_"]["domains"]:
+                                main.irc["_"]["domains"].remove(domain)
+                                success(
+                                    "Successfully removed domain %s to default config"
+                                    % domain
+                                )
+                                main.saveConf("irc")
+                            else:
+                                failure(
+                                    "Domain does not exist in default config: %s"
+                                    % domain
+                                )
+                                return
+
+                    elif spl[1] == "del":
+                        if num not in main.alias.keys():
+                            failure("No such alias: %i" % num)
+                            return
+                        if spl[3] in main.alias[num]["emails"]:
+                            main.alias[num]["emails"].remove(spl[3])
+                            main.saveConf("alias")
+                            success(
+                                "Successfully removed email %s from alias %i"
+                                % (spl[3], num)
+                            )
+                            return
+                        else:
+                            failure(
+                                "Email does not exist in alias %i: %s" % (spl[3], num)
+                            )
+                            return
+            elif length == 2:
+                if spl[1] == "list":
+                    info(dump(main.alias))
+                    return
+                else:
+                    incUsage("email")
+                    return
+            elif length == 3:
+                if spl[1] == "list":
+                    if spl[2] == "domain":
+                        filtered = {
+                            f"{k}:{k2}": v2
+                            for k, v in main.irc.items()
+                            for k2, v2 in v.items()
+                            if k2 == "domains"
+                        }
+                        info(dump(filtered))
+                        return
+                    else:
+                        incUsage("email")
+                        return
+                else:
+                    incUsage("email")
+                    return
+            else:
+                incUsage("email")
+                return
+        else:
+            incUsage(None)

legacy/commands/enable.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class EnableCommand:
     def __init__(self, *args):
         self.enable(*args)
 
-    def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def enable(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 3:
                 if not spl[1] in main.network.keys():
@@ -22,9 +25,9 @@ class EnableCommand:
                 user = main.alias[int(spl[2])]["nick"]
                 network = spl[1]
                 commands = {"status": ["Connect"]}
-                deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network)
+                deliverRelayCommands(int(spl[2]), commands, user=user + "/" + network)
                 main.saveConf("network")
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     main.network[spl[1]].start_bot(int(spl[2]))
                 else:
                     pass

legacy/commands/exec.py

@@ -1,10 +1,10 @@
-import main
-
 class ExecCommand:
     def __init__(self, *args):
         self.exec(*args)
 
-    def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def exec(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 1:
                 try:

legacy/commands/getstr.py

@@ -0,0 +1,35 @@
+import main
+from utils.get import getRelay
+
+
+class GetstrCommand:
+    def __init__(self, *args):
+        self.getstr(*args)
+
+    def getstr(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 3:
+                net = spl[1]
+                num = spl[2]
+                if net not in main.network.keys():
+                    failure("Network does not exist: %s" % net)
+                    return
+                if not num.isdigit():
+                    failure("Must be a number, not %s" % num)
+                    return
+                num = int(num)
+                alias = main.alias[num]["nick"].lower()
+                host, port = getRelay(num)
+                password = main.config["Relay"]["Password"]
+                connstr = f"/connect -ssl {host} {port}"
+                authstr = f"/quote PASS {alias}/{net}:{password}"
+                obj.send(connstr)
+                obj.send(authstr)
+
+            else:
+                incUsage("getstr")
+                return
+        else:
+            incUsage(None)

legacy/commands/help.py

@@ -1,10 +1,13 @@
 import main
 
+
 class HelpCommand:
     def __init__(self, *args):
         self.help(*args)
 
-    def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def help(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             helpMap = []
             for i in main.help.keys():

legacy/commands/join.py

@@ -1,22 +1,33 @@
 import main
+import modules.chankeep
+
 
 class JoinCommand:
     def __init__(self, *args):
         self.join(*args)
 
-    def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def join(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
-            if length == 4:
+            if length == 3:
+                if not spl[1] in main.network.keys():
+                    failure("Network does not exist: %s" % spl[1])
+                    return
+                modules.chankeep.joinSingle(spl[1], spl[2])
+                success("Joined %s" % spl[2])
+
+            elif length == 4:
                 if not spl[1] in main.network.keys():
                     failure("Network does not exist: %s" % spl[1])
                     return
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
                     failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     failure("Name has no instance: %s" % spl[1])
                     return
-                main.IRCPool[spl[1]+spl[2]].join(spl[3])
+                main.IRCPool[spl[1] + spl[2]].join(spl[3])
                 success("Joined %s" % spl[3])
                 return
             elif length == 5:
@@ -26,10 +37,10 @@ class JoinCommand:
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
                     failure("Relay % does not exist on network %", (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     failure("Name has no instance: %s" % spl[1])
                     return
-                main.IRCPool[spl[1]+spl[2]].join(spl[3], spl[4])
+                main.IRCPool[spl[1] + spl[2]].join(spl[3], spl[4])
                 success("Joined %s with key %s" % (spl[3], spl[4]))
                 return
             else:

legacy/commands/list.py

@@ -0,0 +1,38 @@
+import main
+from modules import helpers
+
+
+class ListCommand:
+    def __init__(self, *args):
+        self.list(*args)
+
+    def list(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                for i in main.network.keys():
+                    first_relay = helpers.get_first_relay(i)
+                    ####
+                    if not first_relay:
+                        info("Network has no first instance: %s" % i)
+                        continue
+                    first_relay.list()
+                    success(f"Requested list with instance {first_relay.num} of {i}")
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                first_relay = helpers.get_first_relay(spl[1])
+                if not first_relay:
+                    failure("Could not get first instance")
+                    return
+                first_relay.list()
+                success(f"Requested list with instance {first_relay.num} of {spl[1]}")
+                return
+            else:
+                incUsage("list")
+                return
+        else:
+            incUsage(None)

legacy/commands/load.py

@@ -1,10 +1,13 @@
 import main
 
+
 class LoadCommand:
     def __init__(self, *args):
         self.load(*args)
 
-    def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def load(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if spl[1] in main.filemap.keys():

legacy/commands/loadmod.py

@@ -1,15 +1,17 @@
-import main
 from utils.loaders.single_loader import loadSingle
 
+
 class LoadmodCommand:
     def __init__(self, *args):
         self.loadmod(*args)
 
-    def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def loadmod(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 rtrn = loadSingle(spl[1])
-                if rtrn == True:
+                if rtrn is True:
                     success("Loaded module: %s" % spl[1])
                     return
                 elif rtrn == "RELOAD":

legacy/commands/logout.py

@@ -1,14 +1,12 @@
-import main
-
 class LogoutCommand:
     def __init__(self, *args):
         self.logout(*args)
 
-    def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def logout(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             obj.authed = False
-            if obj.addr in main.MonitorPool:
-                main.MonitorPool.remove(obj.addr)
             success("Logged out")
             return
         else:

legacy/commands/mod.py

@@ -1,12 +1,14 @@
 import main
-from yaml import dump
+
 
 class ModCommand:
     # This could be greatly improved, but not really important right now
     def __init__(self, *args):
         self.mod(*args)
 
-    def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def mod(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 4:
                 if not spl[1] in main.network.keys():
@@ -15,15 +17,17 @@ class ModCommand:
 
                 try:
                     setattr(main.network[spl[1]], spl[2], spl[3])
-                except e:
-                    failure("Something went wrong.")
+                except Exception as e:
+                    failure(f"Something went wrong: {e}")
                     return
 
                 main.saveConf("network")
-                success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]))
+                success(
+                    "Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])
+                )
                 return
             # Find a better way to do this
-            #elif length == 6:
+            # elif length == 6:
             #    if not spl[1] in main.network.keys():
             #        failure("Network does not exist: %s" % spl[1])
             #        return

legacy/commands/msg.py

@@ -1,25 +1,31 @@
 import main
 
+
 class MsgCommand:
     def __init__(self, *args):
         self.msg(*args)
 
-    def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def msg(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length >= 5:
                 if not spl[1] in main.network.keys():
                     failure("Network does not exist: %s" % spl[1])
                     return
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
-                    failure("Relay % does not exist on network %" % (spl[2], spl[1]))
+                    failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     failure("Name has no instance: %s" % spl[1])
                     return
-                if not spl[3] in main.IRCPool[spl[1]+spl[2]].channels:
+                if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels:
                     info("Bot not on channel: %s" % spl[3])
-                main.IRCPool[spl[1]+spl[2]].msg(spl[3], " ".join(spl[4:]))
-                success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1]))
+                main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:]))
+                success(
+                    "Sent %s to %s on relay %s on network %s"
+                    % (" ".join(spl[4:]), spl[3], spl[2], spl[1])
+                )
                 return
             else:
                 incUsage("msg")

legacy/commands/network.py

@@ -1,13 +1,17 @@
-import main
-from yaml import dump
-from modules.network import Network
 from string import digits
 
+import main
+from modules.network import Network
+from yaml import dump
+
+
 class NetworkCommand:
     def __init__(self, *args):
         self.network(*args)
 
-    def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def network(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 7:
                 if spl[1] == "add":
@@ -27,7 +31,9 @@ class NetworkCommand:
                         failure("Auth must be sasl, ns or none, not %s" % spl[5])
                         return
                     else:
-                        main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower())
+                        main.network[spl[2]] = Network(
+                            spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()
+                        )
                         success("Successfully created network: %s" % spl[2])
                         main.saveConf("network")
                         return

legacy/commands/part.py

@@ -1,10 +1,13 @@
 import main
 
+
 class PartCommand:
     def __init__(self, *args):
         self.part(*args)
 
-    def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def part(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 4:
                 if not spl[1] in main.network.keys():
@@ -13,10 +16,10 @@ class PartCommand:
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
                     failure("Relay % does not exist on network %", (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
-                    failure("Name has no instance: %s" % spl[1]+spl[2])
+                if not spl[1] + spl[2] in main.IRCPool.keys():
+                    failure("Name has no instance: %s" % spl[1] + spl[2])
                     return
-                main.IRCPool[spl[1]+spl[2]].part(spl[3])
+                main.IRCPool[spl[1] + spl[2]].part(spl[3])
                 success("Left %s" % spl[3])
                 return
             else:

legacy/commands/pass.py

@@ -1,10 +1,13 @@
 import main
 
+
 class PassCommand:
     def __init__(self, *args):
         self.password(*args)
 
-    def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def password(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             info("You are already authenticated")
             return

legacy/commands/pending.py

@@ -0,0 +1,40 @@
+import main
+
+
+class PendingCommand:
+    def __init__(self, *args):
+        self.pending(*args)
+
+    def pending(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                results = []
+                for i in main.network.keys():
+                    for x in main.network[i].relays.keys():
+                        if not main.network[i].relays[x]["registered"]:
+                            results.append(
+                                "%s: confirm %s %s [code]"
+                                % (main.alias[x]["nick"], i, x)
+                            )
+                info("\n".join(results))
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                results = []
+                for x in main.network[spl[1]].relays.keys():
+                    if not main.network[spl[1]].relays[x]["registered"]:
+                        results.append(
+                            "%s: confirm %s %s [code]"
+                            % (main.alias[x]["nick"], spl[1], x)
+                        )
+                info("\n".join(results))
+                return
+            else:
+                incUsage("pending")
+                return
+        else:
+            incUsage(None)

legacy/commands/recheckauth.py

@@ -0,0 +1,36 @@
+import main
+
+
+class RecheckauthCommand:
+    def __init__(self, *args):
+        self.recheckauth(*args)
+
+    def recheckauth(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                for i in main.IRCPool.keys():
+                    net = main.IRCPool[i].net
+                    main.IRCPool[i].authenticated = False
+                    main.IRCPool[i].regPing()
+                success("Successfully reset authentication status")
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                for i in main.IRCPool.keys():
+                    # num = main.IRCPool[i].num
+                    net = main.IRCPool[i].net
+                    if not net == spl[1]:
+                        continue
+                    main.IRCPool[i].authenticated = False
+                    main.IRCPool[i].regPing()
+                success("Successfully reset authentication status on %s" % spl[1])
+                return
+            else:
+                incUsage("recheckauth")
+                return
+        else:
+            incUsage(None)

legacy/commands/reg.py

@@ -0,0 +1,38 @@
+import main
+from modules import regproc
+
+
+class RegCommand:
+    def __init__(self, *args):
+        self.reg(*args)
+
+    def reg(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                for i in main.network[spl[1]].relays.keys():
+                    regproc.registerAccount(spl[1], i)
+                success("Requested registration for all relays on %s" % spl[1])
+                return
+            elif length == 3:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                if not spl[2].isdigit():
+                    failure("Must be a number, not %s" % spl[2])
+                    return
+                if not int(spl[2]) in main.network[spl[1]].relays.keys():
+                    failure("No such relay on %s: %s" % (spl[2], spl[1]))
+                    return
+                regproc.registerAccount(spl[1], int(spl[2]))
+                success("Requested registration on %s - %s" % (spl[1], spl[2]))
+                return
+            else:
+                incUsage("reg")
+                return
+        else:
+            incUsage(None)

legacy/commands/relay.py

@@ -1,17 +1,23 @@
 import main
 from yaml import dump
 
+
 class RelayCommand:
     def __init__(self, *args):
         self.relay(*args)
 
-    def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def relay(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 3:
                 if spl[1] == "add":
                     if spl[2] in main.network.keys():
                         id, alias = main.network[spl[2]].add_relay()
-                        success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
+                        success(
+                            "Successfully created relay %s on network %s with alias %s"
+                            % (str(id), spl[2], alias)
+                        )
                         main.saveConf("network")
                         return
                     else:
@@ -34,7 +40,10 @@ class RelayCommand:
                             failure("Must be a number, not %s" % spl[3])
                             return
                         id, alias = main.network[spl[2]].add_relay(int(spl[3]))
-                        success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
+                        success(
+                            "Successfully created relay %s on network %s with alias %s"
+                            % (str(id), spl[2], alias)
+                        )
                         main.saveConf("network")
                         return
                     else:
@@ -51,7 +60,9 @@ class RelayCommand:
                         failure("No such relay: %s on network %s" % (spl[3], spl[2]))
                         return
                     main.network[spl[2]].delete_relay(int(spl[3]))
-                    success("Successfully deleted relay %s on network %s" % (spl[3], spl[2]))
+                    success(
+                        "Successfully deleted relay %s on network %s" % (spl[3], spl[2])
+                    )
                     main.saveConf("network")
                     return
                 else:

legacy/commands/save.py

@@ -1,10 +1,13 @@
 import main
 
+
 class SaveCommand:
     def __init__(self, *args):
         self.save(*args)
 
-    def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def save(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if spl[1] in main.filemap.keys():

legacy/commands/stats.py

@@ -1,13 +1,17 @@
+from string import digits
+
 import main
 import modules.counters as count
 import modules.userinfo as userinfo
-from string import digits
+
 
 class StatsCommand:
     def __init__(self, *args):
         self.stats(*args)
 
-    def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def stats(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 1:
                 stats = []
@@ -25,7 +29,7 @@ class StatsCommand:
                 stats.append("User records: %s" % numWhoEntries)
                 stats.append("Events/min: %s" % main.lastMinuteSample)
                 counterEvents = count.getEvents()
-                if counterEvents == None:
+                if counterEvents is None:
                     stats.append("No counters records")
                 else:
                     stats.append("Counters:")
@@ -42,7 +46,7 @@ class StatsCommand:
                 numNodes = 0
 
                 for i in main.IRCPool.keys():
-                    if "".join([x for x in i if not x in digits]) == spl[1]:
+                    if "".join([x for x in i if x not in digits]) == spl[1]:
                         numChannels += len(main.IRCPool[i].channels)
                         found = True
                         numNodes += 1
@@ -53,7 +57,7 @@ class StatsCommand:
                 stats.append("User records: %s" % numWhoEntries)
                 stats.append("Endpoints: %s" % numNodes)
                 counterEvents = count.getEvents(spl[1])
-                if counterEvents == None:
+                if counterEvents is None:
                     stats.append("No counters records")
                 else:
                     stats.append("Counters:")

legacy/commands/swho.py

@@ -1,10 +1,13 @@
 import main
 
+
 class SwhoCommand:
     def __init__(self, *args):
         self.swho(*args)
 
-    def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def swho(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if not spl[1] in main.network.keys():

legacy/commands/token.py

@@ -1,12 +1,16 @@
+from uuid import uuid4
+
 import main
 from yaml import dump
-from uuid import uuid4
+
 
 class TokenCommand:
     def __init__(self, *args):
         self.token(*args)
 
-    def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def token(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if spl[1] == "list":
@@ -31,8 +35,9 @@ class TokenCommand:
             elif length == 4:
                 if spl[1] == "add":
                     if not spl[2] in main.tokens.keys():
-                        if spl[3] in ["relay"]: # more to come!
-                            main.tokens[spl[2]] = {"hello": str(uuid4()),
+                        if spl[3] in ["relay", "api"]:  # more to come!
+                            main.tokens[spl[2]] = {
+                                "hello": str(uuid4()),
                                 "usage": spl[3],
                                 "counter": str(uuid4()),
                             }

legacy/commands/users.py

@@ -1,11 +1,13 @@
-import main
 import modules.userinfo as userinfo
 
+
 class UsersCommand:
     def __init__(self, *args):
         self.users(*args)
 
-    def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def users(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if len(spl) < 2:
                 incUsage("users")
@@ -16,7 +18,7 @@ class UsersCommand:
                 rtrn += "Matches from: %s" % i
                 rtrn += "\n"
                 for x in result[i]:
-                    rtrn += (x)
+                    rtrn += x
                     rtrn += "\n"
                 rtrn += "\n"
             info(rtrn)

legacy/commands/who.py

@@ -1,11 +1,13 @@
-import main
 import modules.userinfo as userinfo
 
+
 class WhoCommand:
     def __init__(self, *args):
         self.who(*args)
 
-    def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def who(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 result = userinfo.getWho(spl[1])
@@ -14,7 +16,7 @@ class WhoCommand:
                     rtrn += "Matches from: %s" % i
                     rtrn += "\n"
                     for x in result[i]:
-                        rtrn += (x)
+                        rtrn += x
                         rtrn += "\n"
                     rtrn += "\n"
                 info(rtrn)

legacy/conf/templates/config.json

@@ -10,19 +10,34 @@
         "Address": "127.0.0.1",
         "UseSSL": true
     },
+    "API": {
+        "Enabled": true,
+        "Port": 13869,
+        "Address": "127.0.0.1"
+    },
     "Key": "key.pem",
     "Certificate": "cert.pem",
-    "RedisSocket": "/tmp/redis.sock",
-    "UsePassword": true,
+    "RedisSocket": "/var/run/redis/redis.sock",
+    "RedisDBEphemeral": 1,
+    "RedisDBPersistent": 0,
+    "UsePassword": false,
     "ConnectOnCreate": false,
+    "AutoReg": false,
     "Debug": false,
+    "Trace": false,
     "Relay": {
         "Host": "127.0.0.1",
-        "Port": "201x",
-        "User": "sir",
-        "Password": "sir"
+        "Port": "2001",
+        "User": "x",
+        "Password": "x"
+    },
+    "Ingest": {
+        "Key": "queue.irc",
+        "Enabled": true
     },
     "ChanKeep": {
+        "Enabled": false,
+        "Provision": false,
         "MaxRelay": 30,
         "SigSwitch": 20
     },
@@ -32,10 +47,10 @@
         "File": "conf/dist.sh"
     },
     "Toggles": {
-        "Who": false,
+        "Who": true,
         "CycleChans": true
     },
-    "Password": "s",
+    "Password": "x",
     "Tweaks": {
         "MaxHash": 10,
         "DedupPrecision": 9,
@@ -43,6 +58,7 @@
             "Prefix": "*"
         },
         "Delays": {
+            "WhoDelay": 3600,
             "WhoLoop": 600,
             "WhoRange": 1800,
             "Timeout": 30,

legacy/conf/templates/help.json

@@ -7,14 +7,12 @@
     "part": "part <name> <num> <channel>",
     "enable": "enable <name> <num>",
     "disable": "disable <name> <num>",
-    "list": "list",
     "stats": "stats [<name>]",
     "save": "save <(file)|list|all>",
     "load": "load <(file)|list|all>",
     "dist": "dist",
     "loadmod": "loadmod <module>",
-    "msg": "msg <name> <target> <message...>",
-    "mon": "mon -h",
+    "msg": "msg <network> <num> <target> <message...>",
     "chans": "chans <nick> [<nick> ...]",
     "users": "users <channel> [<channel> ...]",
     "relay": "relay <add|del|list> [<network>] [<num>]",
@@ -22,11 +20,19 @@
     "alias": "alias [<add|del>] [<num>]",
     "auto": "auto [<network>]",
     "cmd": "cmd <relay> <user> <entity> <text ...>",
-    "token": "token <add|del|list> [<key>] [<relay>]",
+    "token": "token <add|del|list> [<key>] [relay|api]",
     "all": "all <entity> <text ...>",
     "allc": "allc <network|alias> <(network)|(alias)> <entity> <text ...>",
     "admall": "admall <entity> <text ...>",
     "swho": "swho <network> [<channel>]",
-    "list": "list <network>",
-    "exec": "exec <expr ...>"
+    "list": "list [<network>]",
+    "exec": "exec <expr ...>",
+    "reg": "reg <network> [<num>]",
+    "confirm": "confirm <network> <num> <token>",
+    "pending": "pending [<network>]",
+    "authcheck": "authcheck [<network>]",
+    "recheckauth": "recheckauth [<network>]",
+    "blacklist": "blacklist <network> <channel>",
+    "email": "email <add|del|list> [(domain)|<num>] [<email>]",
+    "getstr": "getstr <net> <num>"
 }

legacy/conf/templates/irc.json

@@ -0,0 +1,34 @@
+{
+    "_": {
+        "register": true,
+        "entity": "NickServ",
+        "domains": [],
+        "registermsg": "REGISTER {password} {email}",
+        "identifymsg": "IDENTIFY {password}",
+        "confirm": "CONFIRM {token}",
+        "check": true,
+        "ping": true,
+        "negative": true,
+        "pingmsg": "STATUS",
+        "negativemsg": "INFO {nickname}",
+        "checktype": "mode",
+        "checkmode": "R",
+        "checkmsg": "Password accepted - you are now recognized.",
+        "checkmsg2": "You are logged in as",
+        "checknegativemsg": "has \u0002NOT COMPLETED\u0002 registration verification",
+        "checkendnegative": "End of Info"
+    },
+    "freenode": {
+        "confirm": "VERIFY REGISTER {nickname} {token}"
+    },
+    "libera": {
+        "confirm": "VERIFY REGISTER {nickname} {token}"
+    },
+    "cyberia": {
+        "setmode": "I"
+    },
+    "ircnow": {
+        "negative": false,
+        "ping": false
+    }
+}

legacy/core/bot.py

@@ -0,0 +1,946 @@
+import sys
+from copy import deepcopy
+from datetime import datetime
+from random import randint
+
+import main
+from core.relay import sendRelayNotification
+from modules import chankeep, counters, helpers, monitor, regproc, userinfo
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.internet.protocol import ReconnectingClientFactory
+from twisted.internet.task import LoopingCall
+from twisted.words.protocols.irc import (
+    IRCBadMessage,
+    IRCClient,
+    lowDequote,
+    numeric_to_symbolic,
+)
+from utils.dedup import dedup
+from utils.logging.debug import debug, trace
+from utils.logging.log import error, log, warn
+from utils.logging.send import sendAll
+from utils.parsing import parsen
+
+
+# Copied from the Twisted source so we can fix a bug
+def parsemsg(s):
+    """
+    Breaks a message from an IRC server into its prefix, command, and
+    arguments.
+    @param s: The message to break.
+    @type s: L{bytes}
+    @return: A tuple of (prefix, command, args).
+    @rtype: L{tuple}
+    """
+    prefix = ""
+    trailing = []
+    if not s:
+        raise IRCBadMessage("Empty line.")
+    if s[0:1] == ":":
+        prefix, s = s[1:].split(" ", 1)
+    if s.find(" :") != -1:
+        s, trailing = s.split(" :", 1)
+        args = s.split(" ")  # Twisted bug fixed by adding an argument to split()
+        args.append(trailing)
+    else:
+        args = s.split(" ")  # And again
+    command = args.pop(0)
+    return prefix, command, args
+
+
+class IRCBot(IRCClient):
+    def __init__(self, net, num):
+        self.isconnected = False
+        self.channels = []
+        self.net = net
+        self.authenticated = not regproc.needToAuth(self.net)
+        self.num = num
+        self.buffer = ""
+        self.name = net + str(num)
+        alias = main.alias[num]
+        relay = main.network[self.net].relays[num]
+        self.netinst = main.network[self.net]
+        self.nickname = alias["nick"]
+        self.realname = alias["realname"]
+        self.username = alias["nick"].lower() + "/" + relay["net"]
+        self.password = main.config["Relay"]["Password"]
+        self.userinfo = None  #
+        self.fingerReply = None  #
+        self.versionName = None  # Don't give out information
+        self.versionNum = None  #
+        self.versionEnv = None  #
+        self.sourceURL = None  #
+
+        self._getWho = {}  # LoopingCall objects -- needed to be able to stop them
+
+        self._tempWho = {}  # temporary storage for gathering WHO info
+        self._tempNames = {}  # temporary storage for gathering NAMES info
+        self._tempList = ([], [])  # temporary storage for gathering LIST info
+        self.listOngoing = False  # we are currently receiving a LIST
+        self.listRetried = False  # we asked and got nothing so asked again
+        self.listAttempted = False  # we asked for a list
+        self.listSimple = False  # after asking again we got the list, so use the simple
+        #  syntax from now on
+        self.wantList = (
+            False  # we want to send a LIST, but not all relays are active yet
+        )
+        self.chanlimit = 0
+        self.prefix = {}
+        self.servername = None
+
+        self._regAttempt = None
+        self._negativePass = None
+
+    def lineReceived(self, line):
+        if bytes != str and isinstance(line, bytes):
+            # decode bytes from transport to unicode
+            line = line.decode("utf-8", "replace")
+
+        line = lowDequote(line)
+        try:
+            prefix, command, params = parsemsg(line)
+            if command in numeric_to_symbolic:
+                command = numeric_to_symbolic[command]
+            try:
+                self.handleCommand(command, prefix, params)
+            except Exception as err:
+                error(err)
+        except IRCBadMessage:
+            self.badMessage(line, *sys.exc_info())
+
+    def joinChannels(self, channels):
+        sleeptime = 0.0
+        increment = 0.8
+        for i in channels:
+            if i not in self.channels:
+                if self.net in main.blacklist.keys():
+                    if i in main.blacklist[self.net]:
+                        debug(
+                            "Not joining blacklisted channel %s on %s - %i"
+                            % (i, self.net, self.num)
+                        )
+                        continue
+                trace(
+                    self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds"
+                )
+                reactor.callLater(sleeptime, self.join, i)
+                sleeptime += increment
+                if sleeptime == 10:
+                    sleeptime = 0.0
+                    increment = 0.7
+                increment += 0.1
+            else:
+                error(
+                    "%s - Cannot join channel we are already on: %s - %i"
+                    % (i, self.net, self.num)
+                )
+
+    def checkChannels(self):
+        if chankeep.allRelaysActive(self.net):
+            debug(f"checkChannels() all relays active for {self.net}")
+        else:
+            debug(
+                "checkChannels() skipping channel check as we have inactive relays: %s - %i"
+                % (self.net, self.num)
+            )
+            return
+        if self.net in main.TempChan.keys():
+            if self.num in main.TempChan[self.net].keys():
+                self.joinChannels(main.TempChan[self.net][self.num])
+                del main.TempChan[self.net][self.num]
+                if not main.TempChan[self.net]:
+                    del main.TempChan[self.net]
+
+    def event(self, **cast):
+        if "ts" not in cast.keys():
+            cast["ts"] = int(datetime.now().timestamp())
+
+        # remove odd stuff
+        for i in list(
+            cast.keys()
+        ):  # Make a copy of the .keys() as Python 3 cannot handle iterating over
+            if cast[i] == "":  # a dictionary that changes length with each iteration
+                del cast[i]
+        # remove server stuff
+        if "muser" in cast.keys():
+            if cast["muser"] == self.servername:
+                cast["type"] = "conn"
+        if "channel" in cast.keys():
+            if cast["channel"] == "*":
+                cast["type"] = "conn"
+        ##
+
+        # expand out the hostmask
+        if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
+            if "muser" in cast.keys():
+                cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
+
+        # handle ZNC stuff
+        if {"nick", "ident", "host", "msg"}.issubset(set(cast)):
+            if cast["ident"] == "znc" and cast["host"] == "znc.in":
+                cast["type"] = "znc"
+                cast["num"] = self.num
+                del cast["nick"]
+                del cast["ident"]
+                del cast["host"]
+                del cast["channel"]
+                del cast["muser"]
+                if "Disconnected from IRC" in cast["msg"]:
+                    log("ZNC disconnected on %s - %i" % (self.net, self.num))
+                    self.isconnected = False
+                    self.authenticated = False
+                if "Connected!" in cast["msg"]:
+                    log("ZNC connected on %s - %i" % (self.net, self.num))
+                    self.isconnected = True
+                if "could not be joined, disabling it" in cast["msg"]:
+                    error(cast["msg"])
+        #
+
+        # don't reprocess the same message twice
+        # if the type is in that list, it's already been here, don't run it again
+        if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
+            cast["num"] = self.num
+            if "channel" in cast.keys():
+                if cast["type"] == "mode":
+                    if cast["channel"].lower() == self.nickname.lower():
+                        # castDup = deepcopy(cast)
+                        cast["mtype"] = cast["type"]
+                        cast["type"] = "self"
+                        # self.event(**castDup)
+                    if cast["modearg"]:  # check if modearg is non-NoneType
+                        if self.nickname.lower() == cast["modearg"].lower():
+                            castDup = deepcopy(cast)
+                            castDup["mtype"] = cast["type"]
+                            castDup["type"] = "highlight"
+                            self.event(**castDup)
+                else:
+                    if cast["channel"].lower() == self.nickname.lower() or not cast[
+                        "channel"
+                    ].startswith("#"):
+                        cast["mtype"] = cast["type"]
+                        cast["type"] = "query"
+                        # self.event(**castDup)
+                        # Don't call self.event for this one because queries are not events on a
+                        #  channel, but we still want to see them
+                    if cast["channel"] == "AUTH":
+                        cast["type"] = "conn"
+                        cast["mtype"] = cast["type"]
+
+            # TODO: better way to do this
+            # as we changed the types above, check again
+            if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
+                # we have been kicked
+                if "user" in cast.keys():
+                    if cast["user"].lower() == self.nickname.lower():
+                        castDup = deepcopy(cast)
+                        castDup["mtype"] = cast["type"]
+                        castDup["type"] = "self"
+                        if self.net in main.blacklist.keys():
+                            if cast["channel"] not in main.blacklist[self.net]:
+                                main.blacklist[self.net].append(cast["channel"])
+                        else:
+                            main.blacklist[self.net] = [cast["channel"]]
+                        main.saveConf("blacklist")
+                        self.event(**castDup)
+
+                # we sent a message/left/joined/kick someone/quit
+                if "nick" in cast.keys():
+                    if cast["nick"].lower() == self.nickname.lower():
+                        castDup = deepcopy(cast)
+                        castDup["mtype"] = cast["type"]
+                        castDup["type"] = "self"
+
+                # we have been mentioned in a msg/notice/action/part/quit/topic message
+                if "msg" in cast.keys():  # Don't highlight queries
+                    if cast["msg"] is not None:
+                        if self.nickname.lower() in cast["msg"].lower():
+                            castDup = deepcopy(cast)
+                            castDup["mtype"] = cast["type"]
+                            castDup["type"] = "highlight"
+                            self.event(**castDup)
+
+        if "net" not in cast.keys():
+            cast["net"] = self.net
+        if "num" not in cast.keys():
+            cast["num"] = self.num
+        if not self.authenticated:
+            regproc.registerTest(cast)
+        counters.event(self.net, cast["type"])
+        monitor.event(self.net, cast)
+
+    def privmsg(self, user, channel, msg):
+        self.event(type="msg", muser=user, channel=channel, msg=msg)
+
+    def noticed(self, user, channel, msg):
+        self.event(type="notice", muser=user, channel=channel, msg=msg)
+
+    def action(self, user, channel, msg):
+        self.event(type="action", muser=user, channel=channel, msg=msg)
+
+    def sendmsg(self, channel, msg, in_query=False):
+        query = f"{self.nickname}!*@*"
+        res = userinfo.getWhoSingle(self.net, query)
+        if not res:
+            res = []
+        us = list(res)
+        if len(us) > 0:
+            hostmask = us[0]
+        else:
+            # Close enough...
+            hostmask = f"{self.nickname}!{self.nickname}@{self.servername}"
+            warn(f"Could not get a hostname, using {hostmask}")
+        nick, ident, host = parsen(hostmask)
+        # We sent someone a query reply
+        if in_query:
+            self.event(
+                type="self",
+                mtype="msg",
+                channel=self.nickname,
+                nick=channel,
+                ident=ident,
+                host=host,
+                msg=msg,
+            )
+        else:
+            self.event(
+                type="self",
+                mtype="msg",
+                channel=channel,
+                nick=self.nickname,
+                ident=ident,
+                host=host,
+                msg=msg,
+            )
+            self.event(
+                type="msg",
+                channel=channel,
+                nick=self.nickname,
+                ident=ident,
+                host=host,
+                msg=msg,
+            )
+        self.msg(channel, msg)
+
+    def get(self, var):
+        try:
+            result = getattr(self, var)
+        except AttributeError:
+            result = None
+        return result
+
+    def setNick(self, nickname):
+        self._attemptedNick = nickname
+        self.sendLine("NICK %s" % nickname)
+        self.nickname = nickname
+
+    def alterCollidedNick(self, nickname):
+        newnick = nickname + "_"
+        return newnick
+
+    def nickChanged(self, olduser, newnick):
+        self.nickname = newnick
+        self.event(type="self", mtype="nick", muser=olduser, user=newnick)
+
+    def irc_ERR_NICKNAMEINUSE(self, prefix, params):
+        self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
+        self.setNick(self._attemptedNick)
+
+    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
+        log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username))
+        sendAll(
+            "%s - %i: password mismatch as %s" % (self.net, self.num, self.username)
+        )
+
+    def _who(self, channel):
+        d = Deferred()
+        if channel not in self._tempWho:
+            self._tempWho[channel] = ([], [])
+        self._tempWho[channel][0].append(d)
+        self.sendLine("WHO %s" % channel)
+        return d
+
+    def who(self, channel):
+        self._who(channel).addCallback(self.got_who)
+
+    def irc_RPL_WHOREPLY(self, prefix, params):
+        channel = params[1]
+        ident = params[2]
+        host = params[3]
+        server = params[4]
+        nick = params[5]
+        status = params[6]
+        realname = params[7]
+        if channel not in self._tempWho:
+            return
+        n = self._tempWho[channel][1]
+        n.append([nick, nick, host, server, status, realname])
+        self.event(
+            type="who",
+            nick=nick,
+            ident=ident,
+            host=host,
+            realname=realname,
+            channel=channel,
+            server=server,
+            status=status,
+        )
+
+    def irc_RPL_ENDOFWHO(self, prefix, params):
+        channel = params[1]
+        if channel not in self._tempWho:
+            return
+        callbacks, info = self._tempWho[channel]
+        for cb in callbacks:
+            cb.callback((channel, info))
+        del self._tempWho[channel]
+
+    def got_who(self, whoinfo):
+        userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
+
+    def sanit(self, data):
+        if len(data) >= 1:
+            if data[0] in self.prefix.keys():
+                return (
+                    self.prefix[data[0]],
+                    data[1:],
+                )  # would use a set but it's possible these are the same
+            return (None, data)
+        else:
+            return (None, False)
+
+    def names(self, channel):
+        d = Deferred()
+        if channel not in self._tempNames:
+            self._tempNames[channel] = ([], [])
+        self._tempNames[channel][0].append(d)
+        self.sendLine("NAMES %s" % channel)
+        return d
+
+    def irc_RPL_NAMREPLY(self, prefix, params):
+        channel = params[2]
+        nicklist = params[3].split(" ")
+        if channel not in self._tempNames:
+            return
+        n = self._tempNames[channel][1]
+        n.append(nicklist)
+
+    def irc_RPL_ENDOFNAMES(self, prefix, params):
+        channel = params[1]
+        if channel not in self._tempNames:
+            return
+        callbacks, namelist = self._tempNames[channel]
+        for cb in callbacks:
+            cb.callback((channel, namelist))
+        del self._tempNames[channel]
+
+    def got_names(self, nicklist):
+        newNicklist = []
+        for i in nicklist[1]:
+            for x in i:
+                mode, nick = self.sanit(x)
+                if nick:
+                    newNicklist.append((mode, nick))
+        userinfo.initialNames(self.net, nicklist[0], newNicklist)
+
+    def myInfo(self, servername, version, umodes, cmodes):
+        self.servername = servername
+
+    def _list(self, noargs):
+        d = Deferred()
+        self._tempList = ([], [])
+        self._tempList[0].append(d)
+        if self.listSimple:
+            self.sendLine("LIST")
+            return d  # return early if we know what to do
+
+        if noargs:
+            self.sendLine("LIST")
+        else:
+            self.sendLine("LIST >0")
+        return d
+
+    def list(self, noargs=False, nocheck=False):
+        if not self.authenticated:
+            debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num))
+            return
+        if self.listAttempted:
+            debug(
+                "List request dropped, already asked for LIST - %s - %i"
+                % (self.net, self.num)
+            )
+            return
+        else:
+            self.listAttempted = True
+        if self.listOngoing:
+            debug(
+                "LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)
+            )
+            return
+        else:
+            if nocheck:
+                allRelays = True  # override the system - if this is
+            else:  # specified, we already did this
+                allRelays = chankeep.allRelaysActive(self.net)
+            if not allRelays:
+                self.wantList = True
+                debug("Not all relays were active for LIST request")
+                return
+            self._list(noargs).addCallback(self.got_list)
+
+    def irc_RPL_LISTSTART(self, prefix, params):
+        self.listAttempted = False
+        self.listOngoing = True
+        self.wantList = False
+
+    def irc_RPL_LIST(self, prefix, params):
+        channel = params[1]
+        users = params[2]
+        topic = params[3]
+        self._tempList[1].append([channel, users, topic])
+
+    def irc_RPL_LISTEND(self, prefix, params):
+        if (
+            not len(self._tempList[0]) > 0
+        ):  # there are no callbacks, can't do anything there
+            debug("We didn't ask for this LIST, discarding")
+            self._tempList[0].clear()
+            self._tempList[1].clear()
+            return
+        callbacks, info = self._tempList
+        self.listOngoing = False
+        for cb in callbacks:
+            cb.callback((info))
+        noResults = False
+        if len(self._tempList[1]) == 0:
+            noResults = True
+        self._tempList[0].clear()
+        self._tempList[1].clear()
+        if noResults:
+            if self.listRetried:
+                warn("LIST still empty after retry: %s - %i" % (self.net, self.num))
+                self.listRetried = False
+                return
+            else:
+                self.list(True)
+                self.listRetried = True
+        else:
+            if self.listRetried:
+                self.listRetried = False
+                debug(
+                    "List received after retry - defaulting to simple list syntax: %s - %i"
+                    % (self.net, self.num)
+                )
+                self.listSimple = True
+
+    def got_list(self, listinfo):
+        if len(listinfo) == 0:  # probably ngircd not supporting LIST >0
+            return
+        if main.config["ChanKeep"]["Enabled"]:
+            chankeep.initialList(self.net, self.num, listinfo)
+
+    def recheckList(self):
+        if not main.config["ChanKeep"]["Enabled"]:
+            return
+        allRelays = chankeep.allRelaysActive(self.net)
+        debug(f"recheckList() all relays for {self.net} {allRelays}")
+        if allRelays:
+            debug(f"recheckList() all relays active for {self.net}")
+            first_relay = helpers.get_first_relay(self.net)
+            debug(f"recheckList() first relay for {self.net}: {first_relay.num}")
+            if first_relay:
+                if first_relay.wantList is True:
+                    first_relay.list(nocheck=True)
+                    debug(
+                        f"recheckList() asking for a list for {self.net} after final relay {self.num} connected"
+                    )
+                else:
+                    debug(
+                        f"recheckList() first relay wantList is False for {self.net} ({first_relay.num})"
+                    )
+            # name = self.net + "1"
+
+        # if self.num == 1:  # Only one instance should do a list
+        if helpers.is_first_relay(self.net, self.num):
+            debug(f"recheckList() we are the first relay for {self.net} ({self.num})")
+            if self.chanlimit:
+                if allRelays:
+                    self.list()
+                    debug(
+                        f"recheckList() requested a list for {self.net} from {self.num}"
+                    )
+                else:
+                    debug(f"recheckList() not all relays active for {self.net}")
+                    self.wantList = True
+            else:
+                debug("recheckList() aborting LIST due to bad chanlimit")
+        self.checkChannels()
+
+    def seed_chanlimit(self, chanlimit):
+        if not main.network[self.net].relays[self.num][
+            "registered"
+        ]:  # TODO: add check for register request sent, only send it once
+            if main.config["AutoReg"]:
+                if not self.authenticated:
+                    self._regAttempt = reactor.callLater(
+                        5, regproc.registerAccount, self.net, self.num
+                    )
+                    # regproc.registerAccount(self.net, self.num)
+        try:
+            self.chanlimit = int(chanlimit)
+        except TypeError:
+            warn("Invalid chanlimit: %s" % chanlimit)
+        if self.chanlimit == 0:
+            self.chanlimit = 200  # don't take the piss if it's not limited
+        net_inst_chanlimit = self.netinst.chanlimit
+        if net_inst_chanlimit:
+            if self.chanlimit > net_inst_chanlimit:
+                self.chanlimit = net_inst_chanlimit
+                # warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}")
+
+        if not regproc.needToRegister(
+            self.net
+        ):  # if we need to register, only recheck on auth confirmation
+            if main.config["ChanKeep"]["Enabled"]:
+                self.recheckList()
+
+    def seed_prefix(self, prefix):
+        prefix = prefix.replace(")", "")
+        prefix = prefix.replace("(", "")
+        length = len(prefix)
+        half = int(length / 2)
+        prefixToMode = dict(zip(prefix[half:], prefix[:half]))
+        self.prefix = prefixToMode
+
+    def isupport(self, options):
+        interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"}
+        newOptions = {x for x in options if any(y in x for y in interested)}
+        if len(newOptions) == 0:
+            return
+        if not self.isconnected:
+            log("endpoint connected: %s - %i" % (self.net, self.num))
+            self.isconnected = True
+        for i in newOptions:
+            if i.startswith("PREFIX"):
+                if "=" in i:
+                    split = i.split("=")
+                    if len(split) == 2:
+                        prefix = split[1]
+                        self.seed_prefix(prefix)
+            elif i.startswith("CHANLIMIT"):
+                if ":" in i:
+                    split = i.split(":")
+                    if len(split) >= 2:
+                        chanlimit = split[1]
+                        self.seed_chanlimit(chanlimit)
+            elif i.startswith("MAXCHANNELS"):
+                if "=" in i:
+                    split = i.split("=")
+                    if len(split) == 2:
+                        chanlimit = split[1]
+                        self.seed_chanlimit(chanlimit)
+
+    # We need to override these functions as Twisted discards
+    #  the hostname and other useful information in the functions
+    #  that these call by default
+    def irc_JOIN(self, prefix, params):
+        nick = prefix.split("!")[0]
+        channel = params[-1]
+        if nick == self.nickname:
+            self.joined(channel)
+        else:
+            self.userJoined(prefix, channel)
+
+    def irc_PART(self, prefix, params):
+        nick = prefix.split("!")[0]
+        channel = params[0]
+        if len(params) >= 2:
+            message = params[1]
+        else:
+            message = None
+        if nick == self.nickname:
+            self.left(prefix, channel, message)
+        else:
+            self.userLeft(prefix, channel, message)
+
+    def irc_QUIT(self, prefix, params):
+        # nick = prefix.split("!")[0]
+        self.userQuit(prefix, params[0])
+
+    def irc_NICK(self, prefix, params):
+        nick = prefix.split("!", 1)[0]
+        if nick == self.nickname:
+            self.nickChanged(prefix, params[0])
+        else:
+            self.userRenamed(prefix, params[0])
+
+    def irc_KICK(self, prefix, params):
+        channel = params[0]
+        kicked = params[1]
+        message = params[-1]
+        # Checks on whether it was us that was kicked are done in userKicked
+        self.userKicked(kicked, channel, prefix, message)
+
+    def irc_TOPIC(self, prefix, params):
+        channel = params[0]
+        newtopic = params[1]
+        self.topicUpdated(prefix, channel, newtopic)
+
+    # End of Twisted hackery
+
+    def regPing(self, negativepass=None, reset=True):
+        if not main.config["AutoReg"]:
+            return
+        if self.authenticated:
+            return
+        if not regproc.needToAuth(self.net):
+            self.authenticated = True
+            return
+        sinst = regproc.substitute(self.net, self.num)
+        if not sinst:
+            error(f"regPing() {self.net}: registration ping failed for {self.num}")
+            return
+        if reset:
+            self._negativePass = None
+
+        debug(
+            f"regPing() {self.net} - {self.num}: _negativePass:{self._negativePass} negativepass:{negativepass}"
+        )
+        if self._negativePass is None:
+            # We have failed, the blacklisted message has been found
+            if negativepass is False:
+                self._negativePass = False
+                debug(
+                    (
+                        f"regPing() {self.net} - {self.num} not passed negative:checknegativemsg "
+                        f"check, {sinst['checknegativemsg']} present in message"
+                    )
+                )
+                return
+            # End of negative output reached with no blacklisted message
+            elif negativepass is True:
+                if self._negativePass is None:  # check if it's not failed yet
+                    self._negativePass = True
+                    debug(
+                        (
+                            f"regPing() {self.net} - {self.num} passed negative:checkendnegative "
+                            f"check, {sinst['checkendnegative']} present in message"
+                        )
+                    )
+                else:
+                    debug(
+                        f"regPing() {self.net}: negative registration check - {self.num}"
+                    )
+                    return
+            else:
+                if sinst["negative"]:
+                    self.msg(sinst["entity"], sinst["negativemsg"])
+                    debug(
+                        (
+                            f"regPing() {self.net}: sent negativemsg "
+                            f"'{sinst['negativemsg']}' to {sinst['entity']} - {self.num}"
+                        )
+                    )
+                    return
+                else:
+                    self._negativePass = (
+                        True  # if it's disabled, we still want the next block to run
+                    )
+
+        # Only run if negativepass has completed, or exempted as above
+        if self._negativePass:
+            if sinst["check"]:
+                if sinst["ping"]:
+                    self.msg(sinst["entity"], sinst["pingmsg"])
+                    debug(
+                        f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}"
+                    )
+                    return
+            else:
+                self.authenticated = True
+
+    def signedOn(self):
+        log("signed on: %s - %i" % (self.net, self.num))
+        ctime = str(datetime.now().isoformat())
+        sendRelayNotification(
+            {
+                "type": "conn",
+                "net": self.net,
+                "num": self.num,
+                "status": "signedon",
+                "ts": ctime,
+            }
+        )
+        if not self.authenticated:
+            reactor.callLater(10, self.regPing)
+
+    def setup_who_loop(self, channel):
+        # if main.config["Toggles"]["Who"]:
+        lc = LoopingCall(self.who, channel)
+        self._getWho[channel] = lc
+        intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
+        minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
+        interval = randint(minint, minint + intrange)
+        lc.start(interval)
+
+    def joined(self, channel):
+        if channel not in self.channels:
+            self.channels.append(channel)
+        self.names(channel).addCallback(self.got_names)
+        if main.config["Toggles"]["Who"]:
+            reactor.callLater(
+                main.config["Tweaks"]["Delays"]["WhoDelay"],
+                self.setup_who_loop,
+                channel,
+            )
+            # lc = LoopingCall(self.who, channel)
+            # self._getWho[channel] = lc
+            # intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
+            # minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
+            # interval = randint(minint, minint + intrange)
+            # lc.start(interval)
+
+    def botLeft(self, channel):
+        if channel in self.channels:
+            self.channels.remove(channel)
+        if channel in self._getWho.keys():
+            lc = self._getWho[channel]
+            lc.stop()
+            del self._getWho[channel]
+        userinfo.delChannels(
+            self.net, [channel]
+        )  # < we do not need to deduplicate this
+        # log("Can no longer cover %s, removing records" % channel)#  as it will only be matched once --
+        #  other bots have different nicknames so
+
+    def left(self, user, channel, message):  # even if they saw it, they wouldn't react
+        self.event(type="part", muser=user, channel=channel, msg=message)
+        self.botLeft(channel)
+
+    def userJoined(self, user, channel):
+        self.event(type="join", muser=user, channel=channel)
+
+    def userLeft(self, user, channel, message):
+        self.event(type="part", muser=user, channel=channel, msg=message)
+
+    def userQuit(self, user, quitMessage):
+        self.chanlessEvent({"type": "quit", "muser": user, "msg": quitMessage})
+
+    def userKicked(self, kickee, channel, kicker, message):
+        if kickee.lower() == self.nickname.lower():
+            self.botLeft(channel)
+        self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee)
+
+    def chanlessEvent(self, cast):
+        cast["ts"] = int(datetime.now().timestamp())
+        cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
+        if dedup(self.name, cast):  # Needs to be kept self.name until the dedup
+            #  function is converted to the new net, num
+            #  format
+            return  # stop right there sir!
+        chans = userinfo.getChanList(self.net, cast["nick"])
+        if chans is None:
+            error("No channels returned for chanless event: %s" % cast)
+            #    self.event(**cast) -- no, should NEVER happen
+            return
+        # getChansSingle returns all channels of the user, we only want to use
+        #  ones we have common with them
+        realChans = set(chans).intersection(set(self.channels))
+        for i in realChans:
+            cast["channel"] = i
+            self.event(**cast)
+
+    def userRenamed(self, oldname, newname):
+        self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
+
+    def topicUpdated(self, user, channel, newTopic):
+        self.event(type="topic", muser=user, channel=channel, msg=newTopic)
+
+    def modeChanged(self, user, channel, toset, modes, args):
+        argList = list(args)
+        modeList = [i for i in modes]
+        for a, m in zip(argList, modeList):
+            self.event(
+                type="mode",
+                muser=user,
+                channel=channel,
+                mode=m,
+                status=toset,
+                modearg=a,
+            )
+
+
+# TODO: strip out relay functionality
+class IRCBotFactory(ReconnectingClientFactory):
+    def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
+        if net is None:
+            self.num = num
+            self.net = None
+            self.name = "relay - %i" % num
+            self.relay = True
+        else:
+            self.name = net + str(num)
+            self.num = num
+            self.net = net
+            self.relay = False
+        self.client = None
+        self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
+        self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
+        self.factor = main.config["Tweaks"]["Delays"]["Factor"]
+        self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
+
+        self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
+
+    def buildProtocol(self, addr):
+        entry = IRCBot(self.net, self.num)
+        main.IRCPool[self.name] = entry
+
+        self.client = entry
+        return entry
+
+    def clientConnectionLost(self, connector, reason):
+        if not self.relay:
+            userinfo.delChannels(self.net, self.client.channels)
+        if self.client is not None:
+            self.client.isconnected = False
+            self.client.authenticated = False
+            self.client.channels = []
+        error = reason.getErrorMessage()
+        log("%s - %i: connection lost: %s" % (self.net, self.num, error))
+        sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
+        ctime = str(datetime.now().isoformat())
+        sendRelayNotification(
+            {
+                "type": "conn",
+                "net": self.net,
+                "num": self.num,
+                "status": "lost",
+                "message": error,
+                "ts": ctime,
+            }
+        )
+        self.retry(connector)
+        # ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
+
+    def clientConnectionFailed(self, connector, reason):
+        if self.client is not None:
+            self.client.isconnected = False
+            self.client.authenticated = False
+            self.client.channels = []
+        error = reason.getErrorMessage()
+        log("%s - %i: connection failed: %s" % (self.net, self.num, error))
+        if not self.relay:
+            sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
+            ctime = str(datetime.now().isoformat())
+            sendRelayNotification(
+                {
+                    "type": "conn",
+                    "net": self.net,
+                    "num": self.num,
+                    "status": "failed",
+                    "message": error,
+                    "ts": ctime,
+                }
+            )
+            self.retry(connector)
+        # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

legacy/core/parser.py

@@ -0,0 +1,27 @@
+import main
+from utils.logging.log import warn
+from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess
+
+
+def parseCommand(addr, authed, data):
+    # call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
+    spl = data.split()
+    if addr in main.connections.keys():
+        obj = main.connections[addr]
+    else:
+        warn("Got connection object with no instance in the address pool")
+        return
+
+    success = lambda data: sendSuccess(addr, data)  # noqa: E731
+    failure = lambda data: sendFailure(addr, data)  # noqa: E731
+    info = lambda data: sendInfo(addr, data)  # noqa: E731
+
+    incUsage = lambda mode: incorrectUsage(addr, mode)  # noqa: E731
+    length = len(spl)
+    if spl[0] in main.CommandMap.keys():
+        main.CommandMap[spl[0]](
+            addr, authed, data, obj, spl, success, failure, info, incUsage, length
+        )
+        return
+    incUsage(None)
+    return

legacy/core/relay.py

@@ -1,12 +1,28 @@
-from twisted.internet.protocol import Protocol, Factory, ClientFactory
 from json import dumps, loads
-from copy import deepcopy
-from datetime import datetime
 
 import main
-from utils.logging.log import *
+from twisted.internet.protocol import Factory, Protocol
+from utils.logging.log import log, warn
+
+validTypes = [
+    "msg",
+    "notice",
+    "action",
+    "who",
+    "part",
+    "join",
+    "kick",
+    "quit",
+    "nick",
+    "topic",
+    "mode",
+    "conn",
+    "znc",
+    "query",
+    "self",
+    "highlight",
+]
 
-validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"]
 
 class Relay(Protocol):
     def __init__(self, addr):
@@ -30,10 +46,10 @@ class Relay(Protocol):
         data = data.decode("utf-8", "replace")
         try:
             parsed = loads(data)
-        except:
+        except:  # noqa: E722
             self.sendErr("MALFORMED")
             return
-        if not "type" in parsed.keys():
+        if "type" not in parsed.keys():
             self.sendErr("NOTYPE")
             return
         if parsed["type"] == "hello":
@@ -70,7 +86,7 @@ class Relay(Protocol):
             self.sendErr("NOTLIST")
             return
         for i in lst:
-            if not i in validTypes:
+            if i not in validTypes:
                 self.sendErr("NONEXISTANT")
                 return
             if i in self.subscriptions:
@@ -85,10 +101,10 @@ class Relay(Protocol):
             self.sendErr("NOTLIST")
             return
         for i in lst:
-            if not i in validTypes:
+            if i not in validTypes:
                 self.sendErr("NONEXISTANT")
                 return
-            if not i in self.subscriptions:
+            if i not in self.subscriptions:
                 self.sendErr("NOTSUBSCRIBED")
                 return
             del self.subscriptions[i]
@@ -97,8 +113,13 @@ class Relay(Protocol):
 
     def handleHello(self, parsed):
         if parsed["key"] in main.tokens.keys():
-            if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay":
-                self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]})
+            if (
+                parsed["hello"] == main.tokens[parsed["key"]]["hello"]
+                and main.tokens[parsed["key"]]["usage"] == "relay"
+            ):
+                self.sendMsg(
+                    {"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}
+                )
                 self.authed = True
             else:
                 self.transport.loseConnection()
@@ -109,16 +130,20 @@ class Relay(Protocol):
 
     def connectionMade(self):
         log("Relay connection from %s:%s" % (self.addr.host, self.addr.port))
-        #self.send("Greetings.")
+        # self.send("Greetings.")
 
     def connectionLost(self, reason):
         self.authed = False
-        log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
+        log(
+            "Relay connection lost from %s:%s -- %s"
+            % (self.addr.host, self.addr.port, reason.getErrorMessage())
+        )
         if self.addr in main.relayConnections.keys():
             del main.relayConnections[self.addr]
         else:
             warn("Tried to remove a non-existant relay connection.")
 
+
 class RelayFactory(Factory):
     def buildProtocol(self, addr):
         entry = Relay(addr)
@@ -132,10 +157,9 @@ class RelayFactory(Factory):
         else:
             return
 
+
 def sendRelayNotification(cast):
     for i in main.relayConnections.keys():
         if main.relayConnections[i].authed:
             if cast["type"] in main.relayConnections[i].subscriptions:
-                newCast = deepcopy(cast)
-                newCast["time"] = str(datetime.now().isoformat())
-                main.relayConnections[i].send(dumps(newCast))
+                main.relayConnections[i].send(dumps(cast))

legacy/core/server.py

@@ -1,14 +1,14 @@
-from twisted.internet.protocol import Protocol, Factory, ClientFactory
 import main
-from utils.logging.log import *
-
 from core.parser import parseCommand
+from twisted.internet.protocol import Factory, Protocol
+from utils.logging.log import log, warn
+
 
 class Server(Protocol):
     def __init__(self, addr):
         self.addr = addr
         self.authed = False
-        if main.config["UsePassword"] == False:
+        if main.config["UsePassword"] is False:
             self.authed = True
 
     def send(self, data):
@@ -18,7 +18,7 @@ class Server(Protocol):
 
     def dataReceived(self, data):
         data = data.decode("utf-8", "replace")
-        #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
+        # log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
         if "\n" in data:
             splitData = [x for x in data.split("\n") if x]
             if "\n" in data:
@@ -33,13 +33,15 @@ class Server(Protocol):
 
     def connectionLost(self, reason):
         self.authed = False
-        log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
+        log(
+            "Connection lost from %s:%s -- %s"
+            % (self.addr.host, self.addr.port, reason.getErrorMessage())
+        )
         if self.addr in main.connections.keys():
             del main.connections[self.addr]
         else:
             warn("Tried to remove a non-existant connection.")
-        if self.addr in main.MonitorPool:
-            main.MonitorPool.remove(self.addr)
+
 
 class ServerFactory(Factory):
     def buildProtocol(self, addr):

legacy/docker-compose.yml

@@ -0,0 +1,41 @@
+version: "2"
+   
+services:
+  app:
+    image: pathogen/threshold:latest
+    build: ./docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${THRESHOLD_CONFIG_DIR}:/code/conf/live
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      - ${THRESHOLD_CERT_DIR}:/code/conf/cert
+    ports:
+      - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
+      - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
+      - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
+    env_file:
+      - .env
+    # for development
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes_from:
+        - tmp
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+        - /var/run/redis
+
+  redis:
+      image: redis
+      command: redis-server /etc/redis.conf
+      volumes:
+          - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      volumes_from:
+          - tmp
+
+networks:
+   default:
+      external:
+         name: pathogen

legacy/docker/Dockerfile

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+
+RUN useradd -d /code pathogen
+RUN mkdir /code
+RUN chown pathogen:pathogen /code
+
+RUN mkdir /venv
+RUN chown pathogen:pathogen /venv
+
+USER pathogen
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code/legacy
+COPY requirements.prod.txt /code/legacy
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r /code/legacy/requirements.prod.txt
+CMD . /venv/bin/activate && exec python /code/legacy/threshold

legacy/docker/docker-compose.prod.yml

@@ -0,0 +1,38 @@
+version: "2"
+   
+services:
+  app:
+    image: pathogen/threshold:latest
+    build: ./docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${THRESHOLD_CONFIG_DIR}:/code/conf/live
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      - ${THRESHOLD_CERT_DIR}:/code/conf/cert
+    ports:
+      - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
+      - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
+      - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
+    env_file:
+      - ../stack.env
+    volumes_from:
+        - tmp
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+        - /var/run/redis
+
+  redis:
+      image: redis
+      command: redis-server /etc/redis.conf
+      volumes:
+          - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      volumes_from:
+          - tmp
+
+networks:
+   default:
+      external:
+         name: pathogen

legacy/docker/requirements.prod.txt

@@ -0,0 +1,8 @@
+wheel
+twisted
+pyOpenSSL
+redis
+pyYaML
+service_identity
+siphashc
+Klein

legacy/main.py

@@ -0,0 +1,131 @@
+import json
+import pickle
+from os import urandom
+from os.path import exists
+from string import digits
+
+from redis import StrictRedis
+
+# List of errors ZNC can give us
+ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
+
+configPath = "conf/live/"
+templateConfigPath = "conf/templates/"
+certPath = "conf/cert/"
+
+filemap = {
+    # JSON configs
+    "config": ["config.json", "configuration", "json"],
+    "help": ["help.json", "command help", "json"],
+    "counters": ["counters.json", "counters file", "json"],
+    "tokens": ["tokens.json", "authentication tokens", "json"],
+    "aliasdata": ["aliasdata.json", "data for alias generation", "json"],
+    "alias": ["alias.json", "provisioned alias data", "json"],
+    "irc": ["irc.json", "IRC network definitions", "json"],
+    "blacklist": ["blacklist.json", "IRC channel blacklist", "json"],
+    # Binary (pickle) configs
+    "network": ["network.dat", "network list", "pickle"],
+}
+
+# Connections to the plain-text interface
+connections = {}
+# Connections to the JSON interface
+relayConnections = {}
+
+# Mapping of network names to Protocol (IRCClient) instances
+IRCPool = {}
+
+# Mapping of network names to Reactor instances
+# Needed for calling .disconnect()
+ReactorPool = {}
+
+# Mapping of network names to Factory instances
+# Needed for calling .stopTrying()
+FactoryPool = {}
+
+# Temporary store for channels allocated after a LIST
+# Will get purged as the instances fire up and pop() from
+#  their respective keys in here
+TempChan = {}
+
+# Mapping of command names to their functions
+CommandMap = {}
+
+# Incremented by 1 for each event reaching modules.counters.event()
+#  and cloned into lastMinuteSample every minute
+runningSample = 0
+lastMinuteSample = 0
+
+# Generate 16-byte hex key for message checksums
+hashKey = urandom(16)
+lastEvents = {}
+
+
+# Get networks that are currently online and dedupliate
+def liveNets():
+    networks = set()
+    for i in IRCPool.keys():
+        networks.add("".join([x for x in i if x not in digits]))
+    return networks
+
+
+def saveConf(var):
+    if var in ("help", "aliasdata"):
+        return  # no need to save this
+    if filemap[var][2] == "json":
+        with open(configPath + filemap[var][0], "w") as f:
+            json.dump(globals()[var], f, indent=4)
+    elif filemap[var][2] == "pickle":
+        with open(configPath + filemap[var][0], "wb") as f:
+            pickle.dump(globals()[var], f)
+    else:
+        raise Exception("invalid format")
+
+
+def loadConf(var):
+    if filemap[var][2] == "json":
+        filename = configPath + filemap[var][0]
+        # Only take the help from the templates
+        if var in ("help", "aliasdata"):
+            filename = templateConfigPath + filemap[var][0]
+        if not exists(filename):
+            # Load the template config
+            if var == "config":
+                filename = templateConfigPath + filemap[var][0]
+            else:
+                # Everything else should be blank
+                globals()[var] = {}
+                return
+        with open(filename, "r") as f:
+            globals()[var] = json.load(f)
+        if var == "alias":
+            # This is a workaround to convert all the keys into integers since JSON
+            #  turns them into strings...
+            #  Dammit Jason!
+            global alias
+            alias = {int(x): y for x, y in alias.items()}
+
+    elif filemap[var][2] == "pickle":
+        try:
+            with open(configPath + filemap[var][0], "rb") as f:
+                globals()[var] = pickle.load(f)
+        except FileNotFoundError:
+            globals()[var] = {}
+    else:
+        raise Exception("invalid format")
+
+
+def initConf():
+    for i in filemap.keys():
+        loadConf(i)
+
+
+def initMain():
+    global r, g
+    initConf()
+    r = StrictRedis(
+        unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]  # noqa
+    )  # Ephemeral - flushed on quit
+    g = StrictRedis(
+        unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]
+    )  # noqa

legacy/modules/alias.py

@@ -1,10 +1,13 @@
-import main
 import random
 import re
 
+import main
+
+
 def generate_password():
     return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
 
+
 def generate_alias():
     nick = random.choice(main.aliasdata["stubs"])
     rand = random.randint(1, 2)
@@ -12,8 +15,8 @@ def generate_alias():
         nick = nick.capitalize()
     rand = random.randint(1, 4)
     while rand == 1:
-        split = random.randint(0, len(nick)-1)
-        nick = nick[:split] + nick[split+1:]
+        split = random.randint(0, len(nick) - 1)
+        nick = nick[:split] + nick[split + 1 :]  # noqa: E203
         rand = random.randint(1, 4)
     rand = random.randint(1, 3)
     if rand == 1 or rand == 4:
@@ -51,7 +54,7 @@ def generate_alias():
         ident = namebase.split(" ")[0]
         ident = ident[:10]
     elif rand == 6:
-        ident = re.sub("\s", "", namebase).lower()
+        ident = re.sub("\s", "", namebase).lower()  # noqa: W605
         ident = ident[:10]
 
     realname = nick
@@ -63,6 +66,10 @@ def generate_alias():
     if rand == 3 or rand == 4:
         realname = realname.capitalize()
 
-    password = generate_password()
-
-    return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password}
+    return {
+        "nick": nick,
+        "altnick": altnick,
+        "ident": ident,
+        "realname": realname,
+        "emails": [],
+    }

legacy/modules/chankeep.py

@@ -0,0 +1,600 @@
+from copy import deepcopy
+from math import ceil
+
+import main
+from modules import helpers
+from twisted.internet.threads import deferToThread
+from utils.logging.debug import debug, trace
+from utils.logging.log import error, log, warn
+
+
+def getAllChannels(net=None):
+    """
+    Get a list of all channels on all relays.
+    :return: list of channels
+    """
+    channels = {}
+    if not net:
+        nets = main.network.keys()
+    else:
+        nets = [net]
+    for net in nets:
+        relays = helpers.get_connected_relays(net)
+        for relay in relays:
+            if net not in channels:
+                channels[net] = {}
+            if relay.num not in channels[net]:
+                channels[net][relay.num] = []
+            for channel in relay.channels:
+                channels[net][relay.num].append(channel)
+    # debug(f"getAllChannels(): {channels}")
+    return channels
+
+
+def getDuplicateChannels(net=None, total=False):
+    """
+    Get a list of duplicate channels.
+    :return: list of duplicate channels
+    """
+    allChans = getAllChannels(net)
+    duplicates = {}
+    for net in allChans.keys():
+        net_chans = []
+        inst = {}
+        # add all the channels from this network to a list
+        for num in allChans[net].keys():
+            net_chans.extend(allChans[net][num])
+        for channel in net_chans:
+            count_chan = net_chans.count(channel)
+            # I don't know why but it works
+            # this is used in userinfo.delChannels
+            set_min = 1
+            if total:
+                set_min = 0
+            if count_chan > set_min:
+                inst[channel] = count_chan
+        if inst:
+            duplicates[net] = inst
+
+    if total:
+        return duplicates
+
+    to_part = {}
+    for net in allChans:
+        if net in duplicates:
+            for num in allChans[net].keys():
+                for channel in allChans[net][num]:
+                    if channel in duplicates[net].keys():
+                        if duplicates[net][channel] > 1:
+                            if net not in to_part:
+                                to_part[net] = {}
+                            if num not in to_part[net]:
+                                to_part[net][num] = []
+                            to_part[net][num].append(channel)
+                            duplicates[net][channel] -= 1
+
+    return to_part
+
+
+def partChannels(data):
+    for net in data:
+        for num in data[net]:
+            name = f"{net}{num}"
+            if name in main.IRCPool.keys():
+                for channel in data[net][num]:
+                    if channel in main.IRCPool[name].channels:
+                        main.IRCPool[name].part(channel)
+                        log(f"Parted {channel} on {net} - {num}")
+
+
+def getEnabledRelays(net):
+    """
+    Get a list of enabled relays for a network.
+    :param net: network
+    :rtype: list of int
+    :return: list of enabled relay numbers
+    """
+    enabledRelays = [
+        x
+        for x in main.network[net].relays.keys()
+        if main.network[net].relays[x]["enabled"]
+    ]
+    # debug(f"getEnabledRelays() {net}: {enabledRelays}")
+    return enabledRelays
+
+
+def getConnectedRelays(net):
+    """
+    Get a list of connected relays for a network.
+    :param net: network
+    :rtype: list of int
+    :return: list of relay numbers
+    """
+    enabledRelays = getEnabledRelays(net)
+    connectedRelays = []
+    for i in enabledRelays:
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            if main.IRCPool[name].isconnected:
+                connectedRelays.append(i)
+    # debug(f"getConnectedRelays() {net}: {connectedRelays}")
+    return connectedRelays
+
+
+def getActiveRelays(net):
+    """
+    Get a list of active relays for a network.
+    :param net: network
+    :rtype: list of int
+    :return: list of relay numbers
+    """
+    enabledRelays = getEnabledRelays(net)
+    activeRelays = []
+    for i in enabledRelays:
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            # debug(
+            #     (
+            #         f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
+            #         f"conn:{main.IRCPool[name].isconnected}"
+            #     )
+            # )
+            if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
+                activeRelays.append(i)
+    debug(f"getActiveRelays() {net}: {activeRelays}")
+    return activeRelays
+
+
+def relayIsActive(net, num):
+    """
+    Check if a relay is active.
+    :param net: network
+    :param num: relay number
+    :rtype: bool
+    :return: True if relay is active, False otherwise
+    """
+    activeRelays = getActiveRelays(net)
+    return num in activeRelays
+
+
+def allRelaysActive(net):
+    """
+    Check if all enabled relays are active and authenticated.
+    :param net: network
+    :rtype: bool
+    :return: True if all relays are active and authenticated, False otherwise
+    """
+    activeRelays = getActiveRelays(net)
+    enabledRelays = getEnabledRelays(net)
+    relaysActive = len(activeRelays) == len(enabledRelays)
+    # debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
+    return relaysActive
+
+
+def getAverageChanlimit(net):
+    """
+    Get the average channel limit for a network.
+    :param net: network
+    :rtype: int
+    :return: average channel limit
+    """
+    total = 0
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            total += main.IRCPool[name].chanlimit
+    avg_chanlimit = total / len(getActiveRelays(net))
+    debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
+    return avg_chanlimit
+
+
+def getSumChanlimit(net):
+    """
+    Get the sum of all channel limits for a network.
+    :param net: network
+    :rtype: int
+    :return: sum of channel limits
+    """
+    total = 0
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            total += main.IRCPool[name].chanlimit
+    return total
+
+
+def getChanFree(net):
+    """
+    Get a dictionary with the free channel spaces for
+    each relay, and a channel limit.
+    Example return:
+     ({1: 99}, 100)
+    :param net: network
+    :return: ({relay: channel spaces}, channel limit)
+    """
+    chanfree = {}
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name not in main.IRCPool.keys():
+            continue
+        if not main.IRCPool[name].isconnected:
+            continue
+        chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
+
+    return chanfree
+
+
+def getTotalChans(net):
+    """
+    Get the total number of channels on all relays for a network.
+    :param net: network
+    :rtype: int
+    :return: total number of channels
+    """
+    total = 0
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            total += len(main.IRCPool[name].channels)
+    return total
+
+
+def emptyChanAllocate(net, flist):
+    """
+    Allocate channels to relays.
+    :param net: network
+    :param flist: list of channels to allocate
+    :param new: list of newly provisioned relays to account for
+    :rtype: dict
+    :return: dictionary of {relay: list of channels}"""
+
+    # Get the free channel spaces for each relay
+    chanfree = getChanFree(net)
+    if not chanfree:
+        return
+
+    # Pretend the newly provisioned relays are already on the network
+    # for i in new:
+    #     chanfree[0][i] = chanfree[1]
+    allocated = {}
+
+    newlist = list(flist)
+    chan_slots_used = getTotalChans(net)
+    max_chans = getSumChanlimit(net) - chan_slots_used
+    trunc_list = newlist[:max_chans]
+    debug(
+        f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}"
+    )
+
+    for i in chanfree.keys():
+        for x in range(chanfree[i]):
+            if not len(trunc_list):
+                break
+            if i in allocated.keys():
+                allocated[i].append(trunc_list.pop())
+            else:
+                allocated[i] = [trunc_list.pop()]
+    return allocated
+
+
+def populateChans(net, clist):
+    """
+    Populate channels on relays.
+    Stores channels to join in a list in main.TempChan[net][num]
+    :param net: network
+    :param clist: list of channels to join
+    :param new: list of newly provisioned relays to account for"""
+    # divided = array_split(clist, relay)
+    allocated = emptyChanAllocate(net, clist)
+    trace(f"populateChans() allocated:{allocated}")
+    if not allocated:
+        return
+    for i in allocated.keys():
+        if net in main.TempChan.keys():
+            main.TempChan[net][i] = allocated[i]
+        else:
+            main.TempChan[net] = {i: allocated[i]}
+        trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
+
+
+def notifyJoin(net):
+    """
+    Notify relays to join channels.
+    They will pull from main.TempChan and remove channels they join.
+    :param net: network
+    """
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            trace(f"notifyJoin() {name}")
+            main.IRCPool[name].checkChannels()
+
+
+def minifyChans(net, listinfo, as_list=False):
+    """
+    Remove channels from listinfo that are already covered by a relay.
+    :param net: network
+    :param listinfo: list of channels to check
+    :type listinfo: list of [channel, num_users]
+    :return: list of channels with joined channels removed
+    :rtype: list of [channel, num_users]
+    """
+    # We want to make this reusable for joining a bunch of channels.
+    if as_list:
+        channel_list = listinfo
+
+    if not allRelaysActive(net):
+        error("All relays for %s are not active, cannot minify list" % net)
+        return False
+    for i in getConnectedRelays(net):
+        name = net + str(i)
+        for x in main.IRCPool[name].channels:
+            if as_list:
+                for y in channel_list:
+                    if y == x:
+                        channel_list.remove(y)
+            else:
+                for y in listinfo:
+                    if y[0] == x:
+                        listinfo.remove(y)
+    if not as_list:
+        if not listinfo:
+            log("We're on all the channels we want to be on, dropping LIST")
+            return False
+    if as_list:
+        return channel_list
+    else:
+        return listinfo
+
+
+def keepChannels(net, listinfo, mean, sigrelay, relay):
+    """
+    Minify channels, determine whether we can cover all the channels
+    on the network, or need to use 'significant' mode.
+    Truncate the channel list to available channel spaces.
+    Allocate these channels to relays.
+    Notify relays that they should pull from TempChan to join.
+    :param net: network
+    :param listinfo: list of [channel, num_users] lists
+    :param mean: mean of channel population
+    :param sigrelay: number of relays needed to cover significant channels
+    :param relay: number of relays needed to cover all channels
+    :param chanlimit: maximum number of channels to allocate to a relay
+    """
+    listinfo = minifyChans(net, listinfo)
+    if not listinfo:
+        return
+    if (
+        relay <= main.config["ChanKeep"]["SigSwitch"]
+    ):  # we can cover all of the channels
+        coverAll = True
+    elif (
+        relay > main.config["ChanKeep"]["SigSwitch"]
+    ):  # we cannot cover all of the channels
+        coverAll = False
+    #    if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
+    #        error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
+    #        return
+    num_instances = len(getActiveRelays(net))
+    debug(f"keepChannels() {net} instances:{num_instances}")
+    chan_slots_used = getTotalChans(net)
+    debug(f"keepChannels() slots_used:{chan_slots_used}")
+    # max_chans = (chanlimit * num_instances) - chan_slots_used
+    max_chans = getSumChanlimit(net) - chan_slots_used
+    if max_chans < 0:
+        max_chans = 0
+    debug(f"keepChannels() max_chans:{max_chans}")
+    if coverAll:
+        # needed = relay - len(getActiveRelays(net))
+        # if needed:
+        #     debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
+        #     newNums = modules.provision.provisionMultipleRelays(net, needed)
+        # else:
+        #     newNums = []
+        listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
+        if len(listinfo_sort) > max_chans:
+            max_chans = len(listinfo_sort) - 1
+
+        flist = [i[0] for i in listinfo_sort]
+
+        flist = flist[:max_chans]
+        debug(
+            f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels"
+        )
+        trace(f"keepChannels() {net}: joining:{flist}")
+        populateChans(net, flist)
+    else:
+        # needed = sigrelay - len(getActiveRelays(net))
+        # if needed:
+        #     debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
+        #     newNums = modules.provision.provisionMultipleRelays(net, needed)
+        # else:
+        #     newNums = []
+        listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
+        trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
+        if len(listinfo_sort) > max_chans:
+            max_chans = len(listinfo_sort) - 1
+        debug(f"keepChannels() {net}: new max_chans:{max_chans}")
+
+        siglist = [i[0] for i in listinfo if int(i[1]) > mean]
+        trace(f"keepChannels() {net}: new siglist:{siglist}")
+
+        siglist = siglist[:max_chans]
+        trace(f"keepChannels() {net}: truncated siglist:{siglist}")
+
+        trace(
+            f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}"
+        )
+        debug(
+            f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels"
+        )
+        trace(f"keepChannels() {net}: joining:{siglist}")
+        populateChans(net, siglist)
+    notifyJoin(net)
+
+
+def joinSingle(net, channel):
+    """
+    Join a channel on a relay.
+    Use ECA to determine which relay to join on.
+    :param net: network
+    :param channel: channel to join
+    :return: relay number that joined the channel
+    :rtype: int
+    """
+    if "," in channel:
+        channels = channel.split(",")
+        channels = minifyChans(net, channels, as_list=True)
+
+    else:
+        channels = [channel]
+    populateChans(net, channels)
+    notifyJoin(net)
+    return True
+
+
+def partSingle(net, channel):
+    """
+    Iterate over all the relays of net and part channels matching channel.
+    :param net: network
+    :param channel: channel to part
+    :return: list of relays that parted the channel
+    :rtype: list of str
+    """
+    parted = []
+    for i in getConnectedRelays(net):
+        name = f"{net}{i}"
+        if name in main.IRCPool.keys():
+            if channel in main.IRCPool[name].channels:
+                main.IRCPool[name].part(channel)
+                parted.append(str(i))
+    return parted
+
+
+def nukeNetwork(net):
+    """
+    Remove network records.
+    :param net: network"""
+    # purgeRecords(net)
+    # p = main.g.pipeline()
+    main.g.delete("analytics.list." + net)
+    # p.delete("list."+net)
+    # p.execute()
+
+
+# def nukeNetwork(net):
+#    deferToThread(_nukeNetwork, net)
+
+
+def _initialList(net, num, listinfo):
+    """
+    Called when a relay receives a full LIST response.
+    Run statistics to determine how many channels are significant.
+    This is done by adding all the numbers of users on the channels together,
+    then dividing by the number of channels.
+    * cumul - cumulative sum of all channel membership
+    * siglength - number of significant channels
+    * listlength - number of channels in the list
+    * sigrelay - number of relays needed to cover siglength
+    * relay - number of relays needed to cover all channels
+    :param net: network
+    :param num: relay number
+    :param listinfo: list of [channel, num_users] lists
+    :param chanlimit: maximum number of channels the relay can join
+    """
+    listlength = len(listinfo)
+    cumul = 0
+    try:
+        cumul += sum(int(i[1]) for i in listinfo)
+    except TypeError:
+        warn("Bad LIST data received from %s - %i" % (net, num))
+        return
+    mean = round(cumul / listlength, 2)
+    siglength = 0
+    insiglength = 0
+    sigcumul = 0
+    insigcumul = 0
+    for i in listinfo:
+        if int(i[1]) > mean:
+            siglength += 1
+            sigcumul += int(i[1])
+        elif int(i[1]) < mean:
+            insiglength += 1
+            insigcumul += int(i[1])
+
+    avg_chanlimit = getAverageChanlimit(net)
+    sigrelay = ceil(siglength / avg_chanlimit)
+    relay = ceil(listlength / avg_chanlimit)
+
+    cur_relays = len(getActiveRelays(net))
+    sig_relays_missing = sigrelay - cur_relays
+    all_relays_missing = relay - cur_relays
+
+    abase = "analytics.list.%s" % net
+    main.g.delete(abase)
+    p = main.g.pipeline()
+
+    # See docstring for meanings
+    p.hset(abase, "mean", mean)
+    p.hset(abase, "total_chans", listlength)
+    p.hset(abase, "big_chans", siglength)
+    p.hset(abase, "small_chans", insiglength)
+    p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
+    p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
+    p.hset(abase, "total_cumul_mem", cumul)
+    p.hset(abase, "big_chan_cumul_mem", sigcumul)
+    p.hset(abase, "small_chan_cumul_mem", insigcumul)
+    p.hset(abase, "relays_for_all_chans", relay)
+    p.hset(abase, "relays_for_big_chans", sigrelay)
+    p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
+    p.hset(abase, "sig_relays_missing", sig_relays_missing)
+    p.hset(abase, "all_relays_missing", all_relays_missing)
+    debug(
+        (
+            f"_initialList() net:{net} num:{num} listlength:{listlength} "
+            f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
+            f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
+        )
+    )
+
+    # Purge existing records before writing
+    # purgeRecords(net)
+    # for i in listinfo:
+    #    p.rpush(netbase+"."+i[0], i[1])
+    #    p.rpush(netbase+"."+i[0], i[2])
+    #    p.sadd(netbase, i[0])
+
+    p.execute()
+    debug("List parsing completed on %s" % net)
+    keepChannels(net, listinfo, mean, sigrelay, relay)
+
+    # return (listinfo, mean, sigrelay, relay)
+
+
+def convert(data):
+    """
+    Recursively convert a dictionary.
+    """
+    if isinstance(data, bytes):
+        return data.decode("ascii")
+    if isinstance(data, dict):
+        return dict(map(convert, data.items()))
+    if isinstance(data, tuple):
+        return map(convert, data)
+    if isinstance(data, list):
+        return list(map(convert, data))
+    return data
+
+
+def getListInfo(net):
+    abase = f"analytics.list.{net}"
+    info = main.g.hgetall(abase)
+    return convert(info)
+
+
+def initialList(net, num, listinfo):
+    """
+    Run _initialList in a thread.
+    See above docstring.
+    """
+    deferToThread(_initialList, net, num, deepcopy(listinfo))

legacy/modules/counters.py

@@ -1,12 +1,13 @@
 import main
 from twisted.internet.task import LoopingCall
 
+
 def event(name, eventType):
-    if not "local" in main.counters.keys():
+    if "local" not in main.counters.keys():
         main.counters["local"] = {}
-    if not "global" in main.counters.keys():
+    if "global" not in main.counters.keys():
         main.counters["global"] = {}
-    if not name in main.counters["local"].keys():
+    if name not in main.counters["local"].keys():
         main.counters["local"][name] = {}
     if eventType not in main.counters["local"][name].keys():
         main.counters["local"][name][eventType] = 0
@@ -18,8 +19,9 @@ def event(name, eventType):
     main.counters["global"][eventType] += 1
     main.runningSample += 1
 
+
 def getEvents(name=None):
-    if name == None:
+    if name is None:
         if "global" in main.counters.keys():
             return main.counters["global"]
         else:
@@ -30,10 +32,12 @@ def getEvents(name=None):
         else:
             return None
 
+
 def takeSample():
     main.lastMinuteSample = main.runningSample
     main.runningSample = 0
 
+
 def setupCounterLoop():
     lc = LoopingCall(takeSample)
     lc.start(60)

legacy/modules/helpers.py

@@ -0,0 +1,72 @@
+import main
+from modules import chankeep
+from utils.logging.debug import debug
+
+
+def get_first_relay(net):
+    """
+    Get the first relay in the network.
+    :param net: the network
+    :param num: number or relay
+    :return: IRCPool instance for the IRC bot
+    """
+    cur_relay = 0
+    max_relay = max(main.network[net].relays.keys())
+    debug(f"get_first_relay() {net}: max_relay:{max_relay}")
+    activeRelays = chankeep.getActiveRelays(net)
+
+    debug(f"get_first_relay() {net}: activeRelays:{activeRelays}")
+    while cur_relay != max_relay:
+        cur_relay += 1
+        if cur_relay not in activeRelays:
+            continue
+        name = net + str(cur_relay)
+        if name in main.IRCPool.keys():
+            # debug(f"get_first_relay() {net}: found relay {name}")
+            return main.IRCPool[name]
+    return None
+
+
+def is_first_relay(net, num):
+    """
+    Determine if we are the first relay for the network.
+    :param net: the network
+    :param num: number or relay
+    :return: True if we are the first relay, False otherwise
+    """
+    first_relay = get_first_relay(net)
+    if not first_relay:
+        return False
+    return first_relay.num == num
+
+
+def get_active_relays(net):
+    """
+    Get all active instances for the network.
+    :param net: the network
+    :return: list of active instances
+    :rtype: list of IRCPool instances
+    """
+    active_nums = chankeep.getActiveRelays(net)
+    active_insts = []
+    for num in active_nums:
+        name = net + str(num)
+        if name in main.IRCPool.keys():
+            active_insts.append(main.IRCPool[name])
+    return active_insts
+
+
+def get_connected_relays(net):
+    """
+    Get all connected instances for the network.
+    :param net: the network
+    :return: list of active instances
+    :rtype: list of IRCPool instances
+    """
+    active_nums = chankeep.getConnectedRelays(net)
+    active_insts = []
+    for num in active_nums:
+        name = net + str(num)
+        if name in main.IRCPool.keys():
+            active_insts.append(main.IRCPool[name])
+    return active_insts

legacy/modules/monitor.py

@@ -0,0 +1,89 @@
+import json
+
+import main
+from core.relay import sendRelayNotification
+from modules import userinfo
+from utils.dedup import dedup
+
+order = [
+    "type",
+    "net",
+    "num",
+    "channel",
+    "msg",
+    "nick",
+    "ident",
+    "host",
+    "mtype",
+    "user",
+    "mode",
+    "modearg",
+    "realname",
+    "server",
+    "status",
+    "ts",
+]
+
+
+def parsemeta(numName, c):
+    if "channel" not in c.keys():
+        c["channel"] = None
+    # metadata scraping
+    # need to check if this was received from a relay
+    #  in which case, do not do this
+    if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
+        if "muser" in c.keys():
+            userinfo.editUser(c["net"], c["muser"])
+        # if c["type"] == "mode":
+        #    userinfo.updateMode(c)
+    elif c["type"] == "nick":
+        userinfo.renameUser(
+            c["net"],
+            c["nick"],
+            c["muser"],
+            c["user"],
+            c["user"] + "!" + c["ident"] + "@" + c["host"],
+        )
+    elif c["type"] == "kick":
+        userinfo.editUser(c["net"], c["muser"])
+        userinfo.delUserByNick(c["net"], c["channel"], c["user"])
+    elif c["type"] == "quit":
+        userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"])
+    elif c["type"] == "join":
+        userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"])
+    elif c["type"] == "part":
+        userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"])
+
+    if "mtype" in c.keys():
+        if c["mtype"] == "nick":
+            userinfo.renameUser(
+                c["net"],
+                c["nick"],
+                c["muser"],
+                c["user"],
+                c["user"] + "!" + c["ident"] + "@" + c["host"],
+            )
+
+
+def queue_message(c):
+    message = json.dumps(c)
+    main.g.sadd("queue", message)
+
+
+def event(
+    numName, c
+):  # yes I'm using a short variable because otherwise it goes off the screen
+    if dedup(numName, c):
+        return
+
+    # make a copy of the object with dict() to prevent sending notifications with channel of None
+    parsemeta(numName, dict(c))
+
+    if "muser" in c.keys():
+        del c["muser"]
+    # sortedKeys = {k: c[k] for k in order if k in c}  # Sort dict keys according to order
+    # sortedKeys["src"] = "irc"
+    c["src"] = "irc"
+    if main.config["Ingest"]["Enabled"]:
+        queue_message(c)
+    sendRelayNotification(c)

legacy/modules/network.py

@@ -0,0 +1,163 @@
+from copy import deepcopy
+
+import main
+from core.bot import IRCBotFactory
+from modules import alias
+from modules.chankeep import nukeNetwork
+from modules.provision import provisionRelay
+from modules.regproc import needToRegister
+from twisted.internet import reactor
+from twisted.internet.ssl import DefaultOpenSSLContextFactory
+from utils.deliver_relay_commands import deliverRelayCommands
+from utils.get import getRelay
+from utils.logging.log import log
+
+
+def migrate():
+    existing = deepcopy(main.network)
+    log("Migrating network configuration")
+    log(f"Existing network configuration: {existing}")
+    for net, net_inst in existing.items():
+        log(f"Migrating network {net}")
+        net = net_inst.net
+        host = net_inst.host
+        port = net_inst.port
+        security = net_inst.security
+        auth = net_inst.auth
+        last = net_inst.last
+        relays = net_inst.relays
+        aliases = net_inst.aliases
+
+        new_net = Network(net, host, port, security, auth)
+        log(f"New network for {net}: {new_net}")
+        new_net.last = last
+        new_net.relays = relays
+        new_net.aliases = aliases
+        main.network[net] = new_net
+    main.saveConf("network")
+    log("Finished migrating network configuration")
+
+
+class Network:
+    def __init__(self, net, host, port, security, auth):
+        self.net = net
+        self.host = host
+        self.port = port
+        self.security = security
+        self.auth = auth
+        self.chanlimit = None
+
+        self.last = 1
+        self.relays = {}
+        self.aliases = {}
+
+    def add_relay(self, num=None):
+        # Grrrrrrrrrr
+        self.last = int(self.last)
+        if not num:
+            num = self.last
+            self.last += 1
+        elif num == self.last:
+            self.last += 1
+        registered = False
+        if not needToRegister(self.net):
+            registered = True
+            # Don't need to register if it's been disabled in definitions,
+            #  so we'll pretend we already did
+        self.relays[num] = {
+            "enabled": main.config["ConnectOnCreate"],
+            "net": self.net,
+            "id": num,
+            "registered": registered,
+        }
+        password = alias.generate_password()
+        if num not in main.alias.keys():
+            main.alias[num] = alias.generate_alias()
+            main.saveConf("alias")
+        self.aliases[num] = {"password": password}
+        # if main.config["ConnectOnCreate"]: -- Done in provision
+        #    self.start_bot(num)
+        provisionRelay(num, self.net)
+        return num, main.alias[num]["nick"]
+
+    def enable_relay(self, num):
+        """
+        Enable a relay for this network.
+        Send a command to ZNC to connect.
+        """
+        self.relays[num]["enabled"] = True
+        user = main.alias[num]["nick"]
+        commands = {"status": ["Connect"]}
+        name = f"{self.net}{num}"
+        deliverRelayCommands(num, commands, user=user + "/" + self.net)
+        main.saveConf("network")
+        if name not in main.IRCPool.keys():
+            self.start_bot(num)
+
+    def disable_relay(self, num):
+        """
+        Disable a relay for this network.
+        Send a command to ZNC to disconnect.
+        Stop trying to connect to the relay.
+        """
+        self.relays[num]["enabled"] = False
+        user = main.alias[num]["nick"]
+        # relay = main.network[spl[1]].relays[relayNum]
+        commands = {"status": ["Disconnect"]}
+        name = f"{self.net}{num}"
+        deliverRelayCommands(num, commands, user=user + "/" + self.net)
+        main.saveConf("network")
+        if name in main.ReactorPool.keys():
+            if name in main.FactoryPool.keys():
+                main.FactoryPool[name].stopTrying()
+            main.ReactorPool[name].disconnect()
+            if name in main.IRCPool.keys():
+                del main.IRCPool[name]
+            del main.ReactorPool[name]
+            del main.FactoryPool[name]
+
+    def killAliases(self, aliasList):
+        for i in aliasList:
+            name = self.net + str(i)
+            if name in main.ReactorPool.keys():
+                if name in main.FactoryPool.keys():
+                    main.FactoryPool[name].stopTrying()
+                main.ReactorPool[name].disconnect()
+                if name in main.IRCPool.keys():
+                    del main.IRCPool[name]
+                del main.ReactorPool[name]
+                del main.FactoryPool[name]
+
+    def delete_relay(self, id):
+        del self.relays[id]
+        del self.aliases[id]
+        # del main.alias[id] - Aliases are global per num, so don't delete them!
+        self.killAliases([id])
+
+    def seppuku(self):
+        # Removes all bots in preperation for deletion
+        self.killAliases(self.relays.keys())
+        nukeNetwork(self.net)
+
+    def start_bot(self, num):
+        # a single name is given to relays in the backend
+        # e.g. freenode1 for the first relay on freenode network
+        keyFN = main.certPath + main.config["Key"]
+        certFN = main.certPath + main.config["Certificate"]
+        contextFactory = DefaultOpenSSLContextFactory(
+            keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
+        )
+        bot = IRCBotFactory(self.net, num)
+        # host, port = self.relays[num]["host"], self.relays[num]["port"]
+        host, port = getRelay(num)
+        rct = reactor.connectSSL(host, port, bot, contextFactory)
+        name = self.net + str(num)
+        main.ReactorPool[name] = rct
+        main.FactoryPool[name] = bot
+
+        log("Started bot on relay %s on %s" % (num, self.host))
+
+    def start_bots(self):
+        for num in self.relays.keys():
+            if self.relays[num]["enabled"]:
+                self.start_bot(num)

legacy/modules/provision.py

@@ -0,0 +1,106 @@
+import main
+import modules.regproc
+from twisted.internet import reactor
+from utils.deliver_relay_commands import deliverRelayCommands
+from utils.logging.log import warn
+
+
+def provisionUserNetworkData(
+    num, nick, altnick, ident, realname, network, host, port, security
+):
+    commands = {}
+    stage2commands = {}
+    stage2commands["status"] = []
+    commands["controlpanel"] = []
+    user = nick.lower()
+    commands["controlpanel"].append(
+        "AddUser %s %s" % (user, main.config["Relay"]["Password"])
+    )
+    commands["controlpanel"].append("AddNetwork %s %s" % (user, network))
+    commands["controlpanel"].append("Set Nick %s %s" % (user, nick))
+    commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick))
+    commands["controlpanel"].append("Set Ident %s %s" % (user, ident))
+    commands["controlpanel"].append("Set RealName %s %s" % (user, realname))
+    if security == "ssl":
+        commands["controlpanel"].append(
+            "SetNetwork TrustAllCerts %s %s true" % (user, network)
+        )  # Don't judge me
+        commands["controlpanel"].append(
+            "AddServer %s %s %s +%s" % (user, network, host, port)
+        )
+    elif security == "plain":
+        commands["controlpanel"].append(
+            "AddServer %s %s %s %s" % (user, network, host, port)
+        )
+    if not main.config["ConnectOnCreate"]:
+        stage2commands["status"].append("Disconnect")
+    if main.config["Toggles"]["CycleChans"]:
+        stage2commands["status"].append("LoadMod disconkick")
+    stage2commands["status"].append("LoadMod chansaver")
+    inst = modules.regproc.selectInst(network)
+    if "setmode" in inst.keys():
+        stage2commands["status"].append("LoadMod perform")
+        # stage2commands["perform"].append("add mode %nick% +"+inst["setmode"])
+    deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]])
+
+
+def provisionAuthenticationData(num, nick, network, auth, password):
+    commands = {}
+    commands["status"] = []
+    user = nick.lower()
+    if auth == "sasl":
+        commands["sasl"] = []
+        commands["status"].append("UnloadMod nickserv")
+        commands["status"].append("LoadMod sasl")
+        commands["sasl"].append("Mechanism plain")
+        commands["sasl"].append("Set %s %s" % (nick, password))
+    elif auth == "ns":
+        commands["nickserv"] = []
+        commands["status"].append("UnloadMod sasl")
+        commands["status"].append("LoadMod nickserv")
+        commands["nickserv"].append("Set %s" % password)
+    inst = modules.regproc.selectInst(network)
+    if "setmode" in inst.keys():
+        # perform is loaded above
+        # commands["status"].append("LoadMod perform")
+        commands["perform"] = ["add mode %nick% +" + inst["setmode"]]
+    deliverRelayCommands(num, commands, user=user + "/" + network)
+
+
+def provisionRelay(num, network):  # provision user and network data
+    aliasObj = main.alias[num]
+    # alias = aliasObj["nick"]
+    nick = aliasObj["nick"]
+    altnick = aliasObj["altnick"]
+    ident = aliasObj["ident"]
+    realname = aliasObj["realname"]
+    provisionUserNetworkData(
+        num,
+        nick,
+        altnick,
+        ident,
+        realname,
+        network,
+        main.network[network].host,
+        main.network[network].port,
+        main.network[network].security,
+    )
+    if main.config["ConnectOnCreate"]:
+        reactor.callLater(10, main.network[network].start_bot, num)
+
+
+def provisionMultipleRelays(net, relaysNeeded):
+    if not relaysNeeded:
+        return []
+    if not main.config["ChanKeep"]["Provision"]:
+        warn(
+            f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled"
+        )
+        return []
+    numsProvisioned = []
+    for i in range(relaysNeeded):
+        num, alias = main.network[net].add_relay()
+        numsProvisioned.append(num)
+        provisionRelay(num, net)
+    main.saveConf("network")
+    return numsProvisioned