Begin implementing RTS

This reverts commit 64e3e1160aa76d191740342ab3edc68807f890fb.

.gitignore

@@ -1,13 +1,162 @@
-*.pyc
+# ---> Python
-*.pem
+# Byte-compiled / optimized / DLL files
-*.swp
 __pycache__/
-conf/config.json
+*.py[cod]
-conf/wholist.json
+*$py.class
-conf/counters.json
+
-conf/monitor.json
+# C extensions
-conf/tokens.json
+*.so
-conf/network.dat
+
-conf/alias.json
+# Distribution / packaging
-conf/dist.sh
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+# lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
 env/
+venv/
+env-glibc/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+.bash_history
+.vscode/
+docker/data
+*.pem
+legacy/conf/live/
+legacy/conf/cert/
+stack.env

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+    - id: black
+      exclude: ^core/migrations
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        args: [--max-line-length=88]
+        exclude: ^core/migrations
+  - repo: https://github.com/sirwart/ripsecrets.git
+    rev: v0.1.5
+    hooks:
+      - id: ripsecrets

Dockerfile

@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1
+FROM python:3.10
+
+RUN useradd -d /code xf
+RUN mkdir /code
+RUN chown xf:xf /code
+
+RUN mkdir /venv
+RUN chown xf:xf /venv
+
+USER xf
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY requirements.txt /code/
+COPY docker/discord-patched.tgz /code/
+
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r requirements.txt
+# && python -m spacy download en_core_web_sm
+
+RUN tar xf /code/discord-patched.tgz -C /venv/lib/python3.10/site-packages
+
+CMD . /venv/bin/activate && exec python monolith.py

Makefile

@@ -0,0 +1,20 @@
+run:
+	docker-compose -f docker-compose.prod.yml --env-file=stack.env up -d
+
+build:
+	docker-compose -f docker-compose.prod.yml --env-file=stack.env build
+
+stop:
+	docker-compose -f docker-compose.prod.yml --env-file=stack.env down
+
+log:
+	docker-compose -f docker-compose.prod.yml --env-file=stack.env logs -f --names
+
+run-infra:
+	docker-compose -f docker-compose.infra.yml --env-file=stack.env up -d
+
+stop-infra:
+	docker-compose -f docker-compose.infra.yml --env-file=stack.env down
+
+log-infra:
+	docker-compose -f docker-compose.infra.yml --env-file=stack.env logs -f

commands/list.py

@@ -1,26 +0,0 @@
-import main
-
-class ListCommand:
-    def __init__(self, *args):
-        self.list(*args)
-
-    def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
-        if authed:
-            if length == 2:
-                if not spl[1] in main.network.keys():
-                    failure("No such network: %s" % spl[1])
-                    return
-                if not 1 in main.network[spl[1]].relays.keys():
-                    failure("Network has no first instance")
-                    return
-                if not spl[1]+"1" in main.IRCPool.keys():
-                    failure("No IRC instance: %s - 1" % spl[1])
-                    return
-                main.IRCPool[spl[1]+"1"].list()
-                success("Requested list with first instance of %s" % spl[1])
-                return
-            else:
-                incUsage("list")
-                return
-        else:
-            incUsage(None)

commands/mon.py

@@ -1,280 +0,0 @@
-import main
-import argparse
-import sys
-from io import StringIO
-from yaml import dump
-
-class MonCommand:
-    def __init__(self, *args):
-        self.mon(*args)
-
-    def setup_arguments(self, ArgumentParser):
-        self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.")
-        group1 = self.parser.add_mutually_exclusive_group(required=True)
-        group1.add_argument("-a", "--add", metavar="entry", dest="addEntry",  help="Add an entry")
-        group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry")
-        group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries")
-        group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry")
-
-        group2 = self.parser.add_mutually_exclusive_group()
-        group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing")
-        group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing")
-
-        self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values")
-
-        self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who")
-        self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching")
-        self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching")
-        self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching")
-        self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching")
-        self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who")
-
-        self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)")
-        self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action")
-        self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick")
-        self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode")
-        self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode")
-        self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who")
-        self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who")
-        self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)")
-
-    def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
-        # We need to override the ArgumentParser class because
-            # it's only meant for CLI applications and exits
-            # after running, so we override the exit function
-            # to do nothing. We also need to do it here in order
-            # to catch the error messages and show them to the user.
-            # It sucks. I know.
-        if authed:
-            class ArgumentParser(argparse.ArgumentParser):
-                def exit(self, status=0, message=None):
-                    if message:
-                        failure(message)
-            self.setup_arguments(ArgumentParser)
-            if length == 1:
-                info("Incorrect usage, use mon -h for help")
-                return
-            old_stdout = sys.stdout
-            old_stderr = sys.stderr
-            my_stdout = sys.stdout = StringIO()
-            my_stderr = sys.stderr = StringIO()
-            try:
-                parsed = self.parser.parse_args(spl[1:])
-            except:
-                return
-            sys.stdout = old_stdout
-            sys.stderr = old_stderr
-            stdout = my_stdout.getvalue()
-            stderr = my_stdout.getvalue()
-            if not stdout == "":
-                info(stdout)
-            elif not stderr == "":
-                failure(my_stdout.getvalue())
-                return
-            my_stdout.close()
-            my_stderr.close()
-
-            if parsed.addEntry:
-                if parsed.addEntry in main.monitor.keys():
-                    failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry)
-                    return
-                cast = self.makeCast(parsed, failure, info)
-                if cast == False:
-                    return
-                main.monitor[parsed.addEntry] = cast
-                main.saveConf("monitor")
-                success("Successfully created monitor group %s" % parsed.addEntry)
-                return
-            elif parsed.delEntry:
-                if not parsed.delEntry in main.monitor.keys():
-                    failure("No such monitor group: %s" % parsed.delEntry)
-                    return
-                del main.monitor[parsed.delEntry]
-                main.saveConf("monitor")
-                success("Successfully removed monitor group: %s" % parsed.delEntry)
-                return
-            elif parsed.modEntry:
-                if not parsed.doAppend and not parsed.doRemove:
-                    failure("Specify --append or --remove with --mod")
-                    return
-                if not parsed.modEntry in main.monitor.keys():
-                    failure("No such monitor group: %s" % parsed.modEntry)
-                    return
-                cast = self.makeCast(parsed, failure, info)
-                if cast == False:
-                    return
-                if parsed.doAppend:
-                    merged = self.addCast(main.monitor[parsed.modEntry], cast, info)
-                    main.monitor[parsed.modEntry] = merged
-                elif parsed.doRemove:
-                    merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info)
-                    if merged == {}:
-                        del main.monitor[parsed.modEntry]
-                        info("Group %s deleted due to having no attributes" % parsed.modEntry)
-                        main.saveConf("monitor")
-                        return
-                    main.monitor[parsed.modEntry] = merged
-                main.saveConf("monitor")
-                success("Successfully updated entry %s" % parsed.modEntry)
-                return
-            elif parsed.listEntry:
-                info(dump(main.monitor))
-                return
-
-        else:
-            incUsage(None)
-
-    def dedup(self, data):
-        if not isinstance(data, bool):
-            return list(set(data))
-        return data
-
-    def parseNetworkFormat(self, lst, failure, info):
-        cast = {}
-        if lst == None:
-            return "nil"
-        if len(lst) == 0:
-            failure("List has no entries")
-            return False
-        elif len(lst) > 0:
-            if len(lst[0]) == 0:
-                failure("Nested list has no entries")
-                return False
-        for i in lst:
-            if not i[0] in main.pool.keys():
-                failure("Name does not exist: %s" % i[0])
-                return False
-            if i[0] in main.IRCPool.keys():
-                for x in i[1:]:
-                    if not x in main.IRCPool[i[0]].channels:
-                        info("%s: Bot not on channel: %s" % (i[0], x))
-            if len(i) == 1:
-                cast[i[0]] = True
-            else:
-                if i[0] in cast.keys():
-                    if not cast[i[0]] == True:
-                        for x in self.dedup(i[1:]):
-                            cast[i[0]].append(x)
-                    else:
-                        cast[i[0]] = self.dedup(i[1:])
-                else:
-                    cast[i[0]] = self.dedup(i[1:])
-        for i in cast.keys():
-            deduped = self.dedup(cast[i])
-            cast[i] = deduped
-        return cast
-
-    # Create or modify a monitor group magically
-    def makeCast(self, obj, failure, info):
-        validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"]
-        cast = {}
-        if not obj.specType == None:
-            types = self.dedup(obj.specType)
-            for i in types:
-                if not i in validTypes:
-                    failure("Invalid type: %s" % i)
-                    info("Available types: %s" % ", ".join(validTypes))
-                    return False
-            cast["type"] = types
-        if not obj.source == None:
-            sourceParse = self.parseNetworkFormat(obj.source, failure, info)
-            if not sourceParse:
-                return False
-            if not sourceParse == "nil":
-                cast["sources"] = sourceParse
-
-        if not obj.send == None:
-            sendParse = self.parseNetworkFormat(obj.send, failure, info)
-            if not sendParse:
-                return False
-            if not sendParse == "nil":
-                cast["send"] = sendParse
-
-        if not obj.message == None:
-            cast["message"] = []
-            for i in obj.message:
-                cast["message"].append(" ".join(i))
-
-        if not obj.user == None:
-            cast["user"] = obj.user
-        if not obj.modes == None:
-            cast["modes"] = obj.modes
-        if not obj.modeargs == None:
-            cast["modeargs"] = obj.modeargs
-        if not obj.server == None:
-            cast["server"] = obj.server
-        if not obj.status == None:
-            cast["status"] = obj.status
-        if not obj.free == None:
-            cast["free"] = obj.free
-        if not obj.nick == None:
-            cast["nick"] = obj.nick
-        if not obj.ident == None:
-            cast["ident"] = obj.ident
-        if not obj.host == None:
-            cast["host"] = obj.host
-        if not obj.real == None:
-            cast["real"] = []
-            for i in obj.real:
-                cast["real"].append(" ".join(i))
-        if not obj.inside == None:
-            if obj.inside.lower() in ["yes", "true", "on"]:
-                cast["inside"] = True
-            elif obj.inside.lower() in ["no", "false", "off"]:
-                cast["inside"] = False
-            else:
-                failure("inside: Unknown operand: %s" % obj.inside)
-                return
-        return cast
-
-    def subtractCast(self, source, patch, info):
-        for i in patch.keys():
-            if i in source.keys():
-                if isinstance(source[i], dict):
-                    result = self.subtractCast(source[i], patch[i], info)
-                    if result == {}:
-                        info("Removing upper element: %s" % i)
-                        del source[i]
-                        continue
-                    source[i] = result
-                    continue
-                if isinstance(patch[i], bool):
-                    del source[i]
-                    info("Removing entire element: %s" % i)
-                    continue
-                for x in patch[i]:
-                    if isinstance(source[i], bool):
-                        info("Attempt to remove %s from network-wide definition" % x)
-                        continue
-                    if x in source[i]:
-                        source[i].remove(x)
-                        if source[i] == []:
-                            del source[i]
-                    else:
-                        info("Element %s not in source %s" % (x, i))
-            else:
-                info("Non-matched key: %s" % i)
-        return source
-
-    def addCast(self, source, patch, info):
-        for i in patch.keys():
-            if i in source.keys():
-                if isinstance(source[i], dict):
-                    result = self.addCast(source[i], patch[i], info)
-                    source[i] = result
-                    continue
-                if isinstance(patch[i], bool):
-                    source[i] = patch[i]
-                    info("Overriding local element %s with network-wide definition" % i)
-                    continue
-                for x in patch[i]:
-                    if isinstance(source[i], bool):
-                        source[i] = []
-                        info("Overriding element %s, previously set network-wide" % i)
-                    if x in source[i]:
-                        info("Element %s already in source %s" % (x, i))
-                    else:
-                        source[i].append(x)
-            else:
-                source[i] = patch[i]
-        return source

conf/example/alias.json

@@ -1 +0,0 @@
-{}

conf/example/counters.json

@@ -1 +0,0 @@
-{}

conf/example/masterbuf.json

@@ -1 +0,0 @@
-[]

conf/example/monitor.json

@@ -1 +0,0 @@
-{}

conf/example/tokens.json

@@ -1 +0,0 @@
-{}

core/bot.py

@@ -1,706 +0,0 @@
-from twisted.internet.protocol import ReconnectingClientFactory
-from twisted.words.protocols.irc import IRCClient
-from twisted.internet.defer import Deferred
-from twisted.internet.task import LoopingCall
-from twisted.internet import reactor, task
-from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage
-
-import sys
-from string import digits
-from random import randint
-from copy import deepcopy
-
-from modules import userinfo
-from modules import counters
-from modules import monitor
-from modules import chankeep
-
-from core.relay import sendRelayNotification
-from utils.dedup import dedup
-from utils.get import getRelay
-
-import main
-from utils.logging.log import *
-from utils.logging.debug import *
-from utils.logging.send import *
-from utils.parsing import parsen
-
-from twisted.internet.ssl import DefaultOpenSSLContextFactory
-
-def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
-    keyFN = main.certPath+main.config["Key"]
-    certFN = main.certPath+main.config["Certificate"]
-    contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"),
-            certFN.encode("utf-8", "replace"))
-    bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
-    host, port = getRelay(num)
-    rct = reactor.connectSSL(host,
-            port,
-            bot, contextFactory)
-
-def parsemsg(s):
-    """
-    Breaks a message from an IRC server into its prefix, command, and
-    arguments.
-    @param s: The message to break.
-    @type s: L{bytes}
-    @return: A tuple of (prefix, command, args).
-    @rtype: L{tuple}
-    """
-    prefix = ''
-    trailing = []
-    if not s:
-        raise IRCBadMessage("Empty line.")
-    if s[0:1] == ':':
-        prefix, s = s[1:].split(' ', 1)
-    if s.find(' :') != -1:
-        s, trailing = s.split(' :', 1)
-        args = s.split(' ')
-        args.append(trailing)
-    else:
-        args = s.split(' ')
-    command = args.pop(0)
-    return prefix, command, args
-
-class IRCRelay(IRCClient):
-    def __init__(self, num, relayCommands, user, stage2):
-        self.isconnected = False
-        self.buffer = ""
-        if user == None:
-            self.user = main.config["Relay"]["User"]
-        else:
-            self.user = user
-        password = main.config["Relay"]["Password"]
-        self.nickname = "relay"
-        self.realname = "relay"
-        self.username = self.user
-        self.password = self.user+":"+password
-
-        self.relayCommands = relayCommands
-        self.num = num
-        self.stage2 = stage2
-        self.loop = None
-
-    def privmsg(self, user, channel, msg):
-        nick, ident, host = parsen(user)
-        for i in main.ZNCErrors:
-            if i in msg:
-                error("ZNC issue:", msg)
-        if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
-            nick = nick[1:]
-            if nick in self.relayCommands.keys():
-                sendAll("[%s] %s -> %s" % (self.num, nick, msg))
-
-    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
-        log("%s: relay password mismatch" % self.num)
-        sendAll("%s: relay password mismatch" % self.num)
-
-    def sendStage2(self):
-        if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []]
-            if not len(self.stage2) == 0:
-                user = self.stage2[0].pop(0)
-                commands = self.stage2[0].pop(0)
-                del self.stage2[0]
-                deliverRelayCommands(self.num, commands, user, self.stage2)
-
-    def signedOn(self):
-        if not self.isconnected:
-            self.isconnected = True
-            log("signed on as a relay: %s" % self.num)
-            #sendRelayNotification("Relay", {"type": "conn", "status": "connected"})  nobody actually cares
-            sleeptime = 0
-            increment = 0.8
-            for i in self.relayCommands.keys():
-                for x in self.relayCommands[i]:
-                    reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x)
-                sleeptime += increment
-                increment += 0.8
-            reactor.callLater(sleeptime, self.sendStage2)
-            reactor.callLater(sleeptime+5, self.transport.loseConnection)
-            return
-
-class IRCBot(IRCClient):
-    def __init__(self, net, num):
-        self.isconnected = False
-        self.channels = []
-        self.net = net
-        self.num = num
-        self.buffer = ""
-        self.name = net + str(num)
-        alias = main.alias[num]
-        relay = main.network[self.net].relays[num]
-        self.nickname = alias["nick"]
-        self.realname = alias["realname"]
-        self.username = alias["nick"]+"/"+relay["net"]
-        self.password = main.config["Relay"]["Password"]
-        self.userinfo = None      #
-        self.fingerReply = None   #
-        self.versionName = None   # don't tell anyone information about us
-        self.versionNum = None    #
-        self.versionEnv = None    #
-        self.sourceURL = None     #
-
-        self._getWho = {} # LoopingCall objects -- needed to be able to stop them
-
-        self._tempWho = {} # temporary storage for gathering WHO info
-        self._tempNames = {} # temporary storage for gathering NAMES info
-        self._tempList = ([], []) # temporary storage for gathering LIST info
-        self.listOngoing = False   # we are currently receiving a LIST
-        self.listRetried = False   # we asked and got nothing so asked again
-        self.listAttempted = False # we asked for a list
-        self.listSimple = False    # after asking again we got the list, so use the simple
-                                   #  syntax from now on
-        self.wantList = False      # we want to send a LIST, but not all relays are active yet
-        self.chanlimit = 0
-        self.servername = None
-
-    def lineReceived(self, line):
-        if bytes != str and isinstance(line, bytes):
-            # decode bytes from transport to unicode
-            line = line.decode("utf-8", "replace")
-
-        line = lowDequote(line)
-        try:
-            prefix, command, params = parsemsg(line)
-            if command in numeric_to_symbolic:
-                command = numeric_to_symbolic[command]
-            self.handleCommand(command, prefix, params)
-        except IRCBadMessage:
-            self.badMessage(line, *sys.exc_info())
-
-    def joinChannels(self, channels):
-        sleeptime = 0.0
-        increment = 0.8
-        for i in channels:
-            if not i in self.channels:
-                debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds")
-                reactor.callLater(sleeptime, self.join, i)
-                sleeptime += increment
-                if sleeptime == 10:
-                    sleeptime = 0.0
-                    increment = 0.7
-                increment += 0.1
-            else:
-                error("%s - %i - Cannot join channel we are already on - %s" % (self.net, self.num, i))
-
-    def checkChannels(self):
-        if self.net in main.TempChan.keys():
-            if self.num in main.TempChan[self.net].keys():
-                self.joinChannels(main.TempChan[self.net][self.num])
-                del main.TempChan[self.net][self.num]
-                if not main.TempChan[self.net]:
-                    del main.TempChan[self.net]
-
-    def event(self, **cast):
-        for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over
-            if cast[i] == "":       #  a dictionary that changes length with each iteration
-                del cast[i]
-        if "muser" in cast.keys():
-            if cast["muser"] == self.servername:
-                return
-        if "channel" in cast.keys():
-            if cast["channel"] == "*":
-                return
-        if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
-            cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
-        if set(["nick", "ident", "host", "msg"]).issubset(set(cast)):
-            if "msg" in cast.keys():
-                if cast["ident"] == "znc" and cast["host"] == "znc.in":
-                    cast["type"] = "znc"
-                    cast["num"] = self.num
-                    del cast["nick"]
-                    del cast["ident"]
-                    del cast["host"]
-                    del cast["channel"]
-                    if "Disconnected from IRC" in cast["msg"]:
-                        log("ZNC disconnected on %s - %i" % (self.net, self.num))
-                        self.isconnected = False
-                    if "Connected!" in cast["msg"]:
-                        log("ZNC connected on %s - %i" % (self.net, self.num))
-                        self.isconnected = True
-        if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
-            if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here
-                if cast["channel"].lower() == self.nickname.lower():    #  as they are channel == nickname
-                    #castDup = deepcopy(cast)                          #  however modes are not queries!
-                    cast["mtype"] = cast["type"]
-                    cast["type"] = "query"
-                    cast["num"] = self.num
-                    #self.event(**castDup)
-                    # Don't call self.event for this one because queries are not events on a
-                    #  channel, but we still want to see them
-            if "user" in cast.keys():
-                if cast["user"].lower() == self.nickname.lower():
-                    castDup = deepcopy(cast)
-                    castDup["mtype"] = cast["type"]
-                    castDup["type"] = "self"
-                    cast["num"] = self.num
-                    self.event(**castDup)
-            if "nick" in cast.keys():
-                if cast["nick"].lower() == self.nickname.lower():
-                    castDup = deepcopy(cast)
-                    castDup["mtype"] = cast["type"]
-                    castDup["type"] = "self"
-                    cast["num"] = self.num
-                    if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly
-                        self.event(**castDup)                               #  don't tell anyone else
-            if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
-                if not cast["msg"] == None:
-                    if self.nickname.lower() in cast["msg"].lower():
-                        castDup = deepcopy(cast)
-                        castDup["mtype"] = cast["type"]
-                        castDup["type"] = "highlight"
-                        cast["num"] = self.num
-                        self.event(**castDup)
-
-        if not "net" in cast.keys():
-            cast["net"] = self.net
-        counters.event(self.net, cast["type"])
-        monitor.event(self.net, cast)
-
-    def privmsg(self, user, channel, msg):
-        self.event(type="msg", muser=user, channel=channel, msg=msg)
-
-    def noticed(self, user, channel, msg):
-        self.event(type="notice", muser=user, channel=channel, msg=msg)
-
-    def action(self, user, channel, msg):
-        self.event(type="action", muser=user, channel=channel, msg=msg)
-
-    def get(self, var):
-        try:
-            result = getattr(self, var)
-        except AttributeError:
-            result = None
-        return result
-
-    def setNick(self, nickname):
-        self._attemptedNick = nickname
-        self.sendLine("NICK %s" % nickname)
-        self.nickname = nickname
-
-    def alterCollidedNick(self, nickname):
-        newnick = nickname + "_"
-        return newnick
-
-    def nickChanged(self, olduser, newnick):
-        self.nickname = newnick
-        self.event(type="self", mtype="nick", muser=olduser, user=newnick)
-
-    def irc_ERR_NICKNAMEINUSE(self, prefix, params):
-        self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
-        self.setNick(self._attemptedNick)
-
-    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
-        log("%s - %i: password mismatch" % (self.net, self.num))
-        sendAll("%s - %i: password mismatch" % (self.net, self.num))
-
-    def _who(self, channel):
-        d = Deferred()
-        if channel not in self._tempWho:
-            self._tempWho[channel] = ([], [])
-        self._tempWho[channel][0].append(d)
-        self.sendLine("WHO %s" % channel)
-        return d
-
-    def who(self, channel):
-        self._who(channel).addCallback(self.got_who)
-
-    def irc_RPL_WHOREPLY(self, prefix, params):
-        channel = params[1]
-        ident = params[2]
-        host = params[3]
-        server = params[4]
-        nick = params[5]
-        status = params[6]
-        realname = params[7]
-        if channel not in self._tempWho:
-            return
-        n = self._tempWho[channel][1]
-        n.append([nick, nick, host, server, status, realname])
-        self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status)
-
-    def irc_RPL_ENDOFWHO(self, prefix, params):
-        channel = params[1]
-        if channel not in self._tempWho:
-            return
-        callbacks, info = self._tempWho[channel]
-        for cb in callbacks:
-            cb.callback((channel, info))
-        del self._tempWho[channel]
-
-    def got_who(self, whoinfo):
-        userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
-
-    def sanit(self, data):
-        if len(data) >= 1:
-            if data[0] in ["!", "~", "&", "@", "%", "+"]:
-                data = data[1:]
-                return data
-            return data
-        else:
-            return False
-
-    def names(self, channel):
-        d = Deferred()
-        if channel not in self._tempNames:
-            self._tempNames[channel] = ([], [])
-        self._tempNames[channel][0].append(d)
-        self.sendLine("NAMES %s" % channel)
-        return d
-
-    def irc_RPL_NAMREPLY(self, prefix, params):
-        channel = params[2]
-        nicklist = params[3].split(' ')
-        if channel not in self._tempNames:
-            return
-        n = self._tempNames[channel][1]
-        n.append(nicklist)
-
-    def irc_RPL_ENDOFNAMES(self, prefix, params):
-        channel = params[1]
-        if channel not in self._tempNames:
-            return
-        callbacks, namelist = self._tempNames[channel]
-        for cb in callbacks:
-            cb.callback((channel, namelist))
-        del self._tempNames[channel]
-
-    def got_names(self, nicklist):
-        newNicklist = []
-        for i in nicklist[1]:
-            for x in i:
-                f = self.sanit(x) # need to store this as well, or potentially just do not remove it...
-                if f:
-                    newNicklist.append(f)
-        userinfo.initialNames(self.net, nicklist[0], newNicklist)
-
-    def myInfo(self, servername, version, umodes, cmodes):
-        self.servername = servername
-
-    def _list(self, noargs):
-        d = Deferred()
-        self._tempList = ([], [])
-        self._tempList[0].append(d)
-        if self.listSimple:
-            self.sendLine("LIST")
-            return d            # return early if we know what to do
-
-        if noargs:
-            self.sendLine("LIST")
-        else:
-            self.sendLine("LIST >0")
-        return d
-
-    def list(self, noargs=False, nocheck=False):
-        if self.listAttempted:
-            debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num))
-            return
-        else:
-            self.listAttempted = True
-        if self.listOngoing:
-            debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num))
-            return
-        else:
-            if nocheck:
-                allRelays = True # override the system - if this is
-            else:                #  specified, we already did this
-                allRelays = chankeep.allRelaysActive(self.net)
-            if not allRelays:
-                self.wantList = True
-                debug("Not all relays were active for LIST request")
-                return
-            self._list(noargs).addCallback(self.got_list)
-
-    def irc_RPL_LISTSTART(self, prefix, params):
-        self.listAttempted = False
-        self.listOngoing = True
-        self.wantList = False
-
-    def irc_RPL_LIST(self, prefix, params):
-        channel = params[1]
-        users = params[2]
-        topic = params[3]
-        self._tempList[1].append([channel, users, topic])
-
-    def irc_RPL_LISTEND(self, prefix, params):
-        if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there
-            debug("We didn't ask for this LIST, discarding")
-            self._tempList[0].clear()
-            self._tempList[1].clear()
-            return
-        callbacks, info = self._tempList
-        self.listOngoing = False
-        for cb in callbacks:
-            cb.callback((info))
-        noResults = False
-        if len(self._tempList[1]) == 0:
-            noResults = True
-        self._tempList[0].clear()
-        self._tempList[1].clear()
-        if noResults:
-            if self.listRetried:
-                warn("LIST still empty after retry: %s - %i" % (net, num))
-                self.listRetried = False
-                return
-            else:
-                self.list(True)
-                self.listRetried = True
-        else:
-            if self.listRetried:
-                self.listRetried = False
-                debug("List received after retry - defaulting to simple list syntax")
-                self.listSimple = True
-
-    def got_list(self, listinfo):
-        if len(listinfo) == 0: # probably ngircd not supporting LIST >0
-            return
-        chankeep.initialList(self.net, self.num, listinfo, self.chanlimit)
-
-    def isupport(self, options):
-        interested = ("CHANLIMIT", "MAXCHANNELS")
-        if not any((x for x in options if any(y in x for y in interested))):
-            return          # check if any of interested is in any of options, some networks
-        chanlimit = None    #  call isupport() more than once, so discard swiftly anything
-        if not self.isconnected: #  we don't care about
-            log("endpoint connected: %s - %i" % (self.net, self.num))
-            self.isconnected = True
-        for i in options:
-            if i.startswith("CHANLIMIT"):
-                if ":" in i:
-                    split = i.split(":")
-                    if len(split) >= 2:
-                        chanlimit = split[1]
-                        break
-            elif i.startswith("MAXCHANNELS"):
-                if "=" in i:
-                    split = i.split("=")
-                    if len(split) == 2:
-                        chanlimit = split[1]
-                        break
-        print("chanlimit", chanlimit)
-        try:
-            self.chanlimit = int(chanlimit)
-        except TypeError:
-            warn("Invalid chanlimit: %s" % i)
-        if self.chanlimit == 0:
-            self.chanlimit = 200 # don't take the piss if it's unlimited
-        allRelays = chankeep.allRelaysActive(self.net)
-        print(self.net, self.num, allRelays)
-        if allRelays:
-            for i in main.network.keys():
-                for x in main.network[i].relays.keys():
-                    name = i+str(x)
-                    if main.IRCPool[name].wantList == True:
-                        main.IRCPool[name].list(nocheck=True)
-                        debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
-        if self.num == 1: # Only one instance should do a list
-            if self.chanlimit:
-                if allRelays:
-                    self.list()
-                else:
-                    self.wantList = True
-            else:
-                debug("Aborting LIST due to bad chanlimit")
-        self.checkChannels()
-
-    #twisted sucks so i have to do this to actually get the user info
-    def irc_JOIN(self, prefix, params):
-        """
-        Called when a user joins a channel.
-        """
-        nick = prefix.split('!')[0]
-        channel = params[-1]
-        if nick == self.nickname:
-            self.joined(channel)
-        else:
-            self.userJoined(prefix, channel)
-
-    def irc_PART(self, prefix, params):
-        """
-        Called when a user leaves a channel.
-        """
-        nick = prefix.split('!')[0]
-        channel = params[0]
-        if len(params) >= 2:
-            message = params[1]
-        else:
-            message = None
-        if nick == self.nickname:
-            self.left(prefix, channel, message)
-        else:
-            self.userLeft(prefix, channel, message)
-
-    def irc_QUIT(self, prefix, params):
-        """
-        Called when a user has quit.
-        """
-        nick = prefix.split('!')[0]
-        #if nick == self.nickname:
-        #self.botQuit(prefix, params[0])
-        #else:
-        self.userQuit(prefix, params[0])
-
-    def irc_NICK(self, prefix, params):
-        """
-        Called when a user changes their nickname.
-        """
-        nick = prefix.split('!', 1)[0]
-        if nick == self.nickname:
-            self.nickChanged(prefix, params[0])
-        else:
-            self.userRenamed(prefix, params[0])
-
-    def irc_KICK(self, prefix, params):
-        """
-        Called when a user is kicked from a channel.
-        """
-        #kicker = prefix.split('!')[0]
-        channel = params[0]
-        kicked = params[1]
-        message = params[-1]
-        # Checks on whether it was us that was kicked are done in userKicked
-        self.userKicked(kicked, channel, prefix, message)
-
-    def irc_TOPIC(self, prefix, params):
-        """
-        Someone in the channel set the topic.
-        """
-        #user = prefix.split('!')[0]
-        channel = params[0]
-        newtopic = params[1]
-        self.topicUpdated(prefix, channel, newtopic)
-    #END hacks
-
-    def signedOn(self):
-        log("signed on: %s - %i" % (self.net, self.num))
-        #self.event(type="conn", status="connected")
-        sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"})
-
-    def joined(self, channel):
-        if not channel in self.channels:
-            self.channels.append(channel)
-        self.names(channel).addCallback(self.got_names)
-        if main.config["Toggles"]["Who"]:
-            #self.who(channel).addCallback(self.got_who)
-            #self.who(channel)
-            lc = LoopingCall(self.who, channel)
-            self._getWho[channel] = lc
-            intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
-            minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
-            interval = randint(minint, minint+intrange)
-            lc.start(interval)
-
-    def botLeft(self, channel):
-        if channel in self.channels:
-            self.channels.remove(channel)
-        if channel in self._getWho.keys():
-            lc = self._getWho[channel]
-            lc.stop()
-            del self._getWho[channel]
-        userinfo.delChannels(self.net, [channel])                    # < we do not need to deduplicate this
-        #log("Can no longer cover %s, removing records" % channel)#  as it will only be matched once --
-                                                                  #  other bots have different nicknames so
-    def left(self, user, channel, message):                       #  even if they saw it, they wouldn't react
-        self.event(type="part", muser=user, channel=channel, message=message)
-        self.botLeft(channel)
-
-    def userJoined(self, user, channel):
-        self.event(type="join", muser=user, channel=channel)
-
-    def userLeft(self, user, channel, message):
-        self.event(type="part", muser=user, channel=channel, message=message)
-
-    def userQuit(self, user, quitMessage):
-        self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage})
-
-    def userKicked(self, kickee, channel, kicker, message):
-        if kickee.lower() == self.nickname.lower():
-            self.botLeft(channel)
-        self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee)
-
-    def chanlessEvent(self, cast):
-        cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
-        if dedup(self.name, cast): # Needs to be kept self.name until the dedup
-                                   #  function is converted to the new net, num
-                                   #  format
-            return # stop right there sir!
-        chans = userinfo.getChanList(self.net, cast["nick"])
-        if chans == None:
-            error("No channels returned for chanless event: %s" % cast)
-        #    self.event(**cast) -- no, should NEVER happen
-            return
-        # getChansSingle returns all channels of the user, we only want to use
-        #  ones we have common with them
-        realChans = set(chans).intersection(set(self.channels))
-        for i in realChans:
-            cast["channel"] = i
-            self.event(**cast)
-
-    def userRenamed(self, oldname, newname):
-        self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
-
-    def topicUpdated(self, user, channel, newTopic):
-        self.event(type="topic", muser=user, channel=channel, message= newTopic)
-
-    def modeChanged(self, user, channel, toset, modes, args):
-        argList = list(args)
-        modeList = [i for i in modes]
-        for a, m in zip(argList, modeList):
-            self.event(type="mode", muser=user, channel=channel, modes=m, status=toset, modeargs=a)
-
-class IRCBotFactory(ReconnectingClientFactory):
-    def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
-        if net == None:
-            self.num = num
-            self.net = None
-            self.name = "relay - %i" % num
-            self.relay = True
-        else:
-            self.name = net+str(num)
-            self.num = num
-            self.net = net
-            self.relay = False
-        self.client = None
-        self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
-        self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
-        self.factor = main.config["Tweaks"]["Delays"]["Factor"]
-        self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
-
-        self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
-
-    def buildProtocol(self, addr):
-        if self.relay == False:
-            entry = IRCBot(self.net, self.num)
-            main.IRCPool[self.name] = entry
-        else:
-            entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
-
-        self.client = entry
-        return entry
-
-    def clientConnectionLost(self, connector, reason):
-        if not self.relay:
-            userinfo.delChannels(self.net, self.client.channels)
-        if not self.client == None:
-            self.client.isconnected = False
-            self.client.channels = []
-        error = reason.getErrorMessage()
-        log("%s - %i: connection lost: %s" % (self.net, self.num, error))
-        if not self.relay:
-            sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
-            sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error})
-            self.retry(connector)
-        #ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
-
-    def clientConnectionFailed(self, connector, reason):
-        if not self.client == None:
-            self.client.isconnected = False
-            self.client.channels = []
-        error = reason.getErrorMessage()
-        log("%s - %i: connection failed: %s" % (self.net, self.num, error))
-        if not self.relay:
-            sendAll("%s -%s: connection failed: %s" % (self.net, self.num, error))
-            sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error})
-            self.retry(connector)
-        #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
-

core/parser.py

@@ -1,29 +0,0 @@
-import main
-from utils.logging.log import *
-from utils.logging.send import *
-
-def parseCommand(addr, authed, data):
-    #call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
-    spl = data.split()
-    if addr in main.connections.keys():
-        obj = main.connections[addr]
-    else:
-        warn("Got connection object with no instance in the address pool")
-        return
-
-    success = lambda data: sendSuccess(addr, data)
-    failure = lambda data: sendFailure(addr, data)
-    info = lambda data: sendInfo(addr, data)
-
-    incUsage = lambda mode: incorrectUsage(addr, mode)
-    length = len(spl)
-    if len(spl) > 0:
-        cmd = spl[0]
-    else:
-        failure("No text was sent")
-        return
-    if spl[0] in main.CommandMap.keys():
-        main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length)
-        return
-    incUsage(None)
-    return

db.py

@@ -0,0 +1,241 @@
+import asyncio
+from math import ceil
+from os import getenv
+from time import sleep
+
+import aiomysql
+import aioredis
+import manticoresearch
+import msgpack
+import orjson
+from manticoresearch.rest import ApiException
+from numpy import array_split
+from redis import StrictRedis
+
+import util
+from schemas import mc_s
+
+mysql_pool = None
+
+
+configuration = manticoresearch.Configuration(host="http://127.0.0.1:9308")
+api_client = manticoresearch.ApiClient(configuration)
+api_instance = manticoresearch.IndexApi(api_client)
+
+log = util.get_logger("db")
+
+# Redis (legacy)
+# r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
+r = StrictRedis(
+    host="127.0.0.1",  # Replace with your Redis server's IP address
+    port=1289,  # Replace with your Redis server's port
+    db=0,  # Database number
+)
+# AIORedis
+# ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
+ar = aioredis.from_url("redis://127.0.0.1:1289", db=0)
+
+# /var/run/neptune-redis.sock
+# db = 10
+pr = aioredis.from_url("unix://var/run/neptune-redis.sock", db=10)
+# fr = aioredis.from_url("unix://var/run/fisk-redis.sock", db=10)
+fr = aioredis.from_url("unix://var/run/redis.sock", db=10)
+# pr = aioredis.from_url("redis://redis_neptune:6379", db=10, password=getenv("REDIS_PASSWORD"))
+
+KEYNAME = "queue"
+MESSAGE_KEY = "messages"
+
+OHLC_MESSAGE_KEY = "ohlc"
+
+
+TYPES_MAIN = [
+    "msg",
+    "notice",
+    "action",
+    "part",
+    "join",
+    "kick",
+    "quit",
+    "nick",
+    "mode",
+    "topic",
+    "update",
+]
+TYPES_META = ["who"]
+TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
+
+
+async def init_mysql_pool():
+    """
+    Initialize the MySQL connection pool.
+    """
+    global mysql_pool
+    mysql_pool = await aiomysql.create_pool(
+        host="127.0.0.1", port=9306, db="Manticore", minsize=1, maxsize=10
+    )
+
+
+async def rts_store_message(index, data):
+    """
+    Store a RTS message into MySQL using an existing connection pool.
+    Prioritizes instant PubSub delivery, with minimal data storage overhead.
+    :param index: str
+    :param data: dict
+    """
+    # Publish to Redis PubSub
+    packed_index = msgpack.packb({"index": index, "data": data}, use_bin_type=True)
+
+    try:
+        await fr.publish(OHLC_MESSAGE_KEY, packed_index)
+    except aioredis.exceptions.ConnectionError as e:
+        raise e
+        await asyncio.sleep(0.1)
+
+    # Insert data into MySQL
+    try:
+        async with mysql_pool.acquire() as conn:
+            async with conn.cursor() as cur:
+                # Insert data into the table
+                query = f"""
+                    INSERT INTO {index} (s, o, c, h, l, v, a, i, t, t2, ts)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                """
+                # Bind the values directly
+                await cur.execute(
+                    query,
+                    (
+                        data["s"],  # symbol
+                        data["o"],  # open
+                        data["c"],  # close
+                        data["h"],  # high
+                        data["l"],  # low
+                        data["v"],  # volume_base
+                        data["a"],  # volume_quote
+                        data["i"],  # interval
+                        data["t"],  # start_time
+                        data["t2"],  # end_time
+                        data["ts"],  # event_time
+                    ),
+                )
+                await conn.commit()
+                log.debug(f"Stored data for {data['s']} in MySQL.")
+    except aiomysql.Error as e:
+        log.error(f"MySQL error: {e}")
+
+
+async def store_batch(data):
+    """
+    Store a message into Manticore
+    :param data: list
+    """
+    if not data:
+        return
+    # 10000: maximum inserts we can submit to
+    # Manticore as of Sept 2022
+    split_posts = array_split(data, ceil(len(data) / 10000))
+    for messages in split_posts:
+        total = []
+        indexmap = {}
+        for msg in messages:
+            if msg["type"] in TYPES_MAIN:
+                index = "main"
+                schema = mc_s.schema_main
+            elif msg["type"] in TYPES_META:
+                index = "meta"
+                schema = mc_s.schema_meta
+            elif msg["type"] in TYPES_INT:
+                index = "internal"
+                schema = mc_s.schema_int
+            # normalise fields
+            for key, value in list(msg.items()):
+                if value is None:
+                    del msg[key]
+                if key in schema:
+                    if isinstance(value, int):
+                        if schema[key].startswith("string") or schema[key].startswith(
+                            "text"
+                        ):
+                            msg[key] = str(value)
+
+            body = {"insert": {"index": index, "doc": msg}}
+            total.append(body)
+            if "ts" not in msg:
+                raise Exception("No TS in msg")
+            if index not in indexmap:
+                indexmap[index] = [msg]
+            else:
+                indexmap[index].append(msg)
+        # END MSG IN MESSAGES
+
+        # Pack the indexmap with msgpack and publish it to Neptune
+        packed_index = msgpack.packb(indexmap, use_bin_type=True)
+        completed_publish = False
+        for i in range(10):
+            if completed_publish:
+                break
+            try:
+                await pr.publish(MESSAGE_KEY, packed_index)
+                completed_publish = True
+            except aioredis.exceptions.ConnectionError as e:
+                raise e
+                await asyncio.sleep(0.1)
+        if not completed_publish:
+            log.error("Failed to publish to Neptune")
+
+        body_post = ""
+        for item in total:
+            # print("ITEM", item)
+            body_post += orjson.dumps(item).decode("utf-8")
+            body_post += "\n"
+
+        # print("BODY POST INDEX", index, body_post)
+
+        try:
+            # Bulk index operations
+            api_response = api_instance.bulk(body_post)  # , async_req=True
+        except ApiException as e:
+            log.error("Exception when calling IndexApi->bulk: %s\n" % e)
+            log.error("body_post attempted to send", body_post)
+        log.info(f"Completed ingest to MC of length {len(total)}")
+    # END MESSAGES IN SPLIT
+
+
+def update_schema():
+    pass
+
+
+def create_index(api_client):
+    util_instance = manticoresearch.UtilsApi(api_client)
+    schemas = {
+        "main": mc_s.schema_main,
+        "rule_storage": mc_s.schema_rule_storage,
+        "meta": mc_s.schema_meta,
+        "internal": mc_s.schema_int,
+    }
+    for name, schema in schemas.items():
+        schema_types = ", ".join([f"{k} {v}" for k, v in schema.items()])
+
+        create_query = (
+            f"create table if not exists {name}({schema_types}) engine='columnar'"
+        )
+        print("Schema types", create_query)
+        util_instance.sql(create_query)
+
+
+async def queue_message(msg):
+    """
+    Queue a message on the Redis buffer.
+    """
+    # TODO: msgpack
+    message = orjson.dumps(msg)
+    await ar.lpush(KEYNAME, message)
+
+
+async def queue_message_bulk(data):
+    """
+    Queue multiple messages on the Redis buffer.
+    """
+    for msg in data:
+        # TODO: msgpack
+        message = orjson.dumps(msg)
+        await ar.lpush(KEYNAME, message)

db_old_ref.py

@@ -0,0 +1,174 @@
+import asyncio
+from os import getenv
+
+import aioredis
+import msgpack
+import orjson
+import redis
+
+# Elasticsearch
+from elasticsearch import AsyncElasticsearch
+
+import util
+
+trues = ("true", "1", "t", True)
+
+# INDEX = "msg"
+
+log = util.get_logger("db")
+
+# Redis (legacy)
+# r = redis.from_url("redis://ssdb:1289", db=0)
+
+# AIORedis
+ar = aioredis.from_url("redis://ssdb:1289", db=0)
+
+# Neptune redis for PubSub
+pr = aioredis.from_url("redis://redis_neptune:6379", db=10)
+
+TYPES_MAIN = [
+    "msg",
+    "notice",
+    "action",
+    "part",
+    "join",
+    "kick",
+    "quit",
+    "nick",
+    "mode",
+    "topic",
+    "update",
+]
+MAIN_SRC_MAP = {
+    "dis": "main",
+    "irc": "restricted",
+    "4ch": "main",
+}
+
+TYPES_META = ["who"]
+TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
+KEYNAME = "queue"
+MESSAGE_KEY = "messages"
+
+ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "elastic")
+ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "changeme")
+ELASTICSEARCH_HOST = getenv("ELASTICSEARCH_HOST", "localhost")
+ELASTICSEARCH_TLS = getenv("ELASTICSEARCH_TLS", "false") in trues
+
+client = None
+
+# These are sometimes numeric, sometimes strings.
+# If they are seen to be numeric first, ES will erroneously
+# index them as "long" and then subsequently fail to index messages
+# with strings in the field.
+keyword_fields = ["nick_id", "user_id", "net_id"]
+
+mapping_int = {
+    "mappings": {
+        "properties": {
+            "ts": {"type": "date", "format": "epoch_second"},
+            "file_tim": {"type": "date", "format": "epoch_millis"},
+        }
+    }
+}
+mapping = dict(mapping_int)
+for field in keyword_fields:
+    mapping["mappings"]["properties"][field] = {"type": "text"}
+
+
+del mapping_int["mappings"]["properties"]["file_tim"]
+
+
+async def initialise_elasticsearch():
+    """
+    Initialise the Elasticsearch client.
+    """
+    auth = (ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD)
+    client = AsyncElasticsearch(ELASTICSEARCH_HOST, http_auth=auth, verify_certs=False)
+    for index in ("main", "meta", "restricted", "internal"):
+        if index == "internal":
+            map_dict = mapping_int
+        else:
+            map_dict = mapping
+        if await client.indices.exists(index=index):
+            # update index with mapping
+            await client.indices.put_mapping(
+                index=index, properties=map_dict["mappings"]["properties"]
+            )
+        else:
+            await client.indices.create(index=index, mappings=map_dict["mappings"])
+    return client
+
+
+async def store_batch(data):
+    global client
+    if not client:
+        client = await initialise_elasticsearch()
+    indexmap = {}
+    for msg in data:
+        if msg["type"] in TYPES_MAIN:
+            # index = "main"
+            index = MAIN_SRC_MAP[msg["src"]]
+            # schema = mc_s.schema_main
+        elif msg["type"] in TYPES_META:
+            index = "meta"
+            # schema = mc_s.schema_meta
+        elif msg["type"] in TYPES_INT:
+            index = "internal"
+            # schema = mc_s.schema_int
+
+        INDEX = index
+
+        # if key in schema:
+        #     if isinstance(value, int):
+        #         if schema[key].startswith("string") or schema[key].startswith(
+        #             "text"
+        #         ):
+        #             msg[key] = str(value)
+        # body = orjson.dumps(msg)
+        if "ts" not in msg:
+            raise Exception("No TS in msg")
+        if INDEX not in indexmap:
+            indexmap[INDEX] = [msg]
+        else:
+            indexmap[INDEX].append(msg)
+
+    # Pack the indexmap with msgpack and publish it to Neptune
+    packed_index = msgpack.packb(indexmap, use_bin_type=True)
+    completed_publish = False
+    for i in range(10):
+        if completed_publish:
+            break
+        try:
+            await pr.publish(MESSAGE_KEY, packed_index)
+            completed_publish = True
+        except aioredis.exceptions.ConnectionError:
+            await asyncio.sleep(0.1)
+    if not completed_publish:
+        log.error("Failed to publish to Neptune")
+
+    for index, index_messages in indexmap.items():
+        for message in index_messages:
+            result = await client.index(index=index, body=message)
+            if not result["result"] == "created":
+                log.error(f"Indexing failed: {result}")
+    log.debug(f"Indexed {len(data)} messages in ES")
+
+
+async def queue_message(msg):
+    """
+    Queue a message on the Redis buffer.
+    """
+    # TODO: msgpack
+    message = orjson.dumps(msg)
+    await ar.lpush(KEYNAME, message)
+
+
+async def queue_message_bulk(data):
+    """
+    Queue multiple messages on the Redis buffer.
+    """
+    for msg in data:
+        # TODO: msgpack
+        message = orjson.dumps(msg)
+        await ar.lpush(KEYNAME, message)

docker-compose.prod.yml

@@ -0,0 +1,206 @@
+version: "2.2"
+
+services:
+  rts:
+    image: xf/monolith:latest
+    container_name: rts_monolith
+    command: sh -c '. /venv/bin/activate && exec python rts.py'
+    build: .
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - type: bind
+        source: /code/run
+        target: /var/run
+    environment: 
+      PORTAINER_GIT_DIR: "${PORTAINER_GIT_DIR}"
+      MODULES_ENABLED: "${MODULES_ENABLED}"
+      MONOLITH_RTS_MEXC_API_ACCESS_KEY: "${MONOLITH_RTS_MEXC_API_ACCESS_KEY}"
+      MONOLITH_RTS_MEXC_API_SECRET_KEY: "${MONOLITH_RTS_MEXC_API_SECRET_KEY}"
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 1.0G
+    network_mode: host
+
+  app:
+    image: xf/monolith:latest
+    container_name: monolith
+    #command: sh -c '. /venv/bin/activate && exec python -m cProfile -o /tmp/profile.out monolith.py'
+    build: .
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - type: bind
+        source: /code/run
+        target: /var/run
+    environment: 
+      PORTAINER_GIT_DIR: "${PORTAINER_GIT_DIR}"
+      MODULES_ENABLED: "${MODULES_ENABLED}"
+      DISCORD_TOKEN: "${DISCORD_TOKEN}"
+      THRESHOLD_LISTENER_HOST: "${THRESHOLD_LISTENER_HOST}"
+      THRESHOLD_LISTENER_PORT: "${THRESHOLD_LISTENER_PORT}"
+      THRESHOLD_LISTENER_SSL: "${THRESHOLD_LISTENER_SSL}"
+      THRESHOLD_RELAY_ENABLED: "${THRESHOLD_RELAY_ENABLED}"
+      THRESHOLD_RELAY_HOST: "${THRESHOLD_RELAY_HOST}"
+      THRESHOLD_RELAY_PORT: "${THRESHOLD_RELAY_PORT}"
+      THRESHOLD_RELAY_SSL: "${THRESHOLD_RELAY_SSL}"
+      THRESHOLD_API_ENABLED: "${THRESHOLD_API_ENABLED}"
+      THRESHOLD_API_HOST: "${THRESHOLD_API_HOST}"
+      THRESHOLD_API_PORT: "${THRESHOLD_API_PORT}"
+      THRESHOLD_CONFIG_DIR: "${THRESHOLD_CONFIG_DIR}"
+      #THRESHOLD_TEMPLATE_DIR: "${#THRESHOLD_TEMPLATE_DIR}"
+      THRESHOLD_CERT_DIR: "${THRESHOLD_CERT_DIR}"
+      # How many messages to ingest at once from Redis
+      MONOLITH_INGEST_CHUNK_SIZE: "${MONOLITH_INGEST_CHUNK_SIZE}"
+      # Time to wait between polling Redis again
+      MONOLITH_INGEST_ITER_DELAY: "${MONOLITH_INGEST_ITER_DELAY}"
+      # Number of 4chan threads to request at once
+      MONOLITH_CH4_THREADS_CONCURRENT: "${MONOLITH_CH4_THREADS_CONCURRENT}"
+      # Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
+      MONOLITH_CH4_THREADS_DELAY: "${MONOLITH_CH4_THREADS_DELAY}"
+      # Time to wait after finishing a crawl before starting again
+      MONOLITH_CH4_CRAWL_DELAY: "${MONOLITH_CH4_CRAWL_DELAY}"
+      # Semaphore value
+      MONOLITH_CH4_THREADS_SEMAPHORE: "${MONOLITH_CH4_THREADS_SEMAPHORE}"
+      # Threads to use for data processing
+      # Leave uncommented to use all available threads
+      MONOLITH_PROCESS_THREADS: "${MONOLITH_PROCESS_THREADS}"
+      # Enable performance metrics after message processing
+      MONOLITH_PROCESS_PERFSTATS: "${MONOLITH_PROCESS_PERFSTATS}"
+      MONOLITH_PROCESS_TARGET_CPU_USAGE: "${MONOLITH_PROCESS_TARGET_CPU_USAGE}"
+      MONOLITH_CH4_TARGET_CPU_USAGE: "${MONOLITH_CH4_TARGET_CPU_USAGE}"
+      MONOLITH_CH4_BOARDS: "${MONOLITH_CH4_BOARDS}"
+      REDIS_PASSWORD: "${REDIS_PASSWORD}"
+      MONOLITH_INGEST_INCREASE_BELOW: "${MONOLITH_INGEST_INCREASE_BELOW}"
+      MONOLITH_INGEST_INCREASE_BY: "${MONOLITH_INGEST_INCREASE_BY}"
+      MONOLITH_INGEST_DECREASE_ABOVE: "${MONOLITH_INGEST_DECREASE_ABOVE}"
+      MONOLITH_INGEST_DECREASE_BY: "${MONOLITH_INGEST_DECREASE_BY}"
+      MONOLITH_INGEST_MAX: "${MONOLITH_INGEST_MAX}"
+      MONOLITH_INGEST_MIN: "${MONOLITH_INGEST_MIN}"
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 1.0G
+    network_mode: host
+
+
+  threshold:
+    image: xf/threshold:latest
+    container_name: threshold
+    build: legacy/docker
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
+      #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
+      - ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
+    volumes_from:
+      - tmp
+    ports:
+      - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
+      - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
+      - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
+    environment: 
+      PORTAINER_GIT_DIR: "${PORTAINER_GIT_DIR}"
+      MODULES_ENABLED: "${MODULES_ENABLED}"
+      DISCORD_TOKEN: "${DISCORD_TOKEN}"
+      THRESHOLD_LISTENER_HOST: "${THRESHOLD_LISTENER_HOST}"
+      THRESHOLD_LISTENER_PORT: "${THRESHOLD_LISTENER_PORT}"
+      THRESHOLD_LISTENER_SSL: "${THRESHOLD_LISTENER_SSL}"
+      THRESHOLD_RELAY_ENABLED: "${THRESHOLD_RELAY_ENABLED}"
+      THRESHOLD_RELAY_HOST: "${THRESHOLD_RELAY_HOST}"
+      THRESHOLD_RELAY_PORT: "${THRESHOLD_RELAY_PORT}"
+      THRESHOLD_RELAY_SSL: "${THRESHOLD_RELAY_SSL}"
+      THRESHOLD_API_ENABLED: "${THRESHOLD_API_ENABLED}"
+      THRESHOLD_API_HOST: "${THRESHOLD_API_HOST}"
+      THRESHOLD_API_PORT: "${THRESHOLD_API_PORT}"
+      THRESHOLD_CONFIG_DIR: "${THRESHOLD_CONFIG_DIR}"
+      #THRESHOLD_TEMPLATE_DIR: "${#THRESHOLD_TEMPLATE_DIR}"
+      THRESHOLD_CERT_DIR: "${THRESHOLD_CERT_DIR}"
+      # How many messages to ingest at once from Redis
+      MONOLITH_INGEST_CHUNK_SIZE: "${MONOLITH_INGEST_CHUNK_SIZE}"
+      # Time to wait between polling Redis again
+      MONOLITH_INGEST_ITER_DELAY: "${MONOLITH_INGEST_ITER_DELAY}"
+      # Number of 4chan threads to request at once
+      MONOLITH_CH4_THREADS_CONCURRENT: "${MONOLITH_CH4_THREADS_CONCURRENT}"
+      # Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
+      MONOLITH_CH4_THREADS_DELAY: "${MONOLITH_CH4_THREADS_DELAY}"
+      # Time to wait after finishing a crawl before starting again
+      MONOLITH_CH4_CRAWL_DELAY: "${MONOLITH_CH4_CRAWL_DELAY}"
+      # Semaphore value
+      MONOLITH_CH4_THREADS_SEMAPHORE: "${MONOLITH_CH4_THREADS_SEMAPHORE}"
+      # Threads to use for data processing
+      # Leave uncommented to use all available threads
+      MONOLITH_PROCESS_THREADS: "${MONOLITH_PROCESS_THREADS}"
+      # Enable performance metrics after message processing
+      MONOLITH_PROCESS_PERFSTATS: "${MONOLITH_PROCESS_PERFSTATS}"
+      MONOLITH_CH4_BOARDS: "${MONOLITH_CH4_BOARDS}"
+      REDIS_PASSWORD: "${REDIS_PASSWORD}"
+    # for development
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    network_mode: host
+
+  ssdb:
+    image: tsl0922/ssdb
+    container_name: ssdb_monolith
+    ports:
+      - "1289:1289"
+    environment:
+      - SSDB_PORT=1289
+    volumes:
+      - monolith_ssdb_data:/var/lib/ssdb
+    # networks:
+    #   - default
+    #   - db
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 1.0G
+    network_mode: host
+
+  redis:
+    image: redis
+    container_name: redis_monolith
+    command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+      - monolith_redis_data:/data
+      - type: bind
+        source: /code/run
+        target: /var/run
+    # volumes_from:
+    #   - tmp
+    healthcheck:
+      test: "redis-cli ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
+    # networks:
+    #   - default
+    #   - xf
+    #   - db
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 1.0G
+    network_mode: host
+
+# networks:
+#   default:
+#     driver: bridge
+#   xf:
+#     external: true
+#   db:
+#     external: true
+
+volumes:
+  monolith_redis_data:
+  monolith_ssdb_data:

docker/redis.conf

@@ -0,0 +1,5 @@
+unixsocket /var/run/monolith-redis.sock
+unixsocketperm 777
+port 0
+# port 6379
+# requirepass changeme

env.example

@@ -0,0 +1,50 @@
+PORTAINER_GIT_DIR=..
+MODULES_ENABLED="dis"
+DISCORD_TOKEN=
+THRESHOLD_LISTENER_HOST=0.0.0.0
+THRESHOLD_LISTENER_PORT=13867
+THRESHOLD_LISTENER_SSL=1
+
+THRESHOLD_RELAY_ENABLED=0
+THRESHOLD_RELAY_HOST=0.0.0.0
+THRESHOLD_RELAY_PORT=13868
+THRESHOLD_RELAY_SSL=1
+
+THRESHOLD_API_ENABLED=1
+THRESHOLD_API_HOST=0.0.0.0
+THRESHOLD_API_PORT=13869
+
+THRESHOLD_CONFIG_DIR=../legacy/conf/live/
+#THRESHOLD_TEMPLATE_DIR=../legacy/conf/templates/
+THRESHOLD_CERT_DIR=../legacy/conf/cert/
+
+# How many messages to ingest at once from Redis
+MONOLITH_INGEST_CHUNK_SIZE=70000
+
+# Time to wait between polling Redis again
+MONOLITH_INGEST_ITER_DELAY=2
+
+# Number of 4chan threads to request at once
+MONOLITH_CH4_THREADS_CONCURRENT=1000
+
+# Time to wait between every MONOLITH_CH4_THREADS_CONCURRENT threads
+MONOLITH_CH4_THREADS_DELAY=0.1
+
+# Time to wait after finishing a crawl before starting again
+MONOLITH_CH4_CRAWL_DELAY=60
+
+# Semaphore value
+MONOLITH_CH4_THREADS_SEMAPHORE=1000
+
+# Threads to use for data processing
+# Leave uncommented to use all available threads
+MONOLITH_PROCESS_THREADS=7
+
+# Enable performance metrics after message processing
+MONOLITH_PROCESS_PERFSTATS=0
+
+# Elasticsearch
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=
+ELASTICSEARCH_HOST=https://es01:9200
+ELASTICSEARCH_TLS=1

legacy/.env.example

@@ -0,0 +1,13 @@
+THRESHOLD_LISTENER_HOST=0.0.0.0
+THRESHOLD_LISTENER_PORT=13867
+THRESHOLD_LISTENER_SSL=1
+
+THRESHOLD_RELAY_ENABLED=1
+THRESHOLD_RELAY_HOST=0.0.0.0
+THRESHOLD_RELAY_PORT=13868
+THRESHOLD_RELAY_SSL=1
+
+THRESHOLD_API_ENABLED=1
+THRESHOLD_API_HOST=0.0.0.0
+THRESHOLD_API_PORT=13869
+PORTAINER_GIT_DIR=.

legacy/.gitignore

@@ -0,0 +1,11 @@
+*.pyc
+*.pem
+*.swp
+__pycache__/
+conf/live/
+conf/cert/
+env/
+venv/
+.idea/
+.env
+.bash_history

legacy/api/views.py

@@ -0,0 +1,746 @@
+import functools
+from json import JSONDecodeError, dumps, loads
+from string import digits
+
+import main
+from klein import Klein
+from modules import chankeep, helpers, provision, regproc, userinfo
+from modules.network import Network
+from twisted.web.server import Request
+from utils.logging.log import warn
+
+
+def login_required(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if isinstance(args[0], Request):
+            request = args[0]
+            apikey = request.getHeader("ApiKey")
+            token = request.getHeader("Token")
+            if not apikey:
+                return "No API key provided"
+            if not token:
+                return "No token provided"
+            if apikey not in main.tokens:
+                return "No such API key"
+            config_token = main.tokens[apikey]
+            if not token == config_token["hello"]:
+                return "Invalid token"
+            counter = config_token["counter"]
+            request.setHeader("Counter", counter)
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class API(object):
+    """
+    Our API webapp.
+    """
+
+    app = Klein()
+
+    @app.route("/", methods=["GET"])
+    @login_required
+    def hello(self, request):
+        return "Hello"
+
+    @app.route("/who/<query>/", methods=["POST"])
+    @login_required
+    def who(self, request, query):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "query" not in data:
+            return "No query provided"
+        result = userinfo.getWho(data["query"])
+        # Expand the generator
+        return dumps({k: [x for x in v] for k, v in result.items()})
+
+    @app.route("/chans/", methods=["POST"])
+    @login_required
+    def chans(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No query provided"
+        if not data["query"]:
+            warn(f"No query provided: for chans {data}")
+            return dumps({})
+        result = userinfo.getChansSingle(data["net"], data["query"])
+        if not result:
+            return dumps({})
+        return dumps({"chans": [x for x in result]})
+
+    @app.route("/users/", methods=["POST"])
+    @login_required
+    def users(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No query provided"
+        if not data["query"]:
+            warn(f"No query provided for users: {data}")
+            return dumps({})
+        result = userinfo.getUsersSingle(data["net"], data["query"])
+        if not result:
+            return dumps({})
+        return dumps({"users": [x for x in result]})
+
+    @app.route("/online/", methods=["POST"])
+    @login_required
+    def online(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No users provided"
+        if not data["query"]:
+            warn(f"No users provided: for online {data}")
+            return dumps({})
+        net = data["net"]
+        usermap = {}
+        for user in data["query"]:
+            channels = userinfo.getChansSingle(net, [user])
+            if channels:
+                usermap[user] = True
+            else:
+                usermap[user] = False
+
+        return dumps(usermap)
+
+    @app.route("/num_users/", methods=["POST"])
+    @login_required
+    def num_users(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No users provided"
+        if not data["query"]:
+            warn(f"No chans provided: for num_users {data}")
+            return dumps({})
+        net = data["net"]
+        results = userinfo.getUserNum(net, data["query"])
+
+        return dumps(results)
+
+    @app.route("/num_chans/", methods=["POST"])
+    @login_required
+    def num_chans(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "net" not in data:
+            return "No net provided"
+        if "query" not in data:
+            return "No users provided"
+        if not data["query"]:
+            warn(f"No users provided: for num_chans {data}")
+            return dumps({})
+        net = data["net"]
+        results = userinfo.getChanNum(net, data["query"])
+
+        return dumps(results)
+
+    @app.route("/irc/stats/", methods=["POST"])
+    @login_required
+    def irc_stats(self, request):
+        stats = {}
+        numChannels = 0
+        numWhoEntries = 0
+        for i in main.IRCPool.keys():
+            numChannels += len(main.IRCPool[i].channels)
+        numWhoEntries += userinfo.getNumTotalWhoEntries()
+        numRelays = 0
+        for net in main.network.keys():
+            numRelays += len(main.network[net].relays)
+        stats["servers_total_total"] = numRelays
+        stats["servers_total_unique"] = len(main.network.keys())
+        stats["servers_online_total"] = len(main.IRCPool.keys())
+        stats["servers_online_unique"] = len(main.liveNets())
+        stats["channels"] = numChannels
+        stats["records"] = numWhoEntries
+        stats["eventrate"] = main.lastMinuteSample
+        return dumps(stats)
+
+    @app.route("/irc/networks/", methods=["POST"])
+    @login_required
+    def irc_networks(self, request):
+        networks = {}
+        for net in main.network.keys():
+            networks[net] = {
+                "active": chankeep.allRelaysActive(net),
+                "relays": len(main.network[net].relays),
+                "channels": userinfo.getTotalChanNum(net),
+                "records": userinfo.getNumWhoEntries(net),
+            }
+        return dumps(networks)
+
+    @app.route("/irc/auth/", methods=["POST"])
+    @login_required
+    def irc_network_recheckauth(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "func" not in data:
+            return dumps({"success": False, "reason": "no function specified."})
+        func = data["func"]
+
+        if "net" not in data:
+            return dumps({"success": False, "reason": "no net specified."})
+        net = data["net"]
+        if not net or net == "None":
+            nets = main.network.keys()
+        else:
+            if net not in main.network.keys():
+                return dumps({"success": False, "reason": "no such net."})
+            nets = [net]
+        for net_name in nets:
+            conns = helpers.get_connected_relays(net_name)
+            if not conns:
+                return dumps(
+                    {
+                        "success": False,
+                        "reason": f"failed to get instances for {net_name}.",
+                    }
+                )
+            if func == "recheckauth":
+                for conn in conns:
+                    conn.regPing(reset=True)
+            elif func == "resetauth":
+                for conn in conns:
+                    conn.authenticated = False
+                    conn.regPing(reset=True)
+            elif func == "register":
+                for conn in conns:
+                    regproc.registerAccount(conn.net, conn.num)
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/", methods=["POST"])
+    @login_required
+    def irc_network(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        inst = main.network[net]
+        network = {}
+        network["net"] = inst.net
+        network["auth"] = inst.auth
+        network["host"] = inst.host
+        network["last"] = inst.last
+        network["port"] = inst.port
+        network["security"] = inst.security
+        network["relays"] = len(inst.relays)
+        network["chanlimit"] = inst.chanlimit
+        network["channels"] = userinfo.getTotalChanNum(net)
+        network["records"] = userinfo.getNumWhoEntries(net)
+        return dumps(network)
+
+    @app.route("/irc/network/<net>/", methods=["DELETE"])
+    @login_required
+    def irc_network_delete(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        main.network[net].seppuku()
+        del main.network[net]
+        main.saveConf("network")
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/edit/", methods=["POST"])
+    @login_required
+    def irc_network_edit(self, request, net):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        inst = main.network[net]
+        for item in data:
+            if item == "auth":
+                auth = data[item][0]
+                if auth not in ["sasl", "ns", "none"]:
+                    return dumps({"success": False, "reason": "invalid auth."})
+            elif item == "host":
+                host = data[item][0]
+            elif item == "last":
+                last = data[item][0]
+                if not last.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid last: not a number."}
+                    )
+            elif item == "port":
+                port = data[item][0]
+                if not port.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid port: not a number."}
+                    )
+                port = int(port)
+            elif item == "chanlimit":
+                chanlimit = data[item][0]
+                if chanlimit == "None":
+                    chanlimit = None
+                elif not chanlimit.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid chanlimit: not a number."}
+                    )
+                else:
+                    chanlimit = int(chanlimit)
+                    online_relays = helpers.get_connected_relays(net)
+                    for r in online_relays:
+                        r.chanlimit = chanlimit
+            elif item == "security":
+                security = data[item][0]
+                if security not in ["ssl", "plain"]:
+                    return dumps({"success": False, "reason": "invalid security."})
+        inst.auth = auth
+        inst.host = host
+        inst.last = last
+        inst.port = port
+        inst.security = security
+        inst.chanlimit = chanlimit
+        main.saveConf("network")
+
+        return dumps({"success": True})
+
+    @app.route("/irc/network/create/", methods=["PUT"])
+    @login_required
+    def irc_network_create(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        fields = ["net", "auth", "host", "port", "security"]
+        if not set(fields).issubset(set(data)):
+            return dumps({"success": False, "reason": "not enough fields."})
+        for item in data:
+            if item == "net":
+                net = data[item]
+                if net in main.network.keys():
+                    return dumps(
+                        {"success": False, "reason": "network already exists."}
+                    )
+                if set(net).intersection(set(digits)):
+                    return dumps(
+                        {
+                            "success": False,
+                            "reason": "network name cannot contain numbers.",
+                        }
+                    )
+            elif item == "auth":
+                auth = data[item]
+                if auth not in ["sasl", "ns", "none"]:
+                    return dumps({"success": False, "reason": "invalid auth."})
+            elif item == "host":
+                host = data[item]
+            elif item == "port":
+                port = data[item]
+                if not port.isdigit():
+                    return dumps(
+                        {"success": False, "reason": "invalid port: not a number."}
+                    )
+                port = int(port)
+            elif item == "security":
+                security = data[item]
+                if security not in ["ssl", "plain"]:
+                    return dumps({"success": False, "reason": "invalid security."})
+        main.network[net] = Network(net, host, int(port), security, auth)
+        main.saveConf("network")
+
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/relays/", methods=["POST"])
+    @login_required
+    def irc_network_relays(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        relays_inst = main.network[net].relays
+        relays = []
+        for num in relays_inst.keys():
+            to_append = relays_inst[num]
+            name = f"{net}{num}"
+            if name in main.IRCPool.keys():
+                to_append["chans"] = len(main.IRCPool[name].channels)
+                to_append["nick"] = main.IRCPool[name].nickname
+                to_append["conn"] = main.IRCPool[name].isconnected
+                to_append["limit"] = main.IRCPool[name].chanlimit
+                to_append["authed"] = main.IRCPool[name].authenticated
+            else:
+                to_append["chans"] = 0
+                to_append["nick"] = None
+                to_append["conn"] = False
+                to_append["limit"] = None
+                to_append["authed"] = None
+            relays.append(to_append)
+        return dumps({"relays": relays})
+
+    @app.route("/irc/network/<net>/<num>/", methods=["POST"])
+    @login_required
+    def irc_network_relay(self, request, net, num):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num not in net_inst.relays:
+            return dumps({"success": False, "reason": "network has no such relay."})
+        if "status" in data:
+            if not type(data["status"]) == int:
+                return dumps({"success": False, "reason": "invalid type for enabled."})
+            enabled = data["status"]
+            if enabled:
+                net_inst.enable_relay(num)
+            else:
+                net_inst.disable_relay(num)
+        main.saveConf("network")
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/<num>/", methods=["PUT"])
+    @login_required
+    def irc_network_relay_add(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num in net_inst.relays:
+            return dumps(
+                {"success": False, "reason": "network already has this relay."}
+            )
+        id, alias = net_inst.add_relay(num)
+        main.saveConf("network")
+        return dumps({"success": True, "id": id, "alias": alias})
+
+    @app.route("/irc/network/<net>/", methods=["PUT"])
+    @login_required
+    def irc_network_relay_add_next(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        net_inst = main.network[net]
+        id, alias = net_inst.add_relay()
+        main.saveConf("network")
+        return dumps({"success": True, "id": id, "alias": alias})
+
+    @app.route("/irc/network/<net>/<num>/", methods=["DELETE"])
+    @login_required
+    def irc_network_relay_del(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num not in net_inst.relays:
+            return dumps(
+                {"success": False, "reason": "network does not have this relay."}
+            )
+        net_inst.delete_relay(num)
+        main.saveConf("network")
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/<num>/provision/", methods=["POST"])
+    @login_required
+    def irc_network_relay_provision(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        net_inst = main.network[net]
+        if num not in net_inst.relays:
+            return dumps(
+                {"success": False, "reason": "network does not have this relay."}
+            )
+        provision.provisionRelay(num, net)
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/channels/", methods=["POST"])
+    @login_required
+    def irc_network_channels(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        relays_inst = main.network[net].relays
+        channels = []
+        for num in relays_inst.keys():
+            name = f"{net}{num}"
+            if name in main.IRCPool.keys():
+                net_chans = main.IRCPool[name].channels
+                channels_annotated = userinfo.getUserNum(net, net_chans)
+                for channel in net_chans:
+                    channel_inst = {
+                        "name": channel,
+                        "users": channels_annotated[channel],
+                        "num": num,
+                    }
+                    channels.append(channel_inst)
+                    # channels[channel] = channels_annotated[channel]
+
+        return dumps({"channels": channels})
+
+    # API version
+    @app.route("/irc/network/<net>/channel/", methods=["DELETE"])
+    @login_required
+    def irc_network_channel_part_api(self, request, net):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "channel" not in data:
+            return dumps({"success": False, "reason": "no channel specified."})
+        channel = data["channel"]
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        parted = chankeep.partSingle(net, channel)
+        if not parted:
+            dumps({"success": False, "reason": "no channels matched."})
+        return dumps({"success": True, "relays": parted})
+
+    # API version
+    @app.route("/irc/network/<net>/channel/", methods=["PUT"])
+    @login_required
+    def irc_network_channel_join_api(self, request, net):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "channel" not in data:
+            return dumps({"success": False, "reason": "no channel specified."})
+        channel = data["channel"]
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        joined = chankeep.joinSingle(net, channel)
+        if not joined:
+            return dumps(
+                {"success": False, "reason": "could not allocate channel to relay."}
+            )
+        return dumps({"success": True, "relays": joined})
+
+    @app.route("/irc/network/<net>/channel/<channel>/", methods=["PUT"])
+    @login_required
+    def irc_network_channel_join(self, request, net, channel):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        joined = chankeep.joinSingle(net, channel)
+        if not joined:
+            return dumps(
+                {"success": False, "reason": "could not allocate channel to relay."}
+            )
+        return dumps({"success": True, "relays": joined})
+
+    @app.route("/aliases/", methods=["GET"])
+    @login_required
+    def aliases(self, request):
+        alias_list = []
+        for num, alias in main.alias.items():
+            alias_dup = dict(alias)
+            alias_dup["num"] = num
+            alias_list.append(alias_dup)
+
+        return dumps({"aliases": alias_list})
+
+    @app.route("/aliases/", methods=["POST"])
+    @login_required
+    def aliases_update(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        for alias, fields in data.items():
+            if not alias.isdigit():
+                return dumps({"success": False, "reason": "alias not a number."})
+            alias = int(alias)
+            if alias not in main.alias.keys():
+                return dumps({"success": False, "reason": "alias does not exist."})
+            if fields:
+                for field in fields:
+                    if field in main.alias[alias]:
+                        main.alias[alias][field] = fields[field]
+            if "emails" in fields:
+                if not fields["emails"]:
+                    main.alias[alias]["emails"] = []
+        main.saveConf("alias")
+
+        return dumps({"success": True})
+
+    @app.route("/irc/auto/<net>/", methods=["POST"])
+    @login_required
+    def irc_auto_network(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if 1 in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"first relay exists on {net}"})
+        num, alias = main.network[net].add_relay(1)
+        provision.provisionRelay(num, net)
+        main.saveConf("network")
+
+        return dumps(
+            {
+                "success": True,
+                "message": f"created relay {num} with alias {alias} on {net}",
+            }
+        )
+
+    @app.route("/irc/list/<net>/", methods=["POST"])
+    @login_required
+    def irc_list_network(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        first_relay = helpers.get_first_relay(net)
+        if not first_relay:
+            return dumps(
+                {"success": False, "reason": f"could not get first relay for {net}"}
+            )
+        first_relay.list()
+        return dumps(
+            {
+                "success": True,
+                "message": f"requested list with instance {first_relay.num} of {net}",
+            }
+        )
+
+    @app.route("/irc/list/<net>/", methods=["GET"])
+    @login_required
+    def get_irc_list_info(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        listinfo = chankeep.getListInfo(net)
+        return dumps({"success": True, "listinfo": listinfo})
+
+    @app.route("/irc/msg/<net>/<num>/", methods=["PUT"])
+    @login_required
+    def irc_send_message(self, request, net, num):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        if "msg" not in data:
+            return dumps({"success": False, "reason": "no msg."})
+        if "channel" not in data:
+            return dumps({"success": False, "reason": "no msg."})
+        msg = data["msg"]
+        channel = data["channel"]
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        if num not in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"no relay {num} on {net}"})
+        name = f"{net}{num}"
+        if name not in main.IRCPool.keys():
+            return dumps({"success": False, "reason": f"relay {num} not on {net}"})
+        # We are in a query
+        in_query = False
+        if "nick" in data:
+            nick = data["nick"]
+            if nick == channel:
+                in_query = True
+        else:
+            nick = None
+        if channel == main.IRCPool[name].nickname or nick == channel:
+            in_query = True
+            if not nick:
+                return dumps({"success": False, "reason": "no nick specified to query"})
+            else:
+                main.IRCPool[name].sendmsg(nick, msg, in_query=in_query)
+        else:
+            main.IRCPool[name].sendmsg(channel, msg, in_query=in_query)
+        return dumps(
+            {"success": True, "message": f"sent message to {channel} on {name}"}
+        )
+
+    @app.route("/irc/nick/<net>/<num>/", methods=["GET"])
+    @login_required
+    def irc_get_nick(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        if num not in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"no relay {num} on {net}"})
+        name = f"{net}{num}"
+        if name not in main.IRCPool.keys():
+            return dumps({"success": False, "reason": f"relay {num} not on {net}"})
+        return dumps({"nickname": main.IRCPool[name].nickname})
+
+    @app.route("/irc/reg/<net>/", methods=["GET"])
+    @login_required
+    def irc_get_unreg_net(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        unreg = regproc.get_unregistered_relays(net)
+        return dumps({"success": True, "unreg": unreg})
+
+    @app.route("/irc/reg/", methods=["GET"])
+    @login_required
+    def irc_get_unreg(self, request):
+        unreg = regproc.get_unregistered_relays()
+        return dumps({"success": True, "unreg": unreg})
+
+    @app.route("/irc/reg/", methods=["PUT"])
+    @login_required
+    def irc_confirm_accounts(self, request):
+        try:
+            data = loads(request.content.read())
+        except JSONDecodeError:
+            return "Invalid JSON"
+        for item, token in data.items():
+            if "|" not in item:
+                return dumps({"success": False, "reason": f"malformed item: {item}"})
+            spl = item.split("|")
+            if not len(spl) == 2:
+                return dumps({"success": False, "reason": f"malformed item: {item}"})
+            net, num = spl
+            if not num.isdigit():
+                return dumps({"success": False, "reason": "invalid num: not a number."})
+            num = int(num)
+            if token:
+                regproc.confirmAccount(net, num, token)
+        return dumps({"success": True})
+
+    @app.route("/irc/network/<net>/<num>/auth/", methods=["POST"])
+    @login_required
+    def irc_enable_auth(self, request, net, num):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        if not num.isdigit():
+            return dumps({"success": False, "reason": "invalid num: not a number."})
+        num = int(num)
+        if num not in main.network[net].relays.keys():
+            return dumps({"success": False, "reason": f"no relay {num} on {net}"})
+        regproc.enableAuthentication(net, num, jump=False, run_now=True)
+        return dumps({"success": True})
+
+    @app.route("/irc/sinst/<net>/", methods=["GET"])
+    @login_required
+    def irc_get_authentity(self, request, net):
+        if net not in main.network.keys():
+            return dumps({"success": False, "reason": "no such net."})
+        auth = regproc.selectInst(net)
+        if not auth:
+            return dumps({"success": False, "reason": "error getting results."})
+        return dumps({"success": True, "sinst": auth})

legacy/commands/admall.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class AdmallCommand:
     def __init__(self, *args):
         self.admall(*args)
 
-    def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def admall(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 2:
                 for i in main.network.keys():

legacy/commands/alias.py

@@ -1,12 +1,15 @@
 import main
-from yaml import dump
 from modules import alias
+from yaml import dump
+
 
 class AliasCommand:
     def __init__(self, *args):
         self.alias(*args)
 
-    def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def alias(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 1:
                 info(dump(main.alias))
@@ -21,7 +24,7 @@ class AliasCommand:
                         if len(main.alias.keys()) == 0:
                             nextNum = 1
                         else:
-                            nextNum = max(main.alias.keys())+1
+                            nextNum = max(main.alias.keys()) + 1
                         main.alias[nextNum] = alias.generate_alias()
                         success("Generated new alias: %i" % nextNum)
                     main.saveConf("alias")
@@ -31,7 +34,7 @@ class AliasCommand:
                         failure("Must be a number, not %s" % spl[2])
                         return
                     num = int(spl[2])
-                    if not num in main.alias.keys():
+                    if num not in main.alias.keys():
                         failure("No such alias: %i" % num)
                         return
                     failed = False

legacy/commands/all.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class AllCommand:
     def __init__(self, *args):
         self.all(*args)
 
-    def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def all(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 2:
                 for i in main.network.keys():
@@ -14,8 +17,11 @@ class AllCommand:
                         net = main.network[i].relays[x]["net"]
                         alias = main.alias[x]["nick"]
                         commands = {spl[1]: [" ".join(spl[2:])]}
-                        success("Sending commands to relay %s as user %s" % (num, alias+"/"+net))
+                        success(
-                        deliverRelayCommands(num, commands, user=alias+"/"+net)
+                            "Sending commands to relay %s as user %s"
+                            % (num, alias + "/" + net)
+                        )
+                        deliverRelayCommands(num, commands, user=alias + "/" + net)
                 return
             else:
                 incUsage("all")

legacy/commands/allc.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class AllcCommand:
     def __init__(self, *args):
         self.allc(*args)
 
-    def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def allc(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 4:
                 targets = []
@@ -16,9 +19,12 @@ class AllcCommand:
                                 targets.append((i, x))
                 elif spl[1] == "alias":
                     for i in main.network.keys():
-                        [targets.append((i, x)) for x in main.alias.keys() if
+                        [
-                                main.alias[x]["nick"] == spl[2] and
+                            targets.append((i, x))
-                                x in main.network[i].aliases.keys()]
+                            for x in main.alias.keys()
+                            if main.alias[x]["nick"] == spl[2]
+                            and x in main.network[i].aliases.keys()
+                        ]
                 else:
                     incUsage("allc")
                     return
@@ -30,8 +36,11 @@ class AllcCommand:
                     num = i[1]
                     alias = main.alias[num]["nick"]
                     commands = {spl[3]: [" ".join(spl[4:])]}
-                    success("Sending commands to relay %i as user %s" % (num, alias+"/"+net))
+                    success(
-                    deliverRelayCommands(num, commands, user=alias+"/"+net)
+                        "Sending commands to relay %i as user %s"
+                        % (num, alias + "/" + net)
+                    )
+                    deliverRelayCommands(num, commands, user=alias + "/" + net)
                 return
             else:
                 incUsage("allc")

legacy/commands/authcheck.py

@@ -0,0 +1,43 @@
+import main
+
+
+class AuthcheckCommand:
+    def __init__(self, *args):
+        self.authcheck(*args)
+
+    def authcheck(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                results = []
+                for i in main.IRCPool.keys():
+                    num = main.IRCPool[i].num
+                    net = main.IRCPool[i].net
+                    if not main.IRCPool[i].authenticated:
+                        results.append(
+                            "%s - %s: %s" % (net, num, main.alias[num]["nick"])
+                        )
+                info("\n".join(results))
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                results = []
+                for i in main.IRCPool.keys():
+                    num = main.IRCPool[i].num
+                    net = main.IRCPool[i].net
+                    if not net == spl[1]:
+                        continue
+                    if not main.IRCPool[i].authenticated:
+                        results.append(
+                            "%s - %s: %s" % (net, num, main.alias[num]["nick"])
+                        )
+                info("\n".join(results))
+                return
+            else:
+                incUsage("authcheck")
+                return
+        else:
+            incUsage(None)

legacy/commands/auto.py

@@ -1,11 +1,14 @@
 import main
 from modules import provision
 
+
 class AutoCommand:
     def __init__(self, *args):
         self.auto(*args)
 
-    def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def auto(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 1:
                 for i in main.network.keys():
@@ -13,9 +16,15 @@ class AutoCommand:
                         info("Skipping %s - first relay exists" % i)
                     else:
                         num, alias = main.network[i].add_relay(1)
-                        success("Successfully created first relay on network %s with alias %s" % (i, alias))
+                        success(
+                            "Successfully created first relay on network %s with alias %s"
+                            % (i, alias)
+                        )
                         provision.provisionRelay(num, i)
-                        success("Started provisioning network %s on first relay for alias %s" % (i, alias))
+                        success(
+                            "Started provisioning network %s on first relay for alias %s"
+                            % (i, alias)
+                        )
                 main.saveConf("network")
                 return
             elif length == 2:
@@ -26,9 +35,15 @@ class AutoCommand:
                     failure("First relay exists on %s" % spl[1])
                     return
                 num, alias = main.network[spl[1]].add_relay(1)
-                success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias))
+                success(
+                    "Successfully created relay %i on network %s with alias %s"
+                    % (num, spl[1], alias)
+                )
                 provision.provisionRelay(num, spl[1])
-                success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias))
+                success(
+                    "Started provisioning network %s on relay %s for alias %s"
+                    % (spl[1], num, alias)
+                )
                 main.saveConf("network")
                 return
             else:

legacy/commands/blacklist.py

@@ -0,0 +1,44 @@
+import main
+from yaml import dump
+
+
+class BlacklistCommand:
+    def __init__(self, *args):
+        self.blacklist(*args)
+
+    def blacklist(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                info(dump(main.blacklist))
+                return
+            elif length == 4:
+                if spl[1] == "add":
+                    if spl[2] in main.blacklist.keys():
+                        main.blacklist[spl[2]].append(spl[3])
+                    else:
+                        main.blacklist[spl[2]] = [spl[3]]
+                    success("Blacklisted %s on %s" % (spl[3], spl[2]))
+                    main.saveConf("blacklist")
+                    return
+                elif spl[1] == "del":
+                    if spl[2] in main.blacklist.keys():
+                        if spl[3] in main.blacklist[spl[2]]:
+                            main.blacklist[spl[2]].remove(spl[3])
+                            if len(main.blacklist[spl[2]]) == 0:
+                                del main.blacklist[spl[2]]
+                        else:
+                            failure("Not in list: %s" % spl[3])
+                            return
+                    else:
+                        failure("No such entry: %s" % spl[2])
+                        return
+                    success("Removed blacklist %s on %s" % (spl[3], spl[2]))
+                    main.saveConf("blacklist")
+                    return
+            else:
+                incUsage("blacklist")
+                return
+        else:
+            incUsage(None)

legacy/commands/chans.py

@@ -1,11 +1,13 @@
-import main
 import modules.userinfo as userinfo
 
+
 class ChansCommand:
     def __init__(self, *args):
         self.chans(*args)
 
-    def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def chans(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if len(spl) < 2:
                 incUsage("chans")
@@ -16,7 +18,7 @@ class ChansCommand:
                 rtrn += "Matches from: %s" % i
                 rtrn += "\n"
                 for x in result[i]:
-                    rtrn += (x)
+                    rtrn += x
                     rtrn += "\n"
                 rtrn += "\n"
             info(rtrn)

legacy/commands/cmd.py

@@ -1,11 +1,13 @@
-import main
+from utils.deliver_relay_commands import deliverRelayCommands
-from core.bot import deliverRelayCommands
+
 
 class CmdCommand:
     def __init__(self, *args):
         self.cmd(*args)
 
-    def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def cmd(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 4:
                 if not spl[1].isdigit():

legacy/commands/confirm.py

@@ -0,0 +1,33 @@
+import main
+from modules import regproc
+
+
+class ConfirmCommand:
+    def __init__(self, *args):
+        self.confirm(*args)
+
+    def confirm(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 4:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                if not spl[2].isdigit():
+                    failure("Must be a number, not %s" % spl[2])
+                    return
+                if not int(spl[2]) in main.network[spl[1]].relays.keys():
+                    failure("No such relay on %s: %s" % (spl[1], spl[2]))
+                    return
+                regproc.confirmAccount(spl[1], int(spl[2]), spl[3])
+                success(
+                    "Requested confirmation on %s - %s with token %s"
+                    % (spl[1], spl[2], spl[3])
+                )
+                return
+            else:
+                incUsage("confirm")
+                return
+        else:
+            incUsage(None)

legacy/commands/dedup.py

@@ -0,0 +1,27 @@
+from json import dumps
+
+import main
+from modules import chankeep
+
+
+class DedupCommand:
+    def __init__(self, *args):
+        self.dedup(*args)
+
+    def dedup(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                dupes = chankeep.getDuplicateChannels()
+                chankeep.partChannels(dupes)
+                info(dumps(dupes))
+                return
+            elif length == 2:
+                if spl[1] not in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                dupes = chankeep.getDuplicateChannels(spl[1])
+                chankeep.partChannels(dupes)
+                info(dumps(dupes))
+                return

legacy/commands/disable.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class DisableCommand:
     def __init__(self, *args):
         self.disable(*args)
 
-    def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def disable(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 3:
                 if not spl[1] in main.network.keys():
@@ -15,18 +18,18 @@ class DisableCommand:
                     failure("Must be a number, not %s" % spl[2])
                     return
                 relayNum = int(spl[2])
-                name = spl[1]+spl[2]
+                name = spl[1] + spl[2]
                 if not spl[1] in main.IRCPool.keys():
                     info("Note - instance not running, proceeding anyway")
-                if not relayNum in main.network[spl[1]].relays.keys():
+                if relayNum not in main.network[spl[1]].relays.keys():
                     failure("No such relay: %s in network %s" % (spl[2], spl[1]))
                     return
                 main.network[spl[1]].relays[relayNum]["enabled"] = False
                 user = main.alias[relayNum]["nick"]
                 network = spl[1]
-                relay = main.network[spl[1]].relays[relayNum]
+                # relay = main.network[spl[1]].relays[relayNum]
                 commands = {"status": ["Disconnect"]}
-                deliverRelayCommands(relayNum, commands, user=user+"/"+network)
+                deliverRelayCommands(relayNum, commands, user=user + "/" + network)
                 main.saveConf("network")
                 if name in main.ReactorPool.keys():
                     if name in main.FactoryPool.keys():

legacy/commands/dist.py

@@ -1,11 +1,15 @@
+from subprocess import PIPE, run
+
 import main
-from subprocess import run, PIPE
+
 
 class DistCommand:
     def __init__(self, *args):
         self.dist(*args)
 
-    def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def dist(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if main.config["Dist"]["Enabled"]:
                 rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE)

legacy/commands/email.py

@@ -0,0 +1,119 @@
+import main
+from yaml import dump
+
+
+class EmailCommand:
+    def __init__(self, *args):
+        self.email(*args)
+
+    def email(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 4:
+                if spl[1] == "add":
+                    if spl[2].isdigit():
+                        num = int(spl[2])
+                        if num not in main.alias.keys():
+                            failure("No such alias: %i" % num)
+                            return
+                        if not spl[3] in main.alias[num]["emails"]:
+                            main.alias[num]["emails"].append(spl[3])
+                            main.saveConf("alias")
+                            success(
+                                "Successfully added email %s to alias %i"
+                                % (spl[3], num)
+                            )
+                            return
+                        else:
+                            failure(
+                                "Email already exists in alias %i: %s" % (num, spl[3])
+                            )
+                            return
+                    else:
+                        failure("Must be a number, not %s" % spl[2])
+                        if spl[2] == "domain":
+                            domain = spl[3]
+                            if "@" in domain:
+                                failure("Not a domain: %s" % domain)
+                                return
+                            if domain not in main.irc["_"]["domains"]:
+                                main.irc["_"]["domains"].append(domain)
+                                success(
+                                    "Successfully added domain %s to default config"
+                                    % domain
+                                )
+                                main.saveConf("irc")
+                            else:
+                                failure(
+                                    "Domain already exists in default config: %s"
+                                    % domain
+                                )
+                                return
+
+                elif spl[1] == "del":
+                    if not spl[2].isdigit():
+                        # failure("Must be a number, not %s" % spl[2])
+                        if spl[2] == "domain":
+                            domain = spl[3]
+
+                            if domain in main.irc["_"]["domains"]:
+                                main.irc["_"]["domains"].remove(domain)
+                                success(
+                                    "Successfully removed domain %s to default config"
+                                    % domain
+                                )
+                                main.saveConf("irc")
+                            else:
+                                failure(
+                                    "Domain does not exist in default config: %s"
+                                    % domain
+                                )
+                                return
+
+                    elif spl[1] == "del":
+                        if num not in main.alias.keys():
+                            failure("No such alias: %i" % num)
+                            return
+                        if spl[3] in main.alias[num]["emails"]:
+                            main.alias[num]["emails"].remove(spl[3])
+                            main.saveConf("alias")
+                            success(
+                                "Successfully removed email %s from alias %i"
+                                % (spl[3], num)
+                            )
+                            return
+                        else:
+                            failure(
+                                "Email does not exist in alias %i: %s" % (spl[3], num)
+                            )
+                            return
+            elif length == 2:
+                if spl[1] == "list":
+                    info(dump(main.alias))
+                    return
+                else:
+                    incUsage("email")
+                    return
+            elif length == 3:
+                if spl[1] == "list":
+                    if spl[2] == "domain":
+                        filtered = {
+                            f"{k}:{k2}": v2
+                            for k, v in main.irc.items()
+                            for k2, v2 in v.items()
+                            if k2 == "domains"
+                        }
+                        info(dump(filtered))
+                        return
+                    else:
+                        incUsage("email")
+                        return
+                else:
+                    incUsage("email")
+                    return
+            else:
+                incUsage("email")
+                return
+        else:
+            incUsage(None)

legacy/commands/enable.py

@@ -1,11 +1,14 @@
 import main
-from core.bot import deliverRelayCommands
+from utils.deliver_relay_commands import deliverRelayCommands
+
 
 class EnableCommand:
     def __init__(self, *args):
         self.enable(*args)
 
-    def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def enable(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 3:
                 if not spl[1] in main.network.keys():
@@ -22,9 +25,9 @@ class EnableCommand:
                 user = main.alias[int(spl[2])]["nick"]
                 network = spl[1]
                 commands = {"status": ["Connect"]}
-                deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network)
+                deliverRelayCommands(int(spl[2]), commands, user=user + "/" + network)
                 main.saveConf("network")
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     main.network[spl[1]].start_bot(int(spl[2]))
                 else:
                     pass

legacy/commands/exec.py

@@ -1,10 +1,10 @@
-import main
-
 class ExecCommand:
     def __init__(self, *args):
         self.exec(*args)
 
-    def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def exec(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length > 1:
                 try:

legacy/commands/getstr.py

@@ -0,0 +1,35 @@
+import main
+from utils.get import getRelay
+
+
+class GetstrCommand:
+    def __init__(self, *args):
+        self.getstr(*args)
+
+    def getstr(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 3:
+                net = spl[1]
+                num = spl[2]
+                if net not in main.network.keys():
+                    failure("Network does not exist: %s" % net)
+                    return
+                if not num.isdigit():
+                    failure("Must be a number, not %s" % num)
+                    return
+                num = int(num)
+                alias = main.alias[num]["nick"].lower()
+                host, port = getRelay(num)
+                password = main.config["Relay"]["Password"]
+                connstr = f"/connect -ssl {host} {port}"
+                authstr = f"/quote PASS {alias}/{net}:{password}"
+                obj.send(connstr)
+                obj.send(authstr)
+
+            else:
+                incUsage("getstr")
+                return
+        else:
+            incUsage(None)

legacy/commands/help.py

@@ -1,10 +1,13 @@
 import main
 
+
 class HelpCommand:
     def __init__(self, *args):
         self.help(*args)
 
-    def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def help(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             helpMap = []
             for i in main.help.keys():

legacy/commands/join.py

@@ -1,22 +1,33 @@
 import main
+import modules.chankeep
+
 
 class JoinCommand:
     def __init__(self, *args):
         self.join(*args)
 
-    def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def join(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
-            if length == 4:
+            if length == 3:
+                if not spl[1] in main.network.keys():
+                    failure("Network does not exist: %s" % spl[1])
+                    return
+                modules.chankeep.joinSingle(spl[1], spl[2])
+                success("Joined %s" % spl[2])
+
+            elif length == 4:
                 if not spl[1] in main.network.keys():
                     failure("Network does not exist: %s" % spl[1])
                     return
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
                     failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     failure("Name has no instance: %s" % spl[1])
                     return
-                main.IRCPool[spl[1]+spl[2]].join(spl[3])
+                main.IRCPool[spl[1] + spl[2]].join(spl[3])
                 success("Joined %s" % spl[3])
                 return
             elif length == 5:
@@ -26,10 +37,10 @@ class JoinCommand:
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
                     failure("Relay % does not exist on network %", (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     failure("Name has no instance: %s" % spl[1])
                     return
-                main.IRCPool[spl[1]+spl[2]].join(spl[3], spl[4])
+                main.IRCPool[spl[1] + spl[2]].join(spl[3], spl[4])
                 success("Joined %s with key %s" % (spl[3], spl[4]))
                 return
             else:

legacy/commands/list.py

@@ -0,0 +1,38 @@
+import main
+from modules import helpers
+
+
+class ListCommand:
+    def __init__(self, *args):
+        self.list(*args)
+
+    def list(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                for i in main.network.keys():
+                    first_relay = helpers.get_first_relay(i)
+                    ####
+                    if not first_relay:
+                        info("Network has no first instance: %s" % i)
+                        continue
+                    first_relay.list()
+                    success(f"Requested list with instance {first_relay.num} of {i}")
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                first_relay = helpers.get_first_relay(spl[1])
+                if not first_relay:
+                    failure("Could not get first instance")
+                    return
+                first_relay.list()
+                success(f"Requested list with instance {first_relay.num} of {spl[1]}")
+                return
+            else:
+                incUsage("list")
+                return
+        else:
+            incUsage(None)

legacy/commands/load.py

@@ -1,10 +1,13 @@
 import main
 
+
 class LoadCommand:
     def __init__(self, *args):
         self.load(*args)
 
-    def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def load(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if spl[1] in main.filemap.keys():

legacy/commands/loadmod.py

@@ -1,15 +1,17 @@
-import main
 from utils.loaders.single_loader import loadSingle
 
+
 class LoadmodCommand:
     def __init__(self, *args):
         self.loadmod(*args)
 
-    def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def loadmod(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 rtrn = loadSingle(spl[1])
-                if rtrn == True:
+                if rtrn is True:
                     success("Loaded module: %s" % spl[1])
                     return
                 elif rtrn == "RELOAD":

legacy/commands/logout.py

@@ -1,14 +1,12 @@
-import main
-
 class LogoutCommand:
     def __init__(self, *args):
         self.logout(*args)
 
-    def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def logout(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             obj.authed = False
-            if obj.addr in main.MonitorPool:
-                main.MonitorPool.remove(obj.addr)
             success("Logged out")
             return
         else:

legacy/commands/mod.py

@@ -1,12 +1,14 @@
 import main
-from yaml import dump
+
 
 class ModCommand:
     # This could be greatly improved, but not really important right now
     def __init__(self, *args):
         self.mod(*args)
 
-    def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def mod(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 4:
                 if not spl[1] in main.network.keys():
@@ -15,15 +17,17 @@ class ModCommand:
 
                 try:
                     setattr(main.network[spl[1]], spl[2], spl[3])
-                except e:
+                except Exception as e:
-                    failure("Something went wrong.")
+                    failure(f"Something went wrong: {e}")
                     return
 
                 main.saveConf("network")
-                success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]))
+                success(
+                    "Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])
+                )
                 return
             # Find a better way to do this
-            #elif length == 6:
+            # elif length == 6:
             #    if not spl[1] in main.network.keys():
             #        failure("Network does not exist: %s" % spl[1])
             #        return

legacy/commands/msg.py

@@ -1,25 +1,31 @@
 import main
 
+
 class MsgCommand:
     def __init__(self, *args):
         self.msg(*args)
 
-    def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def msg(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length >= 5:
                 if not spl[1] in main.network.keys():
                     failure("Network does not exist: %s" % spl[1])
                     return
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
-                    failure("Relay % does not exist on network %" % (spl[2], spl[1]))
+                    failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
                     failure("Name has no instance: %s" % spl[1])
                     return
-                if not spl[3] in main.IRCPool[spl[1]+spl[2]].channels:
+                if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels:
                     info("Bot not on channel: %s" % spl[3])
-                main.IRCPool[spl[1]+spl[2]].msg(spl[3], " ".join(spl[4:]))
+                main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:]))
-                success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1]))
+                success(
+                    "Sent %s to %s on relay %s on network %s"
+                    % (" ".join(spl[4:]), spl[3], spl[2], spl[1])
+                )
                 return
             else:
                 incUsage("msg")

legacy/commands/network.py

@@ -1,13 +1,17 @@
-import main
-from yaml import dump
-from modules.network import Network
 from string import digits
 
+import main
+from modules.network import Network
+from yaml import dump
+
+
 class NetworkCommand:
     def __init__(self, *args):
         self.network(*args)
 
-    def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def network(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 7:
                 if spl[1] == "add":
@@ -27,7 +31,9 @@ class NetworkCommand:
                         failure("Auth must be sasl, ns or none, not %s" % spl[5])
                         return
                     else:
-                        main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower())
+                        main.network[spl[2]] = Network(
+                            spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()
+                        )
                         success("Successfully created network: %s" % spl[2])
                         main.saveConf("network")
                         return

legacy/commands/part.py

@@ -1,10 +1,13 @@
 import main
 
+
 class PartCommand:
     def __init__(self, *args):
         self.part(*args)
 
-    def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def part(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 4:
                 if not spl[1] in main.network.keys():
@@ -13,10 +16,10 @@ class PartCommand:
                 if not int(spl[2]) in main.network[spl[1]].relays.keys():
                     failure("Relay % does not exist on network %", (spl[2], spl[1]))
                     return
-                if not spl[1]+spl[2] in main.IRCPool.keys():
+                if not spl[1] + spl[2] in main.IRCPool.keys():
-                    failure("Name has no instance: %s" % spl[1]+spl[2])
+                    failure("Name has no instance: %s" % spl[1] + spl[2])
                     return
-                main.IRCPool[spl[1]+spl[2]].part(spl[3])
+                main.IRCPool[spl[1] + spl[2]].part(spl[3])
                 success("Left %s" % spl[3])
                 return
             else:

legacy/commands/pass.py

@@ -1,10 +1,13 @@
 import main
 
+
 class PassCommand:
     def __init__(self, *args):
         self.password(*args)
 
-    def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def password(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             info("You are already authenticated")
             return

legacy/commands/pending.py

@@ -0,0 +1,40 @@
+import main
+
+
+class PendingCommand:
+    def __init__(self, *args):
+        self.pending(*args)
+
+    def pending(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                results = []
+                for i in main.network.keys():
+                    for x in main.network[i].relays.keys():
+                        if not main.network[i].relays[x]["registered"]:
+                            results.append(
+                                "%s: confirm %s %s [code]"
+                                % (main.alias[x]["nick"], i, x)
+                            )
+                info("\n".join(results))
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                results = []
+                for x in main.network[spl[1]].relays.keys():
+                    if not main.network[spl[1]].relays[x]["registered"]:
+                        results.append(
+                            "%s: confirm %s %s [code]"
+                            % (main.alias[x]["nick"], spl[1], x)
+                        )
+                info("\n".join(results))
+                return
+            else:
+                incUsage("pending")
+                return
+        else:
+            incUsage(None)

legacy/commands/recheckauth.py

@@ -0,0 +1,36 @@
+import main
+
+
+class RecheckauthCommand:
+    def __init__(self, *args):
+        self.recheckauth(*args)
+
+    def recheckauth(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 1:
+                for i in main.IRCPool.keys():
+                    net = main.IRCPool[i].net
+                    main.IRCPool[i].authenticated = False
+                    main.IRCPool[i].regPing()
+                success("Successfully reset authentication status")
+                return
+            elif length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                for i in main.IRCPool.keys():
+                    # num = main.IRCPool[i].num
+                    net = main.IRCPool[i].net
+                    if not net == spl[1]:
+                        continue
+                    main.IRCPool[i].authenticated = False
+                    main.IRCPool[i].regPing()
+                success("Successfully reset authentication status on %s" % spl[1])
+                return
+            else:
+                incUsage("recheckauth")
+                return
+        else:
+            incUsage(None)

legacy/commands/reg.py

@@ -0,0 +1,38 @@
+import main
+from modules import regproc
+
+
+class RegCommand:
+    def __init__(self, *args):
+        self.reg(*args)
+
+    def reg(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
+        if authed:
+            if length == 2:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                for i in main.network[spl[1]].relays.keys():
+                    regproc.registerAccount(spl[1], i)
+                success("Requested registration for all relays on %s" % spl[1])
+                return
+            elif length == 3:
+                if not spl[1] in main.network.keys():
+                    failure("No such network: %s" % spl[1])
+                    return
+                if not spl[2].isdigit():
+                    failure("Must be a number, not %s" % spl[2])
+                    return
+                if not int(spl[2]) in main.network[spl[1]].relays.keys():
+                    failure("No such relay on %s: %s" % (spl[2], spl[1]))
+                    return
+                regproc.registerAccount(spl[1], int(spl[2]))
+                success("Requested registration on %s - %s" % (spl[1], spl[2]))
+                return
+            else:
+                incUsage("reg")
+                return
+        else:
+            incUsage(None)

legacy/commands/relay.py

@@ -1,17 +1,23 @@
 import main
 from yaml import dump
 
+
 class RelayCommand:
     def __init__(self, *args):
         self.relay(*args)
 
-    def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def relay(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 3:
                 if spl[1] == "add":
                     if spl[2] in main.network.keys():
                         id, alias = main.network[spl[2]].add_relay()
-                        success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
+                        success(
+                            "Successfully created relay %s on network %s with alias %s"
+                            % (str(id), spl[2], alias)
+                        )
                         main.saveConf("network")
                         return
                     else:
@@ -34,7 +40,10 @@ class RelayCommand:
                             failure("Must be a number, not %s" % spl[3])
                             return
                         id, alias = main.network[spl[2]].add_relay(int(spl[3]))
-                        success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
+                        success(
+                            "Successfully created relay %s on network %s with alias %s"
+                            % (str(id), spl[2], alias)
+                        )
                         main.saveConf("network")
                         return
                     else:
@@ -51,7 +60,9 @@ class RelayCommand:
                         failure("No such relay: %s on network %s" % (spl[3], spl[2]))
                         return
                     main.network[spl[2]].delete_relay(int(spl[3]))
-                    success("Successfully deleted relay %s on network %s" % (spl[3], spl[2]))
+                    success(
+                        "Successfully deleted relay %s on network %s" % (spl[3], spl[2])
+                    )
                     main.saveConf("network")
                     return
                 else:

legacy/commands/save.py

@@ -1,10 +1,13 @@
 import main
 
+
 class SaveCommand:
     def __init__(self, *args):
         self.save(*args)
 
-    def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def save(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if spl[1] in main.filemap.keys():

legacy/commands/stats.py

@@ -1,13 +1,17 @@
+from string import digits
+
 import main
 import modules.counters as count
 import modules.userinfo as userinfo
-from string import digits
+
 
 class StatsCommand:
     def __init__(self, *args):
         self.stats(*args)
 
-    def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def stats(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 1:
                 stats = []
@@ -25,7 +29,7 @@ class StatsCommand:
                 stats.append("User records: %s" % numWhoEntries)
                 stats.append("Events/min: %s" % main.lastMinuteSample)
                 counterEvents = count.getEvents()
-                if counterEvents == None:
+                if counterEvents is None:
                     stats.append("No counters records")
                 else:
                     stats.append("Counters:")
@@ -42,7 +46,7 @@ class StatsCommand:
                 numNodes = 0
 
                 for i in main.IRCPool.keys():
-                    if "".join([x for x in i if not x in digits]) == spl[1]:
+                    if "".join([x for x in i if x not in digits]) == spl[1]:
                         numChannels += len(main.IRCPool[i].channels)
                         found = True
                         numNodes += 1
@@ -53,7 +57,7 @@ class StatsCommand:
                 stats.append("User records: %s" % numWhoEntries)
                 stats.append("Endpoints: %s" % numNodes)
                 counterEvents = count.getEvents(spl[1])
-                if counterEvents == None:
+                if counterEvents is None:
                     stats.append("No counters records")
                 else:
                     stats.append("Counters:")

legacy/commands/swho.py

@@ -1,10 +1,13 @@
 import main
 
+
 class SwhoCommand:
     def __init__(self, *args):
         self.swho(*args)
 
-    def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def swho(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if not spl[1] in main.network.keys():

legacy/commands/token.py

@@ -1,12 +1,16 @@
+from uuid import uuid4
+
 import main
 from yaml import dump
-from uuid import uuid4
+
 
 class TokenCommand:
     def __init__(self, *args):
         self.token(*args)
 
-    def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def token(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 if spl[1] == "list":
@@ -31,8 +35,9 @@ class TokenCommand:
             elif length == 4:
                 if spl[1] == "add":
                     if not spl[2] in main.tokens.keys():
-                        if spl[3] in ["relay"]: # more to come!
+                        if spl[3] in ["relay", "api"]:  # more to come!
-                            main.tokens[spl[2]] = {"hello": str(uuid4()),
+                            main.tokens[spl[2]] = {
+                                "hello": str(uuid4()),
                                 "usage": spl[3],
                                 "counter": str(uuid4()),
                             }

legacy/commands/users.py

@@ -1,11 +1,13 @@
-import main
 import modules.userinfo as userinfo
 
+
 class UsersCommand:
     def __init__(self, *args):
         self.users(*args)
 
-    def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def users(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if len(spl) < 2:
                 incUsage("users")
@@ -16,7 +18,7 @@ class UsersCommand:
                 rtrn += "Matches from: %s" % i
                 rtrn += "\n"
                 for x in result[i]:
-                    rtrn += (x)
+                    rtrn += x
                     rtrn += "\n"
                 rtrn += "\n"
             info(rtrn)

legacy/commands/who.py

@@ -1,11 +1,13 @@
-import main
 import modules.userinfo as userinfo
 
+
 class WhoCommand:
     def __init__(self, *args):
         self.who(*args)
 
-    def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+    def who(
+        self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
+    ):
         if authed:
             if length == 2:
                 result = userinfo.getWho(spl[1])
@@ -14,7 +16,7 @@ class WhoCommand:
                     rtrn += "Matches from: %s" % i
                     rtrn += "\n"
                     for x in result[i]:
-                        rtrn += (x)
+                        rtrn += x
                         rtrn += "\n"
                     rtrn += "\n"
                 info(rtrn)

legacy/conf/templates/config.json

@@ -10,19 +10,34 @@
         "Address": "127.0.0.1",
         "UseSSL": true
     },
+    "API": {
+        "Enabled": true,
+        "Port": 13869,
+        "Address": "127.0.0.1"
+    },
     "Key": "key.pem",
     "Certificate": "cert.pem",
-    "RedisSocket": "/tmp/redis.sock",
+    "RedisSocket": "/var/run/socks/redis.sock",
-    "UsePassword": true,
+    "RedisDBEphemeral": 1,
+    "RedisDBPersistent": 0,
+    "UsePassword": false,
     "ConnectOnCreate": false,
+    "AutoReg": false,
     "Debug": false,
+    "Trace": false,
     "Relay": {
         "Host": "127.0.0.1",
-        "Port": "201x",
+        "Port": "2001",
-        "User": "sir",
+        "User": "x",
-        "Password": "sir"
+        "Password": "x"
+    },
+    "Ingest": {
+        "Key": "queue.irc",
+        "Enabled": true
     },
     "ChanKeep": {
+        "Enabled": false,
+        "Provision": false,
         "MaxRelay": 30,
         "SigSwitch": 20
     },
@@ -32,10 +47,10 @@
         "File": "conf/dist.sh"
     },
     "Toggles": {
-        "Who": false,
+        "Who": true,
         "CycleChans": true
     },
-    "Password": "s",
+    "Password": "x",
     "Tweaks": {
         "MaxHash": 10,
         "DedupPrecision": 9,
@@ -43,6 +58,7 @@
             "Prefix": "*"
         },
         "Delays": {
+            "WhoDelay": 3600,
             "WhoLoop": 600,
             "WhoRange": 1800,
             "Timeout": 30,

legacy/conf/templates/help.json

@@ -7,14 +7,12 @@
     "part": "part <name> <num> <channel>",
     "enable": "enable <name> <num>",
     "disable": "disable <name> <num>",
-    "list": "list",
     "stats": "stats [<name>]",
     "save": "save <(file)|list|all>",
     "load": "load <(file)|list|all>",
     "dist": "dist",
     "loadmod": "loadmod <module>",
-    "msg": "msg <name> <target> <message...>",
+    "msg": "msg <network> <num> <target> <message...>",
-    "mon": "mon -h",
     "chans": "chans <nick> [<nick> ...]",
     "users": "users <channel> [<channel> ...]",
     "relay": "relay <add|del|list> [<network>] [<num>]",
@@ -22,11 +20,19 @@
     "alias": "alias [<add|del>] [<num>]",
     "auto": "auto [<network>]",
     "cmd": "cmd <relay> <user> <entity> <text ...>",
-    "token": "token <add|del|list> [<key>] [<relay>]",
+    "token": "token <add|del|list> [<key>] [relay|api]",
     "all": "all <entity> <text ...>",
     "allc": "allc <network|alias> <(network)|(alias)> <entity> <text ...>",
     "admall": "admall <entity> <text ...>",
     "swho": "swho <network> [<channel>]",
-    "list": "list <network>",
+    "list": "list [<network>]",
-    "exec": "exec <expr ...>"
+    "exec": "exec <expr ...>",
+    "reg": "reg <network> [<num>]",
+    "confirm": "confirm <network> <num> <token>",
+    "pending": "pending [<network>]",
+    "authcheck": "authcheck [<network>]",
+    "recheckauth": "recheckauth [<network>]",
+    "blacklist": "blacklist <network> <channel>",
+    "email": "email <add|del|list> [(domain)|<num>] [<email>]",
+    "getstr": "getstr <net> <num>"
 }

legacy/conf/templates/irc.json

@@ -0,0 +1,34 @@
+{
+    "_": {
+        "register": true,
+        "entity": "NickServ",
+        "domains": [],
+        "registermsg": "REGISTER {password} {email}",
+        "identifymsg": "IDENTIFY {password}",
+        "confirm": "CONFIRM {token}",
+        "check": true,
+        "ping": true,
+        "negative": true,
+        "pingmsg": "STATUS",
+        "negativemsg": "INFO {nickname}",
+        "checktype": "mode",
+        "checkmode": "R",
+        "checkmsg": "Password accepted - you are now recognized.",
+        "checkmsg2": "You are logged in as",
+        "checknegativemsg": "has \u0002NOT COMPLETED\u0002 registration verification",
+        "checkendnegative": "End of Info"
+    },
+    "freenode": {
+        "confirm": "VERIFY REGISTER {nickname} {token}"
+    },
+    "libera": {
+        "confirm": "VERIFY REGISTER {nickname} {token}"
+    },
+    "cyberia": {
+        "setmode": "I"
+    },
+    "ircnow": {
+        "negative": false,
+        "ping": false
+    }
+}

legacy/core/bot.py

@@ -0,0 +1,946 @@
+import sys
+from copy import deepcopy
+from datetime import datetime
+from random import randint
+
+import main
+from core.relay import sendRelayNotification
+from modules import chankeep, counters, helpers, monitor, regproc, userinfo
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.internet.protocol import ReconnectingClientFactory
+from twisted.internet.task import LoopingCall
+from twisted.words.protocols.irc import (
+    IRCBadMessage,
+    IRCClient,
+    lowDequote,
+    numeric_to_symbolic,
+)
+from utils.dedup import dedup
+from utils.logging.debug import debug, trace
+from utils.logging.log import error, log, warn
+from utils.logging.send import sendAll
+from utils.parsing import parsen
+
+
+# Copied from the Twisted source so we can fix a bug
+def parsemsg(s):
+    """
+    Breaks a message from an IRC server into its prefix, command, and
+    arguments.
+    @param s: The message to break.
+    @type s: L{bytes}
+    @return: A tuple of (prefix, command, args).
+    @rtype: L{tuple}
+    """
+    prefix = ""
+    trailing = []
+    if not s:
+        raise IRCBadMessage("Empty line.")
+    if s[0:1] == ":":
+        prefix, s = s[1:].split(" ", 1)
+    if s.find(" :") != -1:
+        s, trailing = s.split(" :", 1)
+        args = s.split(" ")  # Twisted bug fixed by adding an argument to split()
+        args.append(trailing)
+    else:
+        args = s.split(" ")  # And again
+    command = args.pop(0)
+    return prefix, command, args
+
+
+class IRCBot(IRCClient):
+    def __init__(self, net, num):
+        self.isconnected = False
+        self.channels = []
+        self.net = net
+        self.authenticated = not regproc.needToAuth(self.net)
+        self.num = num
+        self.buffer = ""
+        self.name = net + str(num)
+        alias = main.alias[num]
+        relay = main.network[self.net].relays[num]
+        self.netinst = main.network[self.net]
+        self.nickname = alias["nick"]
+        self.realname = alias["realname"]
+        self.username = alias["nick"].lower() + "/" + relay["net"]
+        self.password = main.config["Relay"]["Password"]
+        self.userinfo = None  #
+        self.fingerReply = None  #
+        self.versionName = None  # Don't give out information
+        self.versionNum = None  #
+        self.versionEnv = None  #
+        self.sourceURL = None  #
+
+        self._getWho = {}  # LoopingCall objects -- needed to be able to stop them
+
+        self._tempWho = {}  # temporary storage for gathering WHO info
+        self._tempNames = {}  # temporary storage for gathering NAMES info
+        self._tempList = ([], [])  # temporary storage for gathering LIST info
+        self.listOngoing = False  # we are currently receiving a LIST
+        self.listRetried = False  # we asked and got nothing so asked again
+        self.listAttempted = False  # we asked for a list
+        self.listSimple = False  # after asking again we got the list, so use the simple
+        #  syntax from now on
+        self.wantList = (
+            False  # we want to send a LIST, but not all relays are active yet
+        )
+        self.chanlimit = 0
+        self.prefix = {}
+        self.servername = None
+
+        self._regAttempt = None
+        self._negativePass = None
+
+    def lineReceived(self, line):
+        if bytes != str and isinstance(line, bytes):
+            # decode bytes from transport to unicode
+            line = line.decode("utf-8", "replace")
+
+        line = lowDequote(line)
+        try:
+            prefix, command, params = parsemsg(line)
+            if command in numeric_to_symbolic:
+                command = numeric_to_symbolic[command]
+            try:
+                self.handleCommand(command, prefix, params)
+            except Exception as err:
+                error(err)
+        except IRCBadMessage:
+            self.badMessage(line, *sys.exc_info())
+
+    def joinChannels(self, channels):
+        sleeptime = 0.0
+        increment = 0.8
+        for i in channels:
+            if i not in self.channels:
+                if self.net in main.blacklist.keys():
+                    if i in main.blacklist[self.net]:
+                        debug(
+                            "Not joining blacklisted channel %s on %s - %i"
+                            % (i, self.net, self.num)
+                        )
+                        continue
+                trace(
+                    self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds"
+                )
+                reactor.callLater(sleeptime, self.join, i)
+                sleeptime += increment
+                if sleeptime == 10:
+                    sleeptime = 0.0
+                    increment = 0.7
+                increment += 0.1
+            else:
+                error(
+                    "%s - Cannot join channel we are already on: %s - %i"
+                    % (i, self.net, self.num)
+                )
+
+    def checkChannels(self):
+        if chankeep.allRelaysActive(self.net):
+            debug(f"checkChannels() all relays active for {self.net}")
+        else:
+            debug(
+                "checkChannels() skipping channel check as we have inactive relays: %s - %i"
+                % (self.net, self.num)
+            )
+            return
+        if self.net in main.TempChan.keys():
+            if self.num in main.TempChan[self.net].keys():
+                self.joinChannels(main.TempChan[self.net][self.num])
+                del main.TempChan[self.net][self.num]
+                if not main.TempChan[self.net]:
+                    del main.TempChan[self.net]
+
+    def event(self, **cast):
+        if "ts" not in cast.keys():
+            cast["ts"] = int(datetime.now().timestamp())
+
+        # remove odd stuff
+        for i in list(
+            cast.keys()
+        ):  # Make a copy of the .keys() as Python 3 cannot handle iterating over
+            if cast[i] == "":  # a dictionary that changes length with each iteration
+                del cast[i]
+        # remove server stuff
+        if "muser" in cast.keys():
+            if cast["muser"] == self.servername:
+                cast["type"] = "conn"
+        if "channel" in cast.keys():
+            if cast["channel"] == "*":
+                cast["type"] = "conn"
+        ##
+
+        # expand out the hostmask
+        if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
+            if "muser" in cast.keys():
+                cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
+
+        # handle ZNC stuff
+        if {"nick", "ident", "host", "msg"}.issubset(set(cast)):
+            if cast["ident"] == "znc" and cast["host"] == "znc.in":
+                cast["type"] = "znc"
+                cast["num"] = self.num
+                del cast["nick"]
+                del cast["ident"]
+                del cast["host"]
+                del cast["channel"]
+                del cast["muser"]
+                if "Disconnected from IRC" in cast["msg"]:
+                    log("ZNC disconnected on %s - %i" % (self.net, self.num))
+                    self.isconnected = False
+                    self.authenticated = False
+                if "Connected!" in cast["msg"]:
+                    log("ZNC connected on %s - %i" % (self.net, self.num))
+                    self.isconnected = True
+                if "could not be joined, disabling it" in cast["msg"]:
+                    error(cast["msg"])
+        #
+
+        # don't reprocess the same message twice
+        # if the type is in that list, it's already been here, don't run it again
+        if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
+            cast["num"] = self.num
+            if "channel" in cast.keys():
+                if cast["type"] == "mode":
+                    if cast["channel"].lower() == self.nickname.lower():
+                        # castDup = deepcopy(cast)
+                        cast["mtype"] = cast["type"]
+                        cast["type"] = "self"
+                        # self.event(**castDup)
+                    if cast["modearg"]:  # check if modearg is non-NoneType
+                        if self.nickname.lower() == cast["modearg"].lower():
+                            castDup = deepcopy(cast)
+                            castDup["mtype"] = cast["type"]
+                            castDup["type"] = "highlight"
+                            self.event(**castDup)
+                else:
+                    if cast["channel"].lower() == self.nickname.lower() or not cast[
+                        "channel"
+                    ].startswith("#"):
+                        cast["mtype"] = cast["type"]
+                        cast["type"] = "query"
+                        # self.event(**castDup)
+                        # Don't call self.event for this one because queries are not events on a
+                        #  channel, but we still want to see them
+                    if cast["channel"] == "AUTH":
+                        cast["type"] = "conn"
+                        cast["mtype"] = cast["type"]
+
+            # TODO: better way to do this
+            # as we changed the types above, check again
+            if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
+                # we have been kicked
+                if "user" in cast.keys():
+                    if cast["user"].lower() == self.nickname.lower():
+                        castDup = deepcopy(cast)
+                        castDup["mtype"] = cast["type"]
+                        castDup["type"] = "self"
+                        if self.net in main.blacklist.keys():
+                            if cast["channel"] not in main.blacklist[self.net]:
+                                main.blacklist[self.net].append(cast["channel"])
+                        else:
+                            main.blacklist[self.net] = [cast["channel"]]
+                        main.saveConf("blacklist")
+                        self.event(**castDup)
+
+                # we sent a message/left/joined/kick someone/quit
+                if "nick" in cast.keys():
+                    if cast["nick"].lower() == self.nickname.lower():
+                        castDup = deepcopy(cast)
+                        castDup["mtype"] = cast["type"]
+                        castDup["type"] = "self"
+
+                # we have been mentioned in a msg/notice/action/part/quit/topic message
+                if "msg" in cast.keys():  # Don't highlight queries
+                    if cast["msg"] is not None:
+                        if self.nickname.lower() in cast["msg"].lower():
+                            castDup = deepcopy(cast)
+                            castDup["mtype"] = cast["type"]
+                            castDup["type"] = "highlight"
+                            self.event(**castDup)
+
+        if "net" not in cast.keys():
+            cast["net"] = self.net
+        if "num" not in cast.keys():
+            cast["num"] = self.num
+        if not self.authenticated:
+            regproc.registerTest(cast)
+        counters.event(self.net, cast["type"])
+        monitor.event(self.net, cast)
+
+    def privmsg(self, user, channel, msg):
+        self.event(type="msg", muser=user, channel=channel, msg=msg)
+
+    def noticed(self, user, channel, msg):
+        self.event(type="notice", muser=user, channel=channel, msg=msg)
+
+    def action(self, user, channel, msg):
+        self.event(type="action", muser=user, channel=channel, msg=msg)
+
+    def sendmsg(self, channel, msg, in_query=False):
+        query = f"{self.nickname}!*@*"
+        res = userinfo.getWhoSingle(self.net, query)
+        if not res:
+            res = []
+        us = list(res)
+        if len(us) > 0:
+            hostmask = us[0]
+        else:
+            # Close enough...
+            hostmask = f"{self.nickname}!{self.nickname}@{self.servername}"
+            warn(f"Could not get a hostname, using {hostmask}")
+        nick, ident, host = parsen(hostmask)
+        # We sent someone a query reply
+        if in_query:
+            self.event(
+                type="self",
+                mtype="msg",
+                channel=self.nickname,
+                nick=channel,
+                ident=ident,
+                host=host,
+                msg=msg,
+            )
+        else:
+            self.event(
+                type="self",
+                mtype="msg",
+                channel=channel,
+                nick=self.nickname,
+                ident=ident,
+                host=host,
+                msg=msg,
+            )
+            self.event(
+                type="msg",
+                channel=channel,
+                nick=self.nickname,
+                ident=ident,
+                host=host,
+                msg=msg,
+            )
+        self.msg(channel, msg)
+
+    def get(self, var):
+        try:
+            result = getattr(self, var)
+        except AttributeError:
+            result = None
+        return result
+
+    def setNick(self, nickname):
+        self._attemptedNick = nickname
+        self.sendLine("NICK %s" % nickname)
+        self.nickname = nickname
+
+    def alterCollidedNick(self, nickname):
+        newnick = nickname + "_"
+        return newnick
+
+    def nickChanged(self, olduser, newnick):
+        self.nickname = newnick
+        self.event(type="self", mtype="nick", muser=olduser, user=newnick)
+
+    def irc_ERR_NICKNAMEINUSE(self, prefix, params):
+        self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
+        self.setNick(self._attemptedNick)
+
+    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
+        log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username))
+        sendAll(
+            "%s - %i: password mismatch as %s" % (self.net, self.num, self.username)
+        )
+
+    def _who(self, channel):
+        d = Deferred()
+        if channel not in self._tempWho:
+            self._tempWho[channel] = ([], [])
+        self._tempWho[channel][0].append(d)
+        self.sendLine("WHO %s" % channel)
+        return d
+
+    def who(self, channel):
+        self._who(channel).addCallback(self.got_who)
+
+    def irc_RPL_WHOREPLY(self, prefix, params):
+        channel = params[1]
+        ident = params[2]
+        host = params[3]
+        server = params[4]
+        nick = params[5]
+        status = params[6]
+        realname = params[7]
+        if channel not in self._tempWho:
+            return
+        n = self._tempWho[channel][1]
+        n.append([nick, nick, host, server, status, realname])
+        self.event(
+            type="who",
+            nick=nick,
+            ident=ident,
+            host=host,
+            realname=realname,
+            channel=channel,
+            server=server,
+            status=status,
+        )
+
+    def irc_RPL_ENDOFWHO(self, prefix, params):
+        channel = params[1]
+        if channel not in self._tempWho:
+            return
+        callbacks, info = self._tempWho[channel]
+        for cb in callbacks:
+            cb.callback((channel, info))
+        del self._tempWho[channel]
+
+    def got_who(self, whoinfo):
+        userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
+
+    def sanit(self, data):
+        if len(data) >= 1:
+            if data[0] in self.prefix.keys():
+                return (
+                    self.prefix[data[0]],
+                    data[1:],
+                )  # would use a set but it's possible these are the same
+            return (None, data)
+        else:
+            return (None, False)
+
+    def names(self, channel):
+        d = Deferred()
+        if channel not in self._tempNames:
+            self._tempNames[channel] = ([], [])
+        self._tempNames[channel][0].append(d)
+        self.sendLine("NAMES %s" % channel)
+        return d
+
+    def irc_RPL_NAMREPLY(self, prefix, params):
+        channel = params[2]
+        nicklist = params[3].split(" ")
+        if channel not in self._tempNames:
+            return
+        n = self._tempNames[channel][1]
+        n.append(nicklist)
+
+    def irc_RPL_ENDOFNAMES(self, prefix, params):
+        channel = params[1]
+        if channel not in self._tempNames:
+            return
+        callbacks, namelist = self._tempNames[channel]
+        for cb in callbacks:
+            cb.callback((channel, namelist))
+        del self._tempNames[channel]
+
+    def got_names(self, nicklist):
+        newNicklist = []
+        for i in nicklist[1]:
+            for x in i:
+                mode, nick = self.sanit(x)
+                if nick:
+                    newNicklist.append((mode, nick))
+        userinfo.initialNames(self.net, nicklist[0], newNicklist)
+
+    def myInfo(self, servername, version, umodes, cmodes):
+        self.servername = servername
+
+    def _list(self, noargs):
+        d = Deferred()
+        self._tempList = ([], [])
+        self._tempList[0].append(d)
+        if self.listSimple:
+            self.sendLine("LIST")
+            return d  # return early if we know what to do
+
+        if noargs:
+            self.sendLine("LIST")
+        else:
+            self.sendLine("LIST >0")
+        return d
+
+    def list(self, noargs=False, nocheck=False):
+        if not self.authenticated:
+            debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num))
+            return
+        if self.listAttempted:
+            debug(
+                "List request dropped, already asked for LIST - %s - %i"
+                % (self.net, self.num)
+            )
+            return
+        else:
+            self.listAttempted = True
+        if self.listOngoing:
+            debug(
+                "LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)
+            )
+            return
+        else:
+            if nocheck:
+                allRelays = True  # override the system - if this is
+            else:  # specified, we already did this
+                allRelays = chankeep.allRelaysActive(self.net)
+            if not allRelays:
+                self.wantList = True
+                debug("Not all relays were active for LIST request")
+                return
+            self._list(noargs).addCallback(self.got_list)
+
+    def irc_RPL_LISTSTART(self, prefix, params):
+        self.listAttempted = False
+        self.listOngoing = True
+        self.wantList = False
+
+    def irc_RPL_LIST(self, prefix, params):
+        channel = params[1]
+        users = params[2]
+        topic = params[3]
+        self._tempList[1].append([channel, users, topic])
+
+    def irc_RPL_LISTEND(self, prefix, params):
+        if (
+            not len(self._tempList[0]) > 0
+        ):  # there are no callbacks, can't do anything there
+            debug("We didn't ask for this LIST, discarding")
+            self._tempList[0].clear()
+            self._tempList[1].clear()
+            return
+        callbacks, info = self._tempList
+        self.listOngoing = False
+        for cb in callbacks:
+            cb.callback((info))
+        noResults = False
+        if len(self._tempList[1]) == 0:
+            noResults = True
+        self._tempList[0].clear()
+        self._tempList[1].clear()
+        if noResults:
+            if self.listRetried:
+                warn("LIST still empty after retry: %s - %i" % (self.net, self.num))
+                self.listRetried = False
+                return
+            else:
+                self.list(True)
+                self.listRetried = True
+        else:
+            if self.listRetried:
+                self.listRetried = False
+                debug(
+                    "List received after retry - defaulting to simple list syntax: %s - %i"
+                    % (self.net, self.num)
+                )
+                self.listSimple = True
+
+    def got_list(self, listinfo):
+        if len(listinfo) == 0:  # probably ngircd not supporting LIST >0
+            return
+        if main.config["ChanKeep"]["Enabled"]:
+            chankeep.initialList(self.net, self.num, listinfo)
+
+    def recheckList(self):
+        if not main.config["ChanKeep"]["Enabled"]:
+            return
+        allRelays = chankeep.allRelaysActive(self.net)
+        debug(f"recheckList() all relays for {self.net} {allRelays}")
+        if allRelays:
+            debug(f"recheckList() all relays active for {self.net}")
+            first_relay = helpers.get_first_relay(self.net)
+            debug(f"recheckList() first relay for {self.net}: {first_relay.num}")
+            if first_relay:
+                if first_relay.wantList is True:
+                    first_relay.list(nocheck=True)
+                    debug(
+                        f"recheckList() asking for a list for {self.net} after final relay {self.num} connected"
+                    )
+                else:
+                    debug(
+                        f"recheckList() first relay wantList is False for {self.net} ({first_relay.num})"
+                    )
+            # name = self.net + "1"
+
+        # if self.num == 1:  # Only one instance should do a list
+        if helpers.is_first_relay(self.net, self.num):
+            debug(f"recheckList() we are the first relay for {self.net} ({self.num})")
+            if self.chanlimit:
+                if allRelays:
+                    self.list()
+                    debug(
+                        f"recheckList() requested a list for {self.net} from {self.num}"
+                    )
+                else:
+                    debug(f"recheckList() not all relays active for {self.net}")
+                    self.wantList = True
+            else:
+                debug("recheckList() aborting LIST due to bad chanlimit")
+        self.checkChannels()
+
+    def seed_chanlimit(self, chanlimit):
+        if not main.network[self.net].relays[self.num][
+            "registered"
+        ]:  # TODO: add check for register request sent, only send it once
+            if main.config["AutoReg"]:
+                if not self.authenticated:
+                    self._regAttempt = reactor.callLater(
+                        5, regproc.registerAccount, self.net, self.num
+                    )
+                    # regproc.registerAccount(self.net, self.num)
+        try:
+            self.chanlimit = int(chanlimit)
+        except TypeError:
+            warn("Invalid chanlimit: %s" % chanlimit)
+        if self.chanlimit == 0:
+            self.chanlimit = 200  # don't take the piss if it's not limited
+        net_inst_chanlimit = self.netinst.chanlimit
+        if net_inst_chanlimit:
+            if self.chanlimit > net_inst_chanlimit:
+                self.chanlimit = net_inst_chanlimit
+                # warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}")
+
+        if not regproc.needToRegister(
+            self.net
+        ):  # if we need to register, only recheck on auth confirmation
+            if main.config["ChanKeep"]["Enabled"]:
+                self.recheckList()
+
+    def seed_prefix(self, prefix):
+        prefix = prefix.replace(")", "")
+        prefix = prefix.replace("(", "")
+        length = len(prefix)
+        half = int(length / 2)
+        prefixToMode = dict(zip(prefix[half:], prefix[:half]))
+        self.prefix = prefixToMode
+
+    def isupport(self, options):
+        interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"}
+        newOptions = {x for x in options if any(y in x for y in interested)}
+        if len(newOptions) == 0:
+            return
+        if not self.isconnected:
+            log("endpoint connected: %s - %i" % (self.net, self.num))
+            self.isconnected = True
+        for i in newOptions:
+            if i.startswith("PREFIX"):
+                if "=" in i:
+                    split = i.split("=")
+                    if len(split) == 2:
+                        prefix = split[1]
+                        self.seed_prefix(prefix)
+            elif i.startswith("CHANLIMIT"):
+                if ":" in i:
+                    split = i.split(":")
+                    if len(split) >= 2:
+                        chanlimit = split[1]
+                        self.seed_chanlimit(chanlimit)
+            elif i.startswith("MAXCHANNELS"):
+                if "=" in i:
+                    split = i.split("=")
+                    if len(split) == 2:
+                        chanlimit = split[1]
+                        self.seed_chanlimit(chanlimit)
+
+    # We need to override these functions as Twisted discards
+    #  the hostname and other useful information in the functions
+    #  that these call by default
+    def irc_JOIN(self, prefix, params):
+        nick = prefix.split("!")[0]
+        channel = params[-1]
+        if nick == self.nickname:
+            self.joined(channel)
+        else:
+            self.userJoined(prefix, channel)
+
+    def irc_PART(self, prefix, params):
+        nick = prefix.split("!")[0]
+        channel = params[0]
+        if len(params) >= 2:
+            message = params[1]
+        else:
+            message = None
+        if nick == self.nickname:
+            self.left(prefix, channel, message)
+        else:
+            self.userLeft(prefix, channel, message)
+
+    def irc_QUIT(self, prefix, params):
+        # nick = prefix.split("!")[0]
+        self.userQuit(prefix, params[0])
+
+    def irc_NICK(self, prefix, params):
+        nick = prefix.split("!", 1)[0]
+        if nick == self.nickname:
+            self.nickChanged(prefix, params[0])
+        else:
+            self.userRenamed(prefix, params[0])
+
+    def irc_KICK(self, prefix, params):
+        channel = params[0]
+        kicked = params[1]
+        message = params[-1]
+        # Checks on whether it was us that was kicked are done in userKicked
+        self.userKicked(kicked, channel, prefix, message)
+
+    def irc_TOPIC(self, prefix, params):
+        channel = params[0]
+        newtopic = params[1]
+        self.topicUpdated(prefix, channel, newtopic)
+
+    # End of Twisted hackery
+
+    def regPing(self, negativepass=None, reset=True):
+        if not main.config["AutoReg"]:
+            return
+        if self.authenticated:
+            return
+        if not regproc.needToAuth(self.net):
+            self.authenticated = True
+            return
+        sinst = regproc.substitute(self.net, self.num)
+        if not sinst:
+            error(f"regPing() {self.net}: registration ping failed for {self.num}")
+            return
+        if reset:
+            self._negativePass = None
+
+        debug(
+            f"regPing() {self.net} - {self.num}: _negativePass:{self._negativePass} negativepass:{negativepass}"
+        )
+        if self._negativePass is None:
+            # We have failed, the blacklisted message has been found
+            if negativepass is False:
+                self._negativePass = False
+                debug(
+                    (
+                        f"regPing() {self.net} - {self.num} not passed negative:checknegativemsg "
+                        f"check, {sinst['checknegativemsg']} present in message"
+                    )
+                )
+                return
+            # End of negative output reached with no blacklisted message
+            elif negativepass is True:
+                if self._negativePass is None:  # check if it's not failed yet
+                    self._negativePass = True
+                    debug(
+                        (
+                            f"regPing() {self.net} - {self.num} passed negative:checkendnegative "
+                            f"check, {sinst['checkendnegative']} present in message"
+                        )
+                    )
+                else:
+                    debug(
+                        f"regPing() {self.net}: negative registration check - {self.num}"
+                    )
+                    return
+            else:
+                if sinst["negative"]:
+                    self.msg(sinst["entity"], sinst["negativemsg"])
+                    debug(
+                        (
+                            f"regPing() {self.net}: sent negativemsg "
+                            f"'{sinst['negativemsg']}' to {sinst['entity']} - {self.num}"
+                        )
+                    )
+                    return
+                else:
+                    self._negativePass = (
+                        True  # if it's disabled, we still want the next block to run
+                    )
+
+        # Only run if negativepass has completed, or exempted as above
+        if self._negativePass:
+            if sinst["check"]:
+                if sinst["ping"]:
+                    self.msg(sinst["entity"], sinst["pingmsg"])
+                    debug(
+                        f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}"
+                    )
+                    return
+            else:
+                self.authenticated = True
+
+    def signedOn(self):
+        log("signed on: %s - %i" % (self.net, self.num))
+        ctime = str(datetime.now().isoformat())
+        sendRelayNotification(
+            {
+                "type": "conn",
+                "net": self.net,
+                "num": self.num,
+                "status": "signedon",
+                "ts": ctime,
+            }
+        )
+        if not self.authenticated:
+            reactor.callLater(10, self.regPing)
+
+    def setup_who_loop(self, channel):
+        # if main.config["Toggles"]["Who"]:
+        lc = LoopingCall(self.who, channel)
+        self._getWho[channel] = lc
+        intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
+        minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
+        interval = randint(minint, minint + intrange)
+        lc.start(interval)
+
+    def joined(self, channel):
+        if channel not in self.channels:
+            self.channels.append(channel)
+        self.names(channel).addCallback(self.got_names)
+        if main.config["Toggles"]["Who"]:
+            reactor.callLater(
+                main.config["Tweaks"]["Delays"]["WhoDelay"],
+                self.setup_who_loop,
+                channel,
+            )
+            # lc = LoopingCall(self.who, channel)
+            # self._getWho[channel] = lc
+            # intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
+            # minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
+            # interval = randint(minint, minint + intrange)
+            # lc.start(interval)
+
+    def botLeft(self, channel):
+        if channel in self.channels:
+            self.channels.remove(channel)
+        if channel in self._getWho.keys():
+            lc = self._getWho[channel]
+            lc.stop()
+            del self._getWho[channel]
+        userinfo.delChannels(
+            self.net, [channel]
+        )  # < we do not need to deduplicate this
+        # log("Can no longer cover %s, removing records" % channel)#  as it will only be matched once --
+        #  other bots have different nicknames so
+
+    def left(self, user, channel, message):  # even if they saw it, they wouldn't react
+        self.event(type="part", muser=user, channel=channel, msg=message)
+        self.botLeft(channel)
+
+    def userJoined(self, user, channel):
+        self.event(type="join", muser=user, channel=channel)
+
+    def userLeft(self, user, channel, message):
+        self.event(type="part", muser=user, channel=channel, msg=message)
+
+    def userQuit(self, user, quitMessage):
+        self.chanlessEvent({"type": "quit", "muser": user, "msg": quitMessage})
+
+    def userKicked(self, kickee, channel, kicker, message):
+        if kickee.lower() == self.nickname.lower():
+            self.botLeft(channel)
+        self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee)
+
+    def chanlessEvent(self, cast):
+        cast["ts"] = int(datetime.now().timestamp())
+        cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
+        if dedup(self.name, cast):  # Needs to be kept self.name until the dedup
+            #  function is converted to the new net, num
+            #  format
+            return  # stop right there sir!
+        chans = userinfo.getChanList(self.net, cast["nick"])
+        if chans is None:
+            error("No channels returned for chanless event: %s" % cast)
+            #    self.event(**cast) -- no, should NEVER happen
+            return
+        # getChansSingle returns all channels of the user, we only want to use
+        #  ones we have common with them
+        realChans = set(chans).intersection(set(self.channels))
+        for i in realChans:
+            cast["channel"] = i
+            self.event(**cast)
+
+    def userRenamed(self, oldname, newname):
+        self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
+
+    def topicUpdated(self, user, channel, newTopic):
+        self.event(type="topic", muser=user, channel=channel, msg=newTopic)
+
+    def modeChanged(self, user, channel, toset, modes, args):
+        argList = list(args)
+        modeList = [i for i in modes]
+        for a, m in zip(argList, modeList):
+            self.event(
+                type="mode",
+                muser=user,
+                channel=channel,
+                mode=m,
+                status=toset,
+                modearg=a,
+            )
+
+
+# TODO: strip out relay functionality
+class IRCBotFactory(ReconnectingClientFactory):
+    def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
+        if net is None:
+            self.num = num
+            self.net = None
+            self.name = "relay - %i" % num
+            self.relay = True
+        else:
+            self.name = net + str(num)
+            self.num = num
+            self.net = net
+            self.relay = False
+        self.client = None
+        self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
+        self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
+        self.factor = main.config["Tweaks"]["Delays"]["Factor"]
+        self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
+
+        self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
+
+    def buildProtocol(self, addr):
+        entry = IRCBot(self.net, self.num)
+        main.IRCPool[self.name] = entry
+
+        self.client = entry
+        return entry
+
+    def clientConnectionLost(self, connector, reason):
+        if not self.relay:
+            userinfo.delChannels(self.net, self.client.channels)
+        if self.client is not None:
+            self.client.isconnected = False
+            self.client.authenticated = False
+            self.client.channels = []
+        error = reason.getErrorMessage()
+        log("%s - %i: connection lost: %s" % (self.net, self.num, error))
+        sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
+        ctime = str(datetime.now().isoformat())
+        sendRelayNotification(
+            {
+                "type": "conn",
+                "net": self.net,
+                "num": self.num,
+                "status": "lost",
+                "message": error,
+                "ts": ctime,
+            }
+        )
+        self.retry(connector)
+        # ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
+
+    def clientConnectionFailed(self, connector, reason):
+        if self.client is not None:
+            self.client.isconnected = False
+            self.client.authenticated = False
+            self.client.channels = []
+        error = reason.getErrorMessage()
+        log("%s - %i: connection failed: %s" % (self.net, self.num, error))
+        if not self.relay:
+            sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
+            ctime = str(datetime.now().isoformat())
+            sendRelayNotification(
+                {
+                    "type": "conn",
+                    "net": self.net,
+                    "num": self.num,
+                    "status": "failed",
+                    "message": error,
+                    "ts": ctime,
+                }
+            )
+            self.retry(connector)
+        # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

legacy/core/parser.py

@@ -0,0 +1,27 @@
+import main
+from utils.logging.log import warn
+from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess
+
+
+def parseCommand(addr, authed, data):
+    # call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
+    spl = data.split()
+    if addr in main.connections.keys():
+        obj = main.connections[addr]
+    else:
+        warn("Got connection object with no instance in the address pool")
+        return
+
+    success = lambda data: sendSuccess(addr, data)  # noqa: E731
+    failure = lambda data: sendFailure(addr, data)  # noqa: E731
+    info = lambda data: sendInfo(addr, data)  # noqa: E731
+
+    incUsage = lambda mode: incorrectUsage(addr, mode)  # noqa: E731
+    length = len(spl)
+    if spl[0] in main.CommandMap.keys():
+        main.CommandMap[spl[0]](
+            addr, authed, data, obj, spl, success, failure, info, incUsage, length
+        )
+        return
+    incUsage(None)
+    return

legacy/core/relay.py

@@ -1,12 +1,28 @@
-from twisted.internet.protocol import Protocol, Factory, ClientFactory
 from json import dumps, loads
-from copy import deepcopy
-from datetime import datetime
 
 import main
-from utils.logging.log import *
+from twisted.internet.protocol import Factory, Protocol
+from utils.logging.log import log, warn
+
+validTypes = [
+    "msg",
+    "notice",
+    "action",
+    "who",
+    "part",
+    "join",
+    "kick",
+    "quit",
+    "nick",
+    "topic",
+    "mode",
+    "conn",
+    "znc",
+    "query",
+    "self",
+    "highlight",
+]
 
-validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"]
 
 class Relay(Protocol):
     def __init__(self, addr):
@@ -30,10 +46,10 @@ class Relay(Protocol):
         data = data.decode("utf-8", "replace")
         try:
             parsed = loads(data)
-        except:
+        except:  # noqa: E722
             self.sendErr("MALFORMED")
             return
-        if not "type" in parsed.keys():
+        if "type" not in parsed.keys():
             self.sendErr("NOTYPE")
             return
         if parsed["type"] == "hello":
@@ -70,7 +86,7 @@ class Relay(Protocol):
             self.sendErr("NOTLIST")
             return
         for i in lst:
-            if not i in validTypes:
+            if i not in validTypes:
                 self.sendErr("NONEXISTANT")
                 return
             if i in self.subscriptions:
@@ -85,10 +101,10 @@ class Relay(Protocol):
             self.sendErr("NOTLIST")
             return
         for i in lst:
-            if not i in validTypes:
+            if i not in validTypes:
                 self.sendErr("NONEXISTANT")
                 return
-            if not i in self.subscriptions:
+            if i not in self.subscriptions:
                 self.sendErr("NOTSUBSCRIBED")
                 return
             del self.subscriptions[i]
@@ -97,8 +113,13 @@ class Relay(Protocol):
 
     def handleHello(self, parsed):
         if parsed["key"] in main.tokens.keys():
-            if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay":
+            if (
-                self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]})
+                parsed["hello"] == main.tokens[parsed["key"]]["hello"]
+                and main.tokens[parsed["key"]]["usage"] == "relay"
+            ):
+                self.sendMsg(
+                    {"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}
+                )
                 self.authed = True
             else:
                 self.transport.loseConnection()
@@ -109,16 +130,20 @@ class Relay(Protocol):
 
     def connectionMade(self):
         log("Relay connection from %s:%s" % (self.addr.host, self.addr.port))
-        #self.send("Greetings.")
+        # self.send("Greetings.")
 
     def connectionLost(self, reason):
         self.authed = False
-        log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
+        log(
+            "Relay connection lost from %s:%s -- %s"
+            % (self.addr.host, self.addr.port, reason.getErrorMessage())
+        )
         if self.addr in main.relayConnections.keys():
             del main.relayConnections[self.addr]
         else:
             warn("Tried to remove a non-existant relay connection.")
 
+
 class RelayFactory(Factory):
     def buildProtocol(self, addr):
         entry = Relay(addr)
@@ -132,10 +157,9 @@ class RelayFactory(Factory):
         else:
             return
 
+
 def sendRelayNotification(cast):
     for i in main.relayConnections.keys():
         if main.relayConnections[i].authed:
             if cast["type"] in main.relayConnections[i].subscriptions:
-                newCast = deepcopy(cast)
+                main.relayConnections[i].send(dumps(cast))
-                newCast["time"] = str(datetime.now().isoformat())
-                main.relayConnections[i].send(dumps(newCast))

legacy/core/server.py

@@ -1,14 +1,14 @@
-from twisted.internet.protocol import Protocol, Factory, ClientFactory
 import main
-from utils.logging.log import *
-
 from core.parser import parseCommand
+from twisted.internet.protocol import Factory, Protocol
+from utils.logging.log import log, warn
+
 
 class Server(Protocol):
     def __init__(self, addr):
         self.addr = addr
         self.authed = False
-        if main.config["UsePassword"] == False:
+        if main.config["UsePassword"] is False:
             self.authed = True
 
     def send(self, data):
@@ -18,7 +18,7 @@ class Server(Protocol):
 
     def dataReceived(self, data):
         data = data.decode("utf-8", "replace")
-        #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
+        # log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
         if "\n" in data:
             splitData = [x for x in data.split("\n") if x]
             if "\n" in data:
@@ -33,13 +33,15 @@ class Server(Protocol):
 
     def connectionLost(self, reason):
         self.authed = False
-        log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
+        log(
+            "Connection lost from %s:%s -- %s"
+            % (self.addr.host, self.addr.port, reason.getErrorMessage())
+        )
         if self.addr in main.connections.keys():
             del main.connections[self.addr]
         else:
             warn("Tried to remove a non-existant connection.")
-        if self.addr in main.MonitorPool:
+
-            main.MonitorPool.remove(self.addr)
 
 class ServerFactory(Factory):
     def buildProtocol(self, addr):

legacy/docker/Dockerfile

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+
+RUN useradd -d /code pathogen
+RUN mkdir /code
+RUN chown pathogen:pathogen /code
+
+RUN mkdir /venv
+RUN chown pathogen:pathogen /venv
+
+USER pathogen
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code/legacy
+COPY requirements.prod.txt /code/legacy
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r /code/legacy/requirements.prod.txt
+CMD . /venv/bin/activate && exec python /code/legacy/threshold

legacy/docker/requirements.prod.txt

@@ -0,0 +1,8 @@
+wheel
+twisted
+pyOpenSSL
+redis
+pyYaML
+service_identity
+siphashc
+Klein

legacy/main.py

@@ -0,0 +1,134 @@
+import json
+import pickle
+from os import urandom
+from os.path import exists
+from string import digits
+
+import redis
+from redis import StrictRedis
+
+# List of errors ZNC can give us
+ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
+
+configPath = "conf/live/"
+templateConfigPath = "conf/templates/"
+certPath = "conf/cert/"
+
+filemap = {
+    # JSON configs
+    "config": ["config.json", "configuration", "json"],
+    "help": ["help.json", "command help", "json"],
+    "counters": ["counters.json", "counters file", "json"],
+    "tokens": ["tokens.json", "authentication tokens", "json"],
+    "aliasdata": ["aliasdata.json", "data for alias generation", "json"],
+    "alias": ["alias.json", "provisioned alias data", "json"],
+    "irc": ["irc.json", "IRC network definitions", "json"],
+    "blacklist": ["blacklist.json", "IRC channel blacklist", "json"],
+    # Binary (pickle) configs
+    "network": ["network.dat", "network list", "pickle"],
+}
+
+# Connections to the plain-text interface
+connections = {}
+# Connections to the JSON interface
+relayConnections = {}
+
+# Mapping of network names to Protocol (IRCClient) instances
+IRCPool = {}
+
+# Mapping of network names to Reactor instances
+# Needed for calling .disconnect()
+ReactorPool = {}
+
+# Mapping of network names to Factory instances
+# Needed for calling .stopTrying()
+FactoryPool = {}
+
+# Temporary store for channels allocated after a LIST
+# Will get purged as the instances fire up and pop() from
+#  their respective keys in here
+TempChan = {}
+
+# Mapping of command names to their functions
+CommandMap = {}
+
+# Incremented by 1 for each event reaching modules.counters.event()
+#  and cloned into lastMinuteSample every minute
+runningSample = 0
+lastMinuteSample = 0
+
+# Generate 16-byte hex key for message checksums
+hashKey = urandom(16)
+lastEvents = {}
+
+
+# Get networks that are currently online and dedupliate
+def liveNets():
+    networks = set()
+    for i in IRCPool.keys():
+        networks.add("".join([x for x in i if x not in digits]))
+    return networks
+
+
+def saveConf(var):
+    if var in ("help", "aliasdata"):
+        return  # no need to save this
+    if filemap[var][2] == "json":
+        with open(configPath + filemap[var][0], "w") as f:
+            json.dump(globals()[var], f, indent=4)
+    elif filemap[var][2] == "pickle":
+        with open(configPath + filemap[var][0], "wb") as f:
+            pickle.dump(globals()[var], f)
+    else:
+        raise Exception("invalid format")
+
+
+def loadConf(var):
+    if filemap[var][2] == "json":
+        filename = configPath + filemap[var][0]
+        # Only take the help from the templates
+        if var in ("help", "aliasdata"):
+            filename = templateConfigPath + filemap[var][0]
+        if not exists(filename):
+            # Load the template config
+            if var == "config":
+                filename = templateConfigPath + filemap[var][0]
+            else:
+                # Everything else should be blank
+                globals()[var] = {}
+                return
+        with open(filename, "r") as f:
+            globals()[var] = json.load(f)
+        if var == "alias":
+            # This is a workaround to convert all the keys into integers since JSON
+            #  turns them into strings...
+            #  Dammit Jason!
+            global alias
+            alias = {int(x): y for x, y in alias.items()}
+
+    elif filemap[var][2] == "pickle":
+        try:
+            with open(configPath + filemap[var][0], "rb") as f:
+                globals()[var] = pickle.load(f)
+        except FileNotFoundError:
+            globals()[var] = {}
+    else:
+        raise Exception("invalid format")
+
+
+def initConf():
+    for i in filemap.keys():
+        loadConf(i)
+
+
+def initMain():
+    global r, g, x
+    initConf()
+    r = StrictRedis(
+        unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]  # noqa
+    )  # Ephemeral - flushed on quit
+    g = StrictRedis(
+        unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]
+    )  # noqa
+    # SSDB for communication with Monolith
+    x = redis.from_url("redis://ssdb:1289", db=0)

legacy/modules/alias.py

@@ -1,10 +1,13 @@
-import main
 import random
 import re
 
+import main
+
+
 def generate_password():
     return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
 
+
 def generate_alias():
     nick = random.choice(main.aliasdata["stubs"])
     rand = random.randint(1, 2)
@@ -12,8 +15,8 @@ def generate_alias():
         nick = nick.capitalize()
     rand = random.randint(1, 4)
     while rand == 1:
-        split = random.randint(0, len(nick)-1)
+        split = random.randint(0, len(nick) - 1)
-        nick = nick[:split] + nick[split+1:]
+        nick = nick[:split] + nick[split + 1 :]  # noqa: E203
         rand = random.randint(1, 4)
     rand = random.randint(1, 3)
     if rand == 1 or rand == 4:
@@ -51,7 +54,7 @@ def generate_alias():
         ident = namebase.split(" ")[0]
         ident = ident[:10]
     elif rand == 6:
-        ident = re.sub("\s", "", namebase).lower()
+        ident = re.sub("\s", "", namebase).lower()  # noqa: W605
         ident = ident[:10]
 
     realname = nick
@@ -63,6 +66,10 @@ def generate_alias():
     if rand == 3 or rand == 4:
         realname = realname.capitalize()
 
-    password = generate_password()
+    return {
-
+        "nick": nick,
-    return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password}
+        "altnick": altnick,
+        "ident": ident,
+        "realname": realname,
+        "emails": [],
+    }

legacy/modules/chankeep.py

@@ -0,0 +1,600 @@
+from copy import deepcopy
+from math import ceil
+
+import main
+from modules import helpers
+from twisted.internet.threads import deferToThread
+from utils.logging.debug import debug, trace
+from utils.logging.log import error, log, warn
+
+
+def getAllChannels(net=None):
+    """
+    Get a list of all channels on all relays.
+    :return: list of channels
+    """
+    channels = {}
+    if not net:
+        nets = main.network.keys()
+    else:
+        nets = [net]
+    for net in nets:
+        relays = helpers.get_connected_relays(net)
+        for relay in relays:
+            if net not in channels:
+                channels[net] = {}
+            if relay.num not in channels[net]:
+                channels[net][relay.num] = []
+            for channel in relay.channels:
+                channels[net][relay.num].append(channel)
+    # debug(f"getAllChannels(): {channels}")
+    return channels
+
+
+def getDuplicateChannels(net=None, total=False):
+    """
+    Get a list of duplicate channels.
+    :return: list of duplicate channels
+    """
+    allChans = getAllChannels(net)
+    duplicates = {}
+    for net in allChans.keys():
+        net_chans = []
+        inst = {}
+        # add all the channels from this network to a list
+        for num in allChans[net].keys():
+            net_chans.extend(allChans[net][num])
+        for channel in net_chans:
+            count_chan = net_chans.count(channel)
+            # I don't know why but it works
+            # this is used in userinfo.delChannels
+            set_min = 1
+            if total:
+                set_min = 0
+            if count_chan > set_min:
+                inst[channel] = count_chan
+        if inst:
+            duplicates[net] = inst
+
+    if total:
+        return duplicates
+
+    to_part = {}
+    for net in allChans:
+        if net in duplicates:
+            for num in allChans[net].keys():
+                for channel in allChans[net][num]:
+                    if channel in duplicates[net].keys():
+                        if duplicates[net][channel] > 1:
+                            if net not in to_part:
+                                to_part[net] = {}
+                            if num not in to_part[net]:
+                                to_part[net][num] = []
+                            to_part[net][num].append(channel)
+                            duplicates[net][channel] -= 1
+
+    return to_part
+
+
+def partChannels(data):
+    for net in data:
+        for num in data[net]:
+            name = f"{net}{num}"
+            if name in main.IRCPool.keys():
+                for channel in data[net][num]:
+                    if channel in main.IRCPool[name].channels:
+                        main.IRCPool[name].part(channel)
+                        log(f"Parted {channel} on {net} - {num}")
+
+
+def getEnabledRelays(net):
+    """
+    Get a list of enabled relays for a network.
+    :param net: network
+    :rtype: list of int
+    :return: list of enabled relay numbers
+    """
+    enabledRelays = [
+        x
+        for x in main.network[net].relays.keys()
+        if main.network[net].relays[x]["enabled"]
+    ]
+    # debug(f"getEnabledRelays() {net}: {enabledRelays}")
+    return enabledRelays
+
+
+def getConnectedRelays(net):
+    """
+    Get a list of connected relays for a network.
+    :param net: network
+    :rtype: list of int
+    :return: list of relay numbers
+    """
+    enabledRelays = getEnabledRelays(net)
+    connectedRelays = []
+    for i in enabledRelays:
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            if main.IRCPool[name].isconnected:
+                connectedRelays.append(i)
+    # debug(f"getConnectedRelays() {net}: {connectedRelays}")
+    return connectedRelays
+
+
+def getActiveRelays(net):
+    """
+    Get a list of active relays for a network.
+    :param net: network
+    :rtype: list of int
+    :return: list of relay numbers
+    """
+    enabledRelays = getEnabledRelays(net)
+    activeRelays = []
+    for i in enabledRelays:
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            # debug(
+            #     (
+            #         f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
+            #         f"conn:{main.IRCPool[name].isconnected}"
+            #     )
+            # )
+            if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
+                activeRelays.append(i)
+    debug(f"getActiveRelays() {net}: {activeRelays}")
+    return activeRelays
+
+
+def relayIsActive(net, num):
+    """
+    Check if a relay is active.
+    :param net: network
+    :param num: relay number
+    :rtype: bool
+    :return: True if relay is active, False otherwise
+    """
+    activeRelays = getActiveRelays(net)
+    return num in activeRelays
+
+
+def allRelaysActive(net):
+    """
+    Check if all enabled relays are active and authenticated.
+    :param net: network
+    :rtype: bool
+    :return: True if all relays are active and authenticated, False otherwise
+    """
+    activeRelays = getActiveRelays(net)
+    enabledRelays = getEnabledRelays(net)
+    relaysActive = len(activeRelays) == len(enabledRelays)
+    # debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
+    return relaysActive
+
+
+def getAverageChanlimit(net):
+    """
+    Get the average channel limit for a network.
+    :param net: network
+    :rtype: int
+    :return: average channel limit
+    """
+    total = 0
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            total += main.IRCPool[name].chanlimit
+    avg_chanlimit = total / len(getActiveRelays(net))
+    debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
+    return avg_chanlimit
+
+
+def getSumChanlimit(net):
+    """
+    Get the sum of all channel limits for a network.
+    :param net: network
+    :rtype: int
+    :return: sum of channel limits
+    """
+    total = 0
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            total += main.IRCPool[name].chanlimit
+    return total
+
+
+def getChanFree(net):
+    """
+    Get a dictionary with the free channel spaces for
+    each relay, and a channel limit.
+    Example return:
+     ({1: 99}, 100)
+    :param net: network
+    :return: ({relay: channel spaces}, channel limit)
+    """
+    chanfree = {}
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name not in main.IRCPool.keys():
+            continue
+        if not main.IRCPool[name].isconnected:
+            continue
+        chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
+
+    return chanfree
+
+
+def getTotalChans(net):
+    """
+    Get the total number of channels on all relays for a network.
+    :param net: network
+    :rtype: int
+    :return: total number of channels
+    """
+    total = 0
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            total += len(main.IRCPool[name].channels)
+    return total
+
+
+def emptyChanAllocate(net, flist):
+    """
+    Allocate channels to relays.
+    :param net: network
+    :param flist: list of channels to allocate
+    :param new: list of newly provisioned relays to account for
+    :rtype: dict
+    :return: dictionary of {relay: list of channels}"""
+
+    # Get the free channel spaces for each relay
+    chanfree = getChanFree(net)
+    if not chanfree:
+        return
+
+    # Pretend the newly provisioned relays are already on the network
+    # for i in new:
+    #     chanfree[0][i] = chanfree[1]
+    allocated = {}
+
+    newlist = list(flist)
+    chan_slots_used = getTotalChans(net)
+    max_chans = getSumChanlimit(net) - chan_slots_used
+    trunc_list = newlist[:max_chans]
+    debug(
+        f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}"
+    )
+
+    for i in chanfree.keys():
+        for x in range(chanfree[i]):
+            if not len(trunc_list):
+                break
+            if i in allocated.keys():
+                allocated[i].append(trunc_list.pop())
+            else:
+                allocated[i] = [trunc_list.pop()]
+    return allocated
+
+
+def populateChans(net, clist):
+    """
+    Populate channels on relays.
+    Stores channels to join in a list in main.TempChan[net][num]
+    :param net: network
+    :param clist: list of channels to join
+    :param new: list of newly provisioned relays to account for"""
+    # divided = array_split(clist, relay)
+    allocated = emptyChanAllocate(net, clist)
+    trace(f"populateChans() allocated:{allocated}")
+    if not allocated:
+        return
+    for i in allocated.keys():
+        if net in main.TempChan.keys():
+            main.TempChan[net][i] = allocated[i]
+        else:
+            main.TempChan[net] = {i: allocated[i]}
+        trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
+
+
+def notifyJoin(net):
+    """
+    Notify relays to join channels.
+    They will pull from main.TempChan and remove channels they join.
+    :param net: network
+    """
+    for i in getActiveRelays(net):
+        name = net + str(i)
+        if name in main.IRCPool.keys():
+            trace(f"notifyJoin() {name}")
+            main.IRCPool[name].checkChannels()
+
+
+def minifyChans(net, listinfo, as_list=False):
+    """
+    Remove channels from listinfo that are already covered by a relay.
+    :param net: network
+    :param listinfo: list of channels to check
+    :type listinfo: list of [channel, num_users]
+    :return: list of channels with joined channels removed
+    :rtype: list of [channel, num_users]
+    """
+    # We want to make this reusable for joining a bunch of channels.
+    if as_list:
+        channel_list = listinfo
+
+    if not allRelaysActive(net):
+        error("All relays for %s are not active, cannot minify list" % net)
+        return False
+    for i in getConnectedRelays(net):
+        name = net + str(i)
+        for x in main.IRCPool[name].channels:
+            if as_list:
+                for y in channel_list:
+                    if y == x:
+                        channel_list.remove(y)
+            else:
+                for y in listinfo:
+                    if y[0] == x:
+                        listinfo.remove(y)
+    if not as_list:
+        if not listinfo:
+            log("We're on all the channels we want to be on, dropping LIST")
+            return False
+    if as_list:
+        return channel_list
+    else:
+        return listinfo
+
+
+def keepChannels(net, listinfo, mean, sigrelay, relay):
+    """
+    Minify channels, determine whether we can cover all the channels
+    on the network, or need to use 'significant' mode.
+    Truncate the channel list to available channel spaces.
+    Allocate these channels to relays.
+    Notify relays that they should pull from TempChan to join.
+    :param net: network
+    :param listinfo: list of [channel, num_users] lists
+    :param mean: mean of channel population
+    :param sigrelay: number of relays needed to cover significant channels
+    :param relay: number of relays needed to cover all channels
+    :param chanlimit: maximum number of channels to allocate to a relay
+    """
+    listinfo = minifyChans(net, listinfo)
+    if not listinfo:
+        return
+    if (
+        relay <= main.config["ChanKeep"]["SigSwitch"]
+    ):  # we can cover all of the channels
+        coverAll = True
+    elif (
+        relay > main.config["ChanKeep"]["SigSwitch"]
+    ):  # we cannot cover all of the channels
+        coverAll = False
+    #    if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
+    #        error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
+    #        return
+    num_instances = len(getActiveRelays(net))
+    debug(f"keepChannels() {net} instances:{num_instances}")
+    chan_slots_used = getTotalChans(net)
+    debug(f"keepChannels() slots_used:{chan_slots_used}")
+    # max_chans = (chanlimit * num_instances) - chan_slots_used
+    max_chans = getSumChanlimit(net) - chan_slots_used
+    if max_chans < 0:
+        max_chans = 0
+    debug(f"keepChannels() max_chans:{max_chans}")
+    if coverAll:
+        # needed = relay - len(getActiveRelays(net))
+        # if needed:
+        #     debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
+        #     newNums = modules.provision.provisionMultipleRelays(net, needed)
+        # else:
+        #     newNums = []
+        listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
+        if len(listinfo_sort) > max_chans:
+            max_chans = len(listinfo_sort) - 1
+
+        flist = [i[0] for i in listinfo_sort]
+
+        flist = flist[:max_chans]
+        debug(
+            f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels"
+        )
+        trace(f"keepChannels() {net}: joining:{flist}")
+        populateChans(net, flist)
+    else:
+        # needed = sigrelay - len(getActiveRelays(net))
+        # if needed:
+        #     debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
+        #     newNums = modules.provision.provisionMultipleRelays(net, needed)
+        # else:
+        #     newNums = []
+        listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
+        trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
+        if len(listinfo_sort) > max_chans:
+            max_chans = len(listinfo_sort) - 1
+        debug(f"keepChannels() {net}: new max_chans:{max_chans}")
+
+        siglist = [i[0] for i in listinfo if int(i[1]) > mean]
+        trace(f"keepChannels() {net}: new siglist:{siglist}")
+
+        siglist = siglist[:max_chans]
+        trace(f"keepChannels() {net}: truncated siglist:{siglist}")
+
+        trace(
+            f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}"
+        )
+        debug(
+            f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels"
+        )
+        trace(f"keepChannels() {net}: joining:{siglist}")
+        populateChans(net, siglist)
+    notifyJoin(net)
+
+
+def joinSingle(net, channel):
+    """
+    Join a channel on a relay.
+    Use ECA to determine which relay to join on.
+    :param net: network
+    :param channel: channel to join
+    :return: relay number that joined the channel
+    :rtype: int
+    """
+    if "," in channel:
+        channels = channel.split(",")
+        channels = minifyChans(net, channels, as_list=True)
+
+    else:
+        channels = [channel]
+    populateChans(net, channels)
+    notifyJoin(net)
+    return True
+
+
+def partSingle(net, channel):
+    """
+    Iterate over all the relays of net and part channels matching channel.
+    :param net: network
+    :param channel: channel to part
+    :return: list of relays that parted the channel
+    :rtype: list of str
+    """
+    parted = []
+    for i in getConnectedRelays(net):
+        name = f"{net}{i}"
+        if name in main.IRCPool.keys():
+            if channel in main.IRCPool[name].channels:
+                main.IRCPool[name].part(channel)
+                parted.append(str(i))
+    return parted
+
+
+def nukeNetwork(net):
+    """
+    Remove network records.
+    :param net: network"""
+    # purgeRecords(net)
+    # p = main.g.pipeline()
+    main.g.delete("analytics.list." + net)
+    # p.delete("list."+net)
+    # p.execute()
+
+
+# def nukeNetwork(net):
+#    deferToThread(_nukeNetwork, net)
+
+
+def _initialList(net, num, listinfo):
+    """
+    Called when a relay receives a full LIST response.
+    Run statistics to determine how many channels are significant.
+    This is done by adding all the numbers of users on the channels together,
+    then dividing by the number of channels.
+    * cumul - cumulative sum of all channel membership
+    * siglength - number of significant channels
+    * listlength - number of channels in the list
+    * sigrelay - number of relays needed to cover siglength
+    * relay - number of relays needed to cover all channels
+    :param net: network
+    :param num: relay number
+    :param listinfo: list of [channel, num_users] lists
+    :param chanlimit: maximum number of channels the relay can join
+    """
+    listlength = len(listinfo)
+    cumul = 0
+    try:
+        cumul += sum(int(i[1]) for i in listinfo)
+    except TypeError:
+        warn("Bad LIST data received from %s - %i" % (net, num))
+        return
+    mean = round(cumul / listlength, 2)
+    siglength = 0
+    insiglength = 0
+    sigcumul = 0
+    insigcumul = 0
+    for i in listinfo:
+        if int(i[1]) > mean:
+            siglength += 1
+            sigcumul += int(i[1])
+        elif int(i[1]) < mean:
+            insiglength += 1
+            insigcumul += int(i[1])
+
+    avg_chanlimit = getAverageChanlimit(net)
+    sigrelay = ceil(siglength / avg_chanlimit)
+    relay = ceil(listlength / avg_chanlimit)
+
+    cur_relays = len(getActiveRelays(net))
+    sig_relays_missing = sigrelay - cur_relays
+    all_relays_missing = relay - cur_relays
+
+    abase = "analytics.list.%s" % net
+    main.g.delete(abase)
+    p = main.g.pipeline()
+
+    # See docstring for meanings
+    p.hset(abase, "mean", mean)
+    p.hset(abase, "total_chans", listlength)
+    p.hset(abase, "big_chans", siglength)
+    p.hset(abase, "small_chans", insiglength)
+    p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
+    p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
+    p.hset(abase, "total_cumul_mem", cumul)
+    p.hset(abase, "big_chan_cumul_mem", sigcumul)
+    p.hset(abase, "small_chan_cumul_mem", insigcumul)
+    p.hset(abase, "relays_for_all_chans", relay)
+    p.hset(abase, "relays_for_big_chans", sigrelay)
+    p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
+    p.hset(abase, "sig_relays_missing", sig_relays_missing)
+    p.hset(abase, "all_relays_missing", all_relays_missing)
+    debug(
+        (
+            f"_initialList() net:{net} num:{num} listlength:{listlength} "
+            f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
+            f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
+        )
+    )
+
+    # Purge existing records before writing
+    # purgeRecords(net)
+    # for i in listinfo:
+    #    p.rpush(netbase+"."+i[0], i[1])
+    #    p.rpush(netbase+"."+i[0], i[2])
+    #    p.sadd(netbase, i[0])
+
+    p.execute()
+    debug("List parsing completed on %s" % net)
+    keepChannels(net, listinfo, mean, sigrelay, relay)
+
+    # return (listinfo, mean, sigrelay, relay)
+
+
+def convert(data):
+    """
+    Recursively convert a dictionary.
+    """
+    if isinstance(data, bytes):
+        return data.decode("ascii")
+    if isinstance(data, dict):
+        return dict(map(convert, data.items()))
+    if isinstance(data, tuple):
+        return map(convert, data)
+    if isinstance(data, list):
+        return list(map(convert, data))
+    return data
+
+
+def getListInfo(net):
+    abase = f"analytics.list.{net}"
+    info = main.g.hgetall(abase)
+    return convert(info)
+
+
+def initialList(net, num, listinfo):
+    """
+    Run _initialList in a thread.
+    See above docstring.
+    """
+    deferToThread(_initialList, net, num, deepcopy(listinfo))

legacy/modules/counters.py

@@ -1,12 +1,13 @@
 import main
 from twisted.internet.task import LoopingCall
 
+
 def event(name, eventType):
-    if not "local" in main.counters.keys():
+    if "local" not in main.counters.keys():
         main.counters["local"] = {}
-    if not "global" in main.counters.keys():
+    if "global" not in main.counters.keys():
         main.counters["global"] = {}
-    if not name in main.counters["local"].keys():
+    if name not in main.counters["local"].keys():
         main.counters["local"][name] = {}
     if eventType not in main.counters["local"][name].keys():
         main.counters["local"][name][eventType] = 0
@@ -18,8 +19,9 @@ def event(name, eventType):
     main.counters["global"][eventType] += 1
     main.runningSample += 1
 
+
 def getEvents(name=None):
-    if name == None:
+    if name is None:
         if "global" in main.counters.keys():
             return main.counters["global"]
         else:
@@ -30,10 +32,12 @@ def getEvents(name=None):
         else:
             return None
 
+
 def takeSample():
     main.lastMinuteSample = main.runningSample
     main.runningSample = 0
 
+
 def setupCounterLoop():
     lc = LoopingCall(takeSample)
     lc.start(60)

legacy/modules/helpers.py

@@ -0,0 +1,72 @@
+import main
+from modules import chankeep
+from utils.logging.debug import debug
+
+
+def get_first_relay(net):
+    """
+    Get the first relay in the network.
+    :param net: the network
+    :param num: number or relay
+    :return: IRCPool instance for the IRC bot
+    """
+    cur_relay = 0
+    max_relay = max(main.network[net].relays.keys())
+    debug(f"get_first_relay() {net}: max_relay:{max_relay}")
+    activeRelays = chankeep.getActiveRelays(net)
+
+    debug(f"get_first_relay() {net}: activeRelays:{activeRelays}")
+    while cur_relay != max_relay:
+        cur_relay += 1
+        if cur_relay not in activeRelays:
+            continue
+        name = net + str(cur_relay)
+        if name in main.IRCPool.keys():
+            # debug(f"get_first_relay() {net}: found relay {name}")
+            return main.IRCPool[name]
+    return None
+
+
+def is_first_relay(net, num):
+    """
+    Determine if we are the first relay for the network.
+    :param net: the network
+    :param num: number or relay
+    :return: True if we are the first relay, False otherwise
+    """
+    first_relay = get_first_relay(net)
+    if not first_relay:
+        return False
+    return first_relay.num == num
+
+
+def get_active_relays(net):
+    """
+    Get all active instances for the network.
+    :param net: the network
+    :return: list of active instances
+    :rtype: list of IRCPool instances
+    """
+    active_nums = chankeep.getActiveRelays(net)
+    active_insts = []
+    for num in active_nums:
+        name = net + str(num)
+        if name in main.IRCPool.keys():
+            active_insts.append(main.IRCPool[name])
+    return active_insts
+
+
+def get_connected_relays(net):
+    """
+    Get all connected instances for the network.
+    :param net: the network
+    :return: list of active instances
+    :rtype: list of IRCPool instances
+    """
+    active_nums = chankeep.getConnectedRelays(net)
+    active_insts = []
+    for num in active_nums:
+        name = net + str(num)
+        if name in main.IRCPool.keys():
+            active_insts.append(main.IRCPool[name])
+    return active_insts

legacy/modules/monitor.py

@@ -0,0 +1,89 @@
+import json
+
+import main
+from core.relay import sendRelayNotification
+from modules import userinfo
+from utils.dedup import dedup
+
+order = [
+    "type",
+    "net",
+    "num",
+    "channel",
+    "msg",
+    "nick",
+    "ident",
+    "host",
+    "mtype",
+    "user",
+    "mode",
+    "modearg",
+    "realname",
+    "server",
+    "status",
+    "ts",
+]
+
+
+def parsemeta(numName, c):
+    if "channel" not in c.keys():
+        c["channel"] = None
+    # metadata scraping
+    # need to check if this was received from a relay
+    #  in which case, do not do this
+    if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
+        if "muser" in c.keys():
+            userinfo.editUser(c["net"], c["muser"])
+        # if c["type"] == "mode":
+        #    userinfo.updateMode(c)
+    elif c["type"] == "nick":
+        userinfo.renameUser(
+            c["net"],
+            c["nick"],
+            c["muser"],
+            c["user"],
+            c["user"] + "!" + c["ident"] + "@" + c["host"],
+        )
+    elif c["type"] == "kick":
+        userinfo.editUser(c["net"], c["muser"])
+        userinfo.delUserByNick(c["net"], c["channel"], c["user"])
+    elif c["type"] == "quit":
+        userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"])
+    elif c["type"] == "join":
+        userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"])
+    elif c["type"] == "part":
+        userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"])
+
+    if "mtype" in c.keys():
+        if c["mtype"] == "nick":
+            userinfo.renameUser(
+                c["net"],
+                c["nick"],
+                c["muser"],
+                c["user"],
+                c["user"] + "!" + c["ident"] + "@" + c["host"],
+            )
+
+
+def queue_message(c):
+    message = json.dumps(c)
+    main.x.lpush("queue", message)
+
+
+def event(
+    numName, c
+):  # yes I'm using a short variable because otherwise it goes off the screen
+    if dedup(numName, c):
+        return
+
+    # make a copy of the object with dict() to prevent sending notifications with channel of None
+    parsemeta(numName, dict(c))
+
+    if "muser" in c.keys():
+        del c["muser"]
+    # sortedKeys = {k: c[k] for k in order if k in c}  # Sort dict keys according to order
+    # sortedKeys["src"] = "irc"
+    c["src"] = "irc"
+    if main.config["Ingest"]["Enabled"]:
+        queue_message(c)
+    sendRelayNotification(c)

legacy/modules/network.py

@@ -0,0 +1,163 @@
+from copy import deepcopy
+
+import main
+from core.bot import IRCBotFactory
+from modules import alias
+from modules.chankeep import nukeNetwork
+from modules.provision import provisionRelay
+from modules.regproc import needToRegister
+from twisted.internet import reactor
+from twisted.internet.ssl import DefaultOpenSSLContextFactory
+from utils.deliver_relay_commands import deliverRelayCommands
+from utils.get import getRelay
+from utils.logging.log import log
+
+
+def migrate():
+    existing = deepcopy(main.network)
+    log("Migrating network configuration")
+    log(f"Existing network configuration: {existing}")
+    for net, net_inst in existing.items():
+        log(f"Migrating network {net}")
+        net = net_inst.net
+        host = net_inst.host
+        port = net_inst.port
+        security = net_inst.security
+        auth = net_inst.auth
+        last = net_inst.last
+        relays = net_inst.relays
+        aliases = net_inst.aliases
+
+        new_net = Network(net, host, port, security, auth)
+        log(f"New network for {net}: {new_net}")
+        new_net.last = last
+        new_net.relays = relays
+        new_net.aliases = aliases
+        main.network[net] = new_net
+    main.saveConf("network")
+    log("Finished migrating network configuration")
+
+
+class Network:
+    def __init__(self, net, host, port, security, auth):
+        self.net = net
+        self.host = host
+        self.port = port
+        self.security = security
+        self.auth = auth
+        self.chanlimit = None
+
+        self.last = 1
+        self.relays = {}
+        self.aliases = {}
+
+    def add_relay(self, num=None):
+        # Grrrrrrrrrr
+        self.last = int(self.last)
+        if not num:
+            num = self.last
+            self.last += 1
+        elif num == self.last:
+            self.last += 1
+        registered = False
+        if not needToRegister(self.net):
+            registered = True
+            # Don't need to register if it's been disabled in definitions,
+            #  so we'll pretend we already did
+        self.relays[num] = {
+            "enabled": main.config["ConnectOnCreate"],
+            "net": self.net,
+            "id": num,
+            "registered": registered,
+        }
+        password = alias.generate_password()
+        if num not in main.alias.keys():
+            main.alias[num] = alias.generate_alias()
+            main.saveConf("alias")
+        self.aliases[num] = {"password": password}
+        # if main.config["ConnectOnCreate"]: -- Done in provision
+        #    self.start_bot(num)
+        provisionRelay(num, self.net)
+        return num, main.alias[num]["nick"]
+
+    def enable_relay(self, num):
+        """
+        Enable a relay for this network.
+        Send a command to ZNC to connect.
+        """
+        self.relays[num]["enabled"] = True
+        user = main.alias[num]["nick"]
+        commands = {"status": ["Connect"]}
+        name = f"{self.net}{num}"
+        deliverRelayCommands(num, commands, user=user + "/" + self.net)
+        main.saveConf("network")
+        if name not in main.IRCPool.keys():
+            self.start_bot(num)
+
+    def disable_relay(self, num):
+        """
+        Disable a relay for this network.
+        Send a command to ZNC to disconnect.
+        Stop trying to connect to the relay.
+        """
+        self.relays[num]["enabled"] = False
+        user = main.alias[num]["nick"]
+        # relay = main.network[spl[1]].relays[relayNum]
+        commands = {"status": ["Disconnect"]}
+        name = f"{self.net}{num}"
+        deliverRelayCommands(num, commands, user=user + "/" + self.net)
+        main.saveConf("network")
+        if name in main.ReactorPool.keys():
+            if name in main.FactoryPool.keys():
+                main.FactoryPool[name].stopTrying()
+            main.ReactorPool[name].disconnect()
+            if name in main.IRCPool.keys():
+                del main.IRCPool[name]
+            del main.ReactorPool[name]
+            del main.FactoryPool[name]
+
+    def killAliases(self, aliasList):
+        for i in aliasList:
+            name = self.net + str(i)
+            if name in main.ReactorPool.keys():
+                if name in main.FactoryPool.keys():
+                    main.FactoryPool[name].stopTrying()
+                main.ReactorPool[name].disconnect()
+                if name in main.IRCPool.keys():
+                    del main.IRCPool[name]
+                del main.ReactorPool[name]
+                del main.FactoryPool[name]
+
+    def delete_relay(self, id):
+        del self.relays[id]
+        del self.aliases[id]
+        # del main.alias[id] - Aliases are global per num, so don't delete them!
+        self.killAliases([id])
+
+    def seppuku(self):
+        # Removes all bots in preperation for deletion
+        self.killAliases(self.relays.keys())
+        nukeNetwork(self.net)
+
+    def start_bot(self, num):
+        # a single name is given to relays in the backend
+        # e.g. freenode1 for the first relay on freenode network
+        keyFN = main.certPath + main.config["Key"]
+        certFN = main.certPath + main.config["Certificate"]
+        contextFactory = DefaultOpenSSLContextFactory(
+            keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
+        )
+        bot = IRCBotFactory(self.net, num)
+        # host, port = self.relays[num]["host"], self.relays[num]["port"]
+        host, port = getRelay(num)
+        rct = reactor.connectSSL(host, port, bot, contextFactory)
+        name = self.net + str(num)
+        main.ReactorPool[name] = rct
+        main.FactoryPool[name] = bot
+
+        log("Started bot on relay %s on %s" % (num, self.host))
+
+    def start_bots(self):
+        for num in self.relays.keys():
+            if self.relays[num]["enabled"]:
+                self.start_bot(num)

legacy/modules/provision.py

@@ -0,0 +1,106 @@
+import main
+import modules.regproc
+from twisted.internet import reactor
+from utils.deliver_relay_commands import deliverRelayCommands
+from utils.logging.log import warn
+
+
+def provisionUserNetworkData(
+    num, nick, altnick, ident, realname, network, host, port, security
+):
+    commands = {}
+    stage2commands = {}
+    stage2commands["status"] = []
+    commands["controlpanel"] = []
+    user = nick.lower()
+    commands["controlpanel"].append(
+        "AddUser %s %s" % (user, main.config["Relay"]["Password"])
+    )
+    commands["controlpanel"].append("AddNetwork %s %s" % (user, network))
+    commands["controlpanel"].append("Set Nick %s %s" % (user, nick))
+    commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick))
+    commands["controlpanel"].append("Set Ident %s %s" % (user, ident))
+    commands["controlpanel"].append("Set RealName %s %s" % (user, realname))
+    if security == "ssl":
+        commands["controlpanel"].append(
+            "SetNetwork TrustAllCerts %s %s true" % (user, network)
+        )  # Don't judge me
+        commands["controlpanel"].append(
+            "AddServer %s %s %s +%s" % (user, network, host, port)
+        )
+    elif security == "plain":
+        commands["controlpanel"].append(
+            "AddServer %s %s %s %s" % (user, network, host, port)
+        )
+    if not main.config["ConnectOnCreate"]:
+        stage2commands["status"].append("Disconnect")
+    if main.config["Toggles"]["CycleChans"]:
+        stage2commands["status"].append("LoadMod disconkick")
+    stage2commands["status"].append("LoadMod chansaver")
+    inst = modules.regproc.selectInst(network)
+    if "setmode" in inst.keys():
+        stage2commands["status"].append("LoadMod perform")
+        # stage2commands["perform"].append("add mode %nick% +"+inst["setmode"])
+    deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]])
+
+
+def provisionAuthenticationData(num, nick, network, auth, password):
+    commands = {}
+    commands["status"] = []
+    user = nick.lower()
+    if auth == "sasl":
+        commands["sasl"] = []
+        commands["status"].append("UnloadMod nickserv")
+        commands["status"].append("LoadMod sasl")
+        commands["sasl"].append("Mechanism plain")
+        commands["sasl"].append("Set %s %s" % (nick, password))
+    elif auth == "ns":
+        commands["nickserv"] = []
+        commands["status"].append("UnloadMod sasl")
+        commands["status"].append("LoadMod nickserv")
+        commands["nickserv"].append("Set %s" % password)
+    inst = modules.regproc.selectInst(network)
+    if "setmode" in inst.keys():
+        # perform is loaded above
+        # commands["status"].append("LoadMod perform")
+        commands["perform"] = ["add mode %nick% +" + inst["setmode"]]
+    deliverRelayCommands(num, commands, user=user + "/" + network)
+
+
+def provisionRelay(num, network):  # provision user and network data
+    aliasObj = main.alias[num]
+    # alias = aliasObj["nick"]
+    nick = aliasObj["nick"]
+    altnick = aliasObj["altnick"]
+    ident = aliasObj["ident"]
+    realname = aliasObj["realname"]
+    provisionUserNetworkData(
+        num,
+        nick,
+        altnick,
+        ident,
+        realname,
+        network,
+        main.network[network].host,
+        main.network[network].port,
+        main.network[network].security,
+    )
+    if main.config["ConnectOnCreate"]:
+        reactor.callLater(10, main.network[network].start_bot, num)
+
+
+def provisionMultipleRelays(net, relaysNeeded):
+    if not relaysNeeded:
+        return []
+    if not main.config["ChanKeep"]["Provision"]:
+        warn(
+            f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled"
+        )
+        return []
+    numsProvisioned = []
+    for i in range(relaysNeeded):
+        num, alias = main.network[net].add_relay()
+        numsProvisioned.append(num)
+        provisionRelay(num, net)
+    main.saveConf("network")
+    return numsProvisioned

legacy/modules/regproc.py

@@ -0,0 +1,237 @@
+from copy import deepcopy
+from random import choice
+
+import main
+from modules import provision
+from utils.logging.debug import debug
+from utils.logging.log import error
+
+
+def needToRegister(net):
+    # Check if the network does not support authentication
+    networkObj = main.network[net]
+    if networkObj.auth == "none":
+        return False
+    # Check if the IRC network definition has registration disabled
+    inst = selectInst(net)
+    if "register" in inst.keys():
+        if inst["register"]:
+            return True
+        else:
+            return False
+
+
+def needToAuth(net):
+    networkObj = main.network[net]
+    if networkObj.auth == "none":
+        return False
+    return True
+
+
+def selectInst(net):
+    if net in main.irc.keys():
+        inst = deepcopy(main.irc[net])
+        for i in main.irc["_"].keys():
+            if i not in inst:
+                inst[i] = main.irc["_"][i]
+    else:
+        inst = deepcopy(main.irc["_"])
+    return inst
+
+
+def substitute(net, num, token=None):
+    inst = selectInst(net)
+    alias = main.alias[num]
+    gotemail = False
+    if "emails" in alias:
+        # First priority is explicit email lists
+        if alias["emails"]:
+            email = choice(alias["emails"])
+            gotemail = True
+    if "domains" in inst:
+        if inst["domains"]:
+            if not gotemail:
+                domain = choice(inst["domains"])
+                email = f"{alias['nickname']}@{domain}"
+                gotemail = True
+    if not gotemail:
+        inst["email"] = False
+    nickname = alias["nick"]
+    # username = nickname + "/" + net
+    password = main.network[net].aliases[num]["password"]
+    # inst["email"] = inst["email"].replace("{nickname}", nickname)
+
+    name = f"{net}{num}"
+    if name in main.IRCPool:
+        curnick = main.IRCPool[name].nickname
+    else:
+        curnick = nickname
+    for i in inst.keys():
+        if not isinstance(inst[i], str):
+            continue
+        inst[i] = inst[i].replace("{nickname}", nickname)
+        inst[i] = inst[i].replace("{curnick}", curnick)
+        inst[i] = inst[i].replace("{password}", password)
+        if gotemail:
+            inst[i] = inst[i].replace("{email}", email)
+        if token:
+            inst[i] = inst[i].replace("{token}", token)
+    return inst
+
+
+def registerAccount(net, num):
+    debug("Attempting to register: %s - %i" % (net, num))
+    sinst = substitute(net, num)
+    if not sinst:
+        error(f"Register account failed for {net} - {num}")
+        return
+    if not sinst["email"]:
+        error(f"Could not get email for {net} - {num}")
+        return
+    if not sinst["register"]:
+        error("Cannot register for %s: function disabled" % (net))
+        return False
+    name = net + str(num)
+    if not main.IRCPool[name].authenticated:
+        main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"])
+
+
+def confirmAccount(net, num, token):
+    sinst = substitute(net, num, token=token)
+    name = net + str(num)
+    if name in main.IRCPool:
+        main.IRCPool[name].msg(sinst["entity"], sinst["confirm"])
+    enableAuthentication(net, num)
+
+
+def confirmRegistration(net, num, negativepass=None):
+    obj = main.network[net]
+    name = net + str(num)
+    if name in main.IRCPool.keys():
+        if negativepass is not None:
+            main.IRCPool[name].regPing(negativepass=negativepass)
+            return
+        debug("Relay authenticated: %s - %i" % (net, num))
+        main.IRCPool[name].authenticated = True
+        main.IRCPool[name].recheckList()
+    if obj.relays[num]["registered"]:
+        return
+    if name in main.IRCPool.keys():
+        if main.IRCPool[name]._regAttempt:
+            try:
+                main.IRCPool[name]._regAttempt.cancel()
+            except:  # noqa
+                pass
+    obj.relays[num]["registered"] = True
+    main.saveConf("network")
+
+
+def attemptManualAuthentication(net, num):
+    sinst = substitute(net, num)
+    identifymsg = sinst["identifymsg"]
+    entity = sinst["entity"]
+    name = f"{net}{num}"
+    if name not in main.IRCPool:
+        return
+    main.IRCPool[name].sendmsg(entity, identifymsg, in_query=True)
+
+
+def enableAuthentication(net, num, jump=True, run_now=False):
+    obj = main.network[net]
+    nick = main.alias[num]["nick"]
+    auth = obj.auth
+    name = f"{net}{num}"
+    if name not in main.IRCPool:
+        return
+    # uname = main.alias[num]["nick"] + "/" + net
+    password = main.network[net].aliases[num]["password"]
+    provision.provisionAuthenticationData(
+        num, nick, net, auth, password
+    )  # Set up for auth
+    if jump:
+        main.IRCPool[name].msg(
+            main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump"
+        )
+    if run_now:
+        attemptManualAuthentication(net, num)
+    if selectInst(net)["check"] is False:
+        confirmRegistration(net, num)
+
+
+def get_unregistered_relays(net=None):
+    """
+    Get a dict of unregistereed relays, either globally or
+    for a network.
+    Returns:
+    {"net": [["nick1", 1], ["nick2", 2], ...]}
+    """
+    unreg = {}
+    if net:
+        nets = [net]
+    else:
+        nets = main.network.keys()
+    for i in nets:
+        for num in main.network[i].relays.keys():
+            if not main.network[i].relays[num]["registered"]:
+                nick = main.alias[num]["nick"]
+                if i in unreg:
+                    unreg[i].append([nick, num])
+                else:
+                    unreg[i] = [[nick, num]]
+    return unreg
+
+
+def registerTest(c):
+    sinst = substitute(c["net"], c["num"])
+    name = c["net"] + str(c["num"])
+    net = c["net"]
+    num = c["num"]
+    if sinst["check"] is False:
+        return
+    if "msg" in c.keys() and not c["msg"] is None:
+        if sinst["negative"]:
+            if name in main.IRCPool.keys():
+                if main.IRCPool[name]._negativePass is not True:
+                    if c["type"] == "query" and c["nick"] == sinst["entity"]:
+                        if sinst["checknegativemsg"] in c["msg"]:
+                            confirmRegistration(
+                                c["net"], c["num"], negativepass=False
+                            )  # Not passed negative check, report back
+                            return
+                        if sinst["checkendnegative"] in c["msg"]:
+                            confirmRegistration(
+                                c["net"], c["num"], negativepass=True
+                            )  # Passed the negative check, report back
+                            return
+        if sinst["ping"]:
+            if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]:
+                confirmRegistration(c["net"], c["num"])
+                debug(
+                    (
+                        f"registerTest() {net} - {num} passed ping:checkmsg2 "
+                        f"check, {sinst['checkmsg2']} present in message"
+                    )
+                )
+                return
+        if sinst["checktype"] == "msg":
+            if sinst["checkmsg"] in c["msg"]:
+                confirmRegistration(c["net"], c["num"])
+                debug(
+                    (
+                        f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, "
+                        f"{sinst['checkmsg']} present in message"
+                    )
+                )
+                return
+    elif sinst["checktype"] == "mode":
+        if c["type"] == "self":
+            if c["mtype"] == "mode":
+                if sinst["checkmode"] in c["mode"] and c["status"] is True:
+                    confirmRegistration(c["net"], c["num"])
+                    debug(
+                        (
+                            f"registerTest() {net} - {num} passed checktype:mode:checkmost check, "
+                            f"{sinst['checkmode']} present in mode"
+                        )
+                    )
+                    return

legacy/modules/userinfo.py

@@ -1,17 +1,18 @@
-from twisted.internet.threads import deferToThread
-from string import digits
-
 import main
-from utils.logging.log import *
+from modules import chankeep
-from utils.logging.debug import debug
+from twisted.internet.threads import deferToThread
+from utils.logging.debug import debug, trace
+from utils.logging.log import warn
 from utils.parsing import parsen
 
+
 def getWhoSingle(name, query):
-    result = main.r.sscan("live.who."+name, 0, query, count=999999)
+    result = main.r.sscan("live.who." + name, 0, query, count=999999)
     if result[1] == []:
         return None
     return (i.decode() for i in result[1])
 
+
 def getWho(query):
     result = {}
     for i in main.network.keys():
@@ -20,20 +21,53 @@ def getWho(query):
             result[i] = f
     return result
 
+
 def getChansSingle(name, nick):
-    nick = ["live.chan."+name+"."+i for i in nick]
+    nick = ("live.chan." + name + "." + i for i in nick)
     result = main.r.sinter(*nick)
     if len(result) == 0:
         return None
     return (i.decode() for i in result)
 
+
 def getChanList(name, nick):
-    chanspace =  "live.chan."+name+"."+nick
+    chanspace = "live.chan." + name + "." + nick
     result = main.r.smembers(chanspace)
     if len(result) == 0:
         return None
     return (i.decode() for i in result)
 
+
+def getTotalChanNum(net):
+    """
+    Get the number of channels a network has.
+    """
+    chans = main.r.keys(f"live.who.{net}.*")
+    return len(chans)
+
+
+def getUserNum(name, channels):
+    """
+    Get the number of users on a list of channels.
+    """
+    chanspace = ("live.who." + name + "." + i for i in channels)
+    results = {}
+    for channel, space in zip(channels, chanspace):
+        results[channel] = main.r.scard(space)
+    return results
+
+
+def getChanNum(name, nicks):
+    """
+    Get the number of channels a list of users are on.
+    """
+    nickspace = ("live.chan." + name + "." + i for i in nicks)
+    results = {}
+    for nick, space in zip(nicks, nickspace):
+        results[nick] = main.r.scard(space)
+    return results
+
+
 def getChans(nick):
     result = {}
     for i in main.network.keys():
@@ -42,13 +76,15 @@ def getChans(nick):
             result[i] = f
     return result
 
+
 def getUsersSingle(name, nick):
-    nick = ("live.who."+name+"."+i for i in nick)
+    nick = ("live.who." + name + "." + i for i in nick)
     result = main.r.sinter(*nick)
     if len(result) == 0:
         return None
     return (i.decode() for i in result)
 
+
 def getUsers(nick):
     result = {}
     for i in main.network.keys():
@@ -57,8 +93,10 @@ def getUsers(nick):
             result[i] = f
     return result
 
+
 def getNumWhoEntries(name):
-    return main.r.scard("live.who."+name)
+    return main.r.scard("live.who." + name)
+
 
 def getNumTotalWhoEntries():
     total = 0
@@ -66,6 +104,7 @@ def getNumTotalWhoEntries():
         total += getNumWhoEntries(i)
     return total
 
+
 def getNamespace(name, channel, nick):
     gnamespace = "live.who.%s" % name
     namespace = "live.who.%s.%s" % (name, channel)
@@ -73,33 +112,40 @@ def getNamespace(name, channel, nick):
     mapspace = "live.map.%s" % name
     return (gnamespace, namespace, chanspace, mapspace)
 
+
 def _initialUsers(name, channel, users):
     gnamespace = "live.who.%s" % name
     mapspace = "live.map.%s" % name
     p = main.r.pipeline()
     for i in users:
-        user = i[0]+"!"+i[1]+"@"+i[2]
+        user = i[0] + "!" + i[1] + "@" + i[2]
         p.hset(mapspace, i[0], user)
         p.sadd(gnamespace, user)
     p.execute()
 
+
 def initialUsers(name, channel, users):
-    debug("Initialising WHO records for %s on %s" % (channel, name))
+    trace("Initialising WHO records for %s on %s" % (channel, name))
-    d = deferToThread(_initialUsers, name, channel, users)
+    deferToThread(_initialUsers, name, channel, users)
-    #d.addCallback(testCallback)
+    # d.addCallback(testCallback)
+
 
 def _initialNames(name, channel, names):
     namespace = "live.who.%s.%s" % (name, channel)
     p = main.r.pipeline()
-    for i in names:
+    for mode, nick in names:
-        p.sadd(namespace, i)
+        p.sadd(namespace, nick)
-        p.sadd("live.chan."+name+"."+i, channel)
+        p.sadd("live.chan." + name + "." + nick, channel)
+        if mode:
+            p.hset("live.prefix." + name + "." + channel, nick, mode)
     p.execute()
 
+
 def initialNames(name, channel, names):
-    debug("Initialising NAMES records for %s on %s" % (channel, name))
+    trace("Initialising NAMES records for %s on %s" % (channel, name))
-    d = deferToThread(_initialNames, name, channel, names)
+    deferToThread(_initialNames, name, channel, names)
-    #d.addCallback(testCallback)
+    # d.addCallback(testCallback)
+
 
 def editUser(name, user):
     gnamespace = "live.who.%s" % name
@@ -110,6 +156,7 @@ def editUser(name, user):
     p.hset(mapspace, parsed[0], user)  # add nick -> user mapping
     p.execute()
 
+
 def addUser(name, channel, nick, user):
     gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
     p = main.r.pipeline()
@@ -119,6 +166,7 @@ def addUser(name, channel, nick, user):
     p.hset(mapspace, nick, user)
     p.execute()
 
+
 def delUser(name, channel, nick, user):
     gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
     p = main.r.pipeline()
@@ -126,6 +174,9 @@ def delUser(name, channel, nick, user):
     p.srem(namespace, nick)
     if channels == {channel.encode()}:  # can we only see them on this channel?
         p.delete(chanspace)  # remove channel tracking entry
+        p.hdel(
+            "live.prefix." + name + "." + channel, nick
+        )  # remove prefix tracking entry
         p.hdel(mapspace, nick)  # remove nick mapping entry
         if user:
             p.srem(gnamespace, user)  # remove global userinfo entry
@@ -135,24 +186,30 @@ def delUser(name, channel, nick, user):
         p.srem(chanspace, channel)  # keep up - remove the channel from their list
     p.execute()
 
+
 def escape(text):
     chars = ["[", "]", "^", "-", "*", "?"]
     text = text.replace("\\", "\\\\")
     for i in chars:
-        text = text.replace(i, "\\"+i)
+        text = text.replace(i, "\\" + i)
     return text
 
+
 def getUserByNick(name, nick):
     gnamespace = "live.who.%s" % name  # "nick": "nick!ident@host"
     mapspace = "live.map.%s" % name
     if main.r.hexists(mapspace, nick):
         return main.r.hget(mapspace, nick)
     else:
-        warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace))
+        warn(
-        #return Falsedd
+            "Entry doesn't exist: %s on %s - attempting auxiliary lookup"
+            % (nick, mapspace)
+        )
+        # return False
     # legacy code below - remove when map is reliable
-    usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1)
+    usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999)
     if usermatch[1] == []:
+        warn("No matches found for user query: %s on %s" % (nick, name))
         return False
     else:
         if len(usermatch[1]) == 1:
@@ -162,6 +219,7 @@ def getUserByNick(name, nick):
             warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
             return False
 
+
 def renameUser(name, oldnick, olduser, newnick, newuser):
     gnamespace = "live.who.%s" % name
     chanspace = "live.chan.%s.%s" % (name, oldnick)
@@ -172,20 +230,36 @@ def renameUser(name, oldnick, olduser, newnick, newuser):
     p.sadd(gnamespace, newuser)
     for i in main.r.smembers(chanspace):
         i = i.decode()
-        p.srem("live.who."+name+"."+i, oldnick)
+        p.srem("live.who." + name + "." + i, oldnick)
-        p.sadd("live.who."+name+"."+i, newnick)
+        p.sadd("live.who." + name + "." + i, newnick)
         p.hdel(mapspace, oldnick)
         p.hset(mapspace, newnick, newuser)
+        if main.r.exists(
+            "live.prefix." + name + "." + i
+        ):  # if there's a prefix entry for the channel
+            if main.r.hexists(
+                "live.prefix." + name + "." + i, oldnick
+            ):  # if the old nick is in it
+                mode = main.r.hget(
+                    "live.prefix." + name + "." + i, oldnick
+                )  # retrieve old modes
+                p.hset(
+                    "live.prefix." + name + "." + i, newnick, mode
+                )  # set old modes to new nickname
     if main.r.exists(chanspace):
         p.rename(chanspace, newchanspace)
     else:
         warn("Key doesn't exist: %s" % chanspace)
     p.execute()
 
+
 def delUserByNick(name, channel, nick):  # kick
     user = getUserByNick(name, nick)
+    if not user:
+        return
     delUser(name, channel, nick, user)
 
+
 def delUserByNetwork(name, nick, user):  # quit
     gnamespace = "live.who.%s" % name
     chanspace = "live.chan.%s.%s" % (name, nick)
@@ -193,11 +267,14 @@ def delUserByNetwork(name, nick, user): # quit
     p = main.r.pipeline()
     p.srem(gnamespace, user)
     for i in main.r.smembers(chanspace):
-        p.srem("live.who."+name+"."+i.decode(), nick)
+        p.srem("live.who." + name + "." + i.decode(), nick)
+        p.hdel("live.prefix." + name + "." + i.decode(), nick)
+
     p.delete(chanspace)
     p.hdel(mapspace, nick)
     p.execute()
 
+
 def _delChannels(net, channels):
     gnamespace = "live.who.%s" % net
     mapspace = "live.map.%s" % net
@@ -206,21 +283,36 @@ def _delChannels(net, channels):
         namespace = "live.who.%s.%s" % (net, channel)
         for i in main.r.smembers(namespace):
             nick = i.decode()
-            #user = getUserByNick(net, nick) -- far too many function calls
+            # user = getUserByNick(net, nick) -- far too many function calls
             user = main.r.hget(mapspace, nick)
             if not user:
                 warn("User lookup failed: %s on %s" % (nick, net))
-            if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}:
+            if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}:
                 if user:
                     p.srem(gnamespace, user)
-                p.delete("live.chan."+net+"."+nick)
+
+                p.delete("live.chan." + net + "." + nick)
                 p.hdel(mapspace, nick)  # remove map entry
             else:
-                p.srem("live.chan."+net+"."+nick, channel)
+                p.srem("live.chan." + net + "." + nick, channel)
         p.delete(namespace)
+        p.delete("live.prefix." + net + "." + channel)
     p.execute()
 
-def delChannels(net, channels):
+
-    debug("Purging channel %s for %s" % (", ".join(channels), net))
+def delChannels(net, channels):  # we have left a channel
-    d = deferToThread(_delChannels, net, channels)
+    trace("Purging channel %s for %s" % (", ".join(channels), net))
-    #d.addCallback(testCallback)
+    dupes = chankeep.getDuplicateChannels(net, total=True)
+    print("dupes: %s" % dupes)
+    if not dupes:
+        deferToThread(_delChannels, net, channels)
+    else:
+        for channel in channels:
+            if channel in dupes[net]:
+                if dupes[net][channel] != 0:
+                    channels.remove(channel)
+                    debug(
+                        f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it"
+                    )
+        deferToThread(_delChannels, net, channels)
+    # d.addCallback(testCallback)

legacy/runtest.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+#pre-commit run -a
+python -m unittest discover -s tests -p 'test_*.py'

legacy/stack.env

@@ -0,0 +1,17 @@
+THRESHOLD_LISTENER_HOST=0.0.0.0
+THRESHOLD_LISTENER_PORT=13867
+THRESHOLD_LISTENER_SSL=1
+
+THRESHOLD_RELAY_ENABLED=0
+THRESHOLD_RELAY_HOST=0.0.0.0
+THRESHOLD_RELAY_PORT=13868
+THRESHOLD_RELAY_SSL=1
+
+THRESHOLD_API_ENABLED=1
+THRESHOLD_API_HOST=0.0.0.0
+THRESHOLD_API_PORT=13869
+PORTAINER_GIT_DIR=..
+
+THRESHOLD_CONFIG_DIR=../conf/live/
+THRESHOLD_TEMPLATE_DIR=../conf/example/
+THRESHOLD_CERT_DIR=../conf/cert/

legacy/tests/test_chankeep.py

@@ -0,0 +1,119 @@
+from math import ceil
+from random import randint
+from unittest import TestCase
+from unittest.mock import MagicMock, patch
+
+from modules import chankeep
+
+
+class TestChanKeep(TestCase):
+    def setUp(self):
+        self.net = "testnet"
+        self.num = 1
+        self.chanlimit = 100
+        chankeep.main.initConf()
+        chankeep.main.r = MagicMock()
+        chankeep.main.g = MagicMock()
+        chankeep.main.g.pipeline = MagicMock()
+        chankeep.main.config["ChanKeep"]["Provision"] = False
+        chankeep.getAverageChanlimit = MagicMock()
+        chankeep.getAverageChanlimit.return_value = self.chanlimit
+
+        self.listinfo = self.generate_listinfo()
+        self.chan_name_list = [x[0] for x in self.listinfo]
+        self.chan_member_list = [x[1] for x in self.listinfo]
+
+    def generate_listinfo(self, ranges=None):
+        """
+        Create a fake listinfo.
+        Where #channel has 192 users, and #channel2 has 188 users.
+        listinfo = [["#channel", 192], ["#channel2", 188]]
+        """
+        if not ranges:
+            ranges = [[100, 5, 10], [400, 100, 200], [2, 500, 1000]]
+        listinfo = []
+        for channum, min, max in ranges:
+            for i in range(channum):
+                chan_name = f"#num-{channum}-{i}"
+                chan_users = randint(min, max)
+                listinfo.append([chan_name, chan_users])
+        return listinfo
+
+    def percent_diff(self, a, b):
+        return (abs(b - a) / a) * 100.0
+
+    def test_alt_listinfo(self):
+        # We're looking for a perc of 1000-1100
+        # And a sigrelay of 2
+        # We only want those 10 big channels
+        instances = 1
+        chanlimit = 5
+        max_chans = instances * chanlimit
+        listinfo = self.generate_listinfo(
+            ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]]
+        )
+        # listinfo_num = [x[1] for x in listinfo]
+
+        listlength = len(listinfo)
+        cumul = 0
+        try:
+            cumul += sum(int(i[1]) for i in listinfo)
+        except TypeError:
+            return
+        mean = round(cumul / listlength, 2)
+        siglength = 0
+        insiglength = 0
+        sigcumul = 0
+        insigcumul = 0
+        for i in listinfo:
+            if int(i[1]) > mean:
+                siglength += 1
+                sigcumul += int(i[1])
+            elif int(i[1]) < mean:
+                insiglength += 1
+                insigcumul += int(i[1])
+
+        sigrelay = ceil(siglength / chanlimit)
+        relay = ceil(listlength / chanlimit)
+        print(
+            (
+                f"len:{listlength} cumul:{cumul} mean:{mean} "
+                f"siglength:{siglength} insiglength:{insiglength} "
+                f"sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} "
+                f"insigcumul:{insigcumul}"
+            )
+        )
+        # We want a return between 1000 and 1100
+
+        # list_insig = [x for x in listinfo_num if x < mean]
+        list_sig = [x for x in listinfo if x[1] > mean]
+        chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans]
+        self.assertEqual(len(chosen), 5)
+
+    @patch("modules.chankeep.keepChannels")
+    def test__initialList(self, keepchannels):
+        chankeep._initialList(self.net, self.num, self.listinfo)
+        net, passed_list, mean, sigrelay, relay = keepchannels.call_args_list[0][0]
+        self.assertEqual(net, self.net)
+        self.assertEqual(passed_list, self.listinfo)
+        # self.assertEqual(chanlimit, self.chanlimit)
+        # print(net, mean, sigrelay, relay)
+
+    @patch("modules.chankeep.getChanFree")
+    def test_empty_chan_allocate(self, getchanfree):
+        getchanfree.return_value = ({1: 600}, 600)  # pretend we have 600 channels free
+        eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list)
+        self.assertEqual(len(eca), 1)
+        num = list(eca.keys())[0]
+        chans = eca[list(eca.keys())[0]]
+        self.assertEqual(num, self.num)
+        self.assertCountEqual(chans, self.chan_name_list)
+
+        getchanfree.return_value = ({1: 100}, 10)
+        eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list)
+        self.assertEqual(len(eca), 1)
+        num = list(eca.keys())[0]
+        chans = eca[list(eca.keys())[0]]
+        self.assertEqual(num, self.num)
+        self.assertEqual(len(chans), 100)
+        # self.assertCountEqual(chans, self.chan_name_list)

legacy/threshold

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+import sys
+from codecs import getwriter
+from os import getenv
+from signal import SIGINT, signal
+from sys import stderr, stdout
+
+import main
+import modules.counters
+from api.views import API
+from core.relay import RelayFactory
+from core.server import ServerFactory
+from twisted.internet import reactor
+
+# Webapp stuff
+from twisted.internet.protocol import Factory
+from twisted.internet.ssl import DefaultOpenSSLContextFactory
+from utils.cleanup import handler
+from utils.loaders.command_loader import loadCommands
+from utils.logging.log import log
+
+Factory.noisy = False
+
+main.initMain()
+
+if "--debug" in sys.argv:  # yes really
+    main.config["Debug"] = True
+if "--trace" in sys.argv:
+    main.config["Trace"] = True
+if "--migrate" in sys.argv:
+    from modules.network import migrate
+
+    migrate()
+    exit()
+loadCommands()
+
+
+# core.logstash.init_logstash()
+signal(SIGINT, handler)  # Handle Ctrl-C and run the cleanup routine
+stdout = getwriter("utf8")(stdout)  # this is a generic fix but we all know
+stderr = getwriter("utf8")(stderr)  # it's just for the retards on Rizon using
+#  unicode quit messages for no reason
+
+trues = ("true", "1", "t", True)
+
+# Main listener
+listener_address = getenv("THRESHOLD_LISTENER_HOST", main.config["Listener"]["Address"])
+listener_port = int(getenv("THRESHOLD_LISTENER_PORT", main.config["Listener"]["Port"]))
+listener_ssl = (
+    getenv("THRESHOLD_LISTENER_SSL", main.config["Listener"]["UseSSL"]) in trues
+)
+
+# RelayAPI
+relay_enabled = (
+    getenv("THRESHOLD_RELAY_ENABLED", main.config["RelayAPI"]["Enabled"]) in trues
+)
+relay_address = getenv("THRESHOLD_RELAY_HOST", main.config["RelayAPI"]["Address"])
+relay_port = int(getenv("THRESHOLD_RELAY_PORT", main.config["RelayAPI"]["Port"]))
+relay_ssl = getenv("THRESHOLD_RELAY_SSL", main.config["RelayAPI"]["UseSSL"]) in trues
+
+# Web API
+api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in trues
+api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"])
+api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"]))
+
+# Debugging
+debug_enabled = getenv("THRESHOLD_DEBUG", main.config["Debug"]) in trues
+trace_enabled = getenv("THRESHOLD_TRACE", main.config["Trace"]) in trues
+if debug_enabled:
+    main.config["Debug"] = True
+if trace_enabled:
+    main.config["Trace"] = True
+
+
+if __name__ == "__main__":
+    listener = ServerFactory()
+    if listener_ssl is True:
+        reactor.listenSSL(
+            listener_port,
+            listener,
+            DefaultOpenSSLContextFactory(
+                main.certPath + main.config["Key"],
+                main.certPath + main.config["Certificate"],
+            ),
+            interface=listener_address,
+        )
+        log("Threshold running with SSL on %s:%s" % (listener_address, listener_port))
+    else:
+        reactor.listenTCP(
+            listener_port,
+            listener,
+            interface=listener_address,
+        )
+        log("Threshold running on %s:%s" % (listener_address, listener_port))
+    if relay_enabled:
+        relay = RelayFactory()
+        if relay_ssl is True:
+            reactor.listenSSL(
+                relay_port,
+                relay,
+                DefaultOpenSSLContextFactory(
+                    main.certPath + main.config["Key"],
+                    main.certPath + main.config["Certificate"],
+                ),
+                interface=relay_address,
+            )
+            log(
+                "Threshold relay running with SSL on %s:%s"
+                % (relay_address, relay_port)
+            )
+        else:
+            reactor.listenTCP(
+                relay_port,
+                relay,
+                interface=relay_address,
+            )
+            log("Threshold relay running on %s:%s" % (relay_address, relay_port))
+    for net in main.network.keys():
+        main.network[net].start_bots()
+    modules.counters.setupCounterLoop()
+    if api_enabled:
+        api = API()
+        api.app.run(api_address, api_port)
+    else:
+        reactor.run()

legacy/utils/cleanup.py

@@ -1,15 +1,16 @@
 import main
 from twisted.internet import reactor
 from utils.logging.debug import debug
-from utils.logging.log import * 
+from utils.logging.log import log
-import sys
+
 
 def handler(sig, frame):
     log("Received SIGINT, cleaning up")
     cleanup()
 
+
 def cleanup():
     debug("Flushing Redis database")
     main.r.flushdb()
     reactor.stop()
-    #sys.exit(1)
+    # sys.exit(1)

legacy/utils/dedup.py

@@ -0,0 +1,34 @@
+from copy import deepcopy
+from datetime import datetime
+from json import dumps
+
+import main
+from siphashc import siphash
+from utils.logging.debug import debug
+
+
+def dedup(numName, b):
+    c = deepcopy(b)
+    if "ts" in c.keys():
+        del c["ts"]
+    c["approxtime"] = str(datetime.utcnow().timestamp())[
+        : main.config["Tweaks"]["DedupPrecision"]
+    ]
+    castHash = siphash(main.hashKey, dumps(c, sort_keys=True))
+    del c["approxtime"]
+    isDuplicate = any(
+        castHash in main.lastEvents[x]
+        for x in main.lastEvents.keys()
+        if not x == numName
+    )
+    if isDuplicate:
+        debug("Duplicate: %s" % (c))
+        return True
+    if numName in main.lastEvents.keys():
+        main.lastEvents[numName].insert(0, castHash)
+        main.lastEvents[numName] = main.lastEvents[numName][
+            0 : main.config["Tweaks"]["MaxHash"]
+        ]  # noqa
+    else:
+        main.lastEvents[numName] = [castHash]
+    return False

legacy/utils/deliver_relay_commands.py

@@ -0,0 +1,165 @@
+from datetime import datetime
+
+import main
+from core.relay import sendRelayNotification
+from modules import userinfo
+from twisted.internet import reactor
+from twisted.internet.protocol import ReconnectingClientFactory
+from twisted.internet.ssl import DefaultOpenSSLContextFactory
+from twisted.words.protocols.irc import IRCClient
+from utils.get import getRelay
+from utils.logging.log import error, log
+from utils.logging.send import sendAll
+from utils.parsing import parsen
+
+
+# TODO: strip out non-relay functionality
+class IRCRelay(IRCClient):
+    def __init__(self, num, relayCommands, user, stage2):
+        self.isconnected = False
+        self.buffer = ""
+        if user is None:
+            self.user = main.config["Relay"]["User"]
+        else:
+            self.user = user.lower()
+        password = main.config["Relay"]["Password"]
+        self.nickname = "relay"
+        self.realname = "relay"
+        self.username = self.user
+        self.password = self.user + ":" + password
+
+        self.relayCommands = relayCommands
+        self.num = num
+        self.stage2 = stage2
+        self.loop = None
+
+    def privmsg(self, user, channel, msg):
+        nick, ident, host = parsen(user)
+        for i in main.ZNCErrors:
+            if i in msg:
+                error("ZNC issue:", msg)
+        if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
+            nick = nick[1:]
+            if nick in self.relayCommands.keys():
+                sendAll("[%s] %s -> %s" % (self.num, nick, msg))
+
+    def irc_ERR_PASSWDMISMATCH(self, prefix, params):
+        log("%s: relay password mismatch" % self.num)
+        sendAll("%s: relay password mismatch" % self.num)
+
+    def sendStage2(self):
+        # [["user", {"sasl": ["message1", "message2"]}], []]
+        if not len(self.stage2) == 0:
+            user = self.stage2[0].pop(0)
+            commands = self.stage2[0].pop(0)
+            del self.stage2[0]
+            deliverRelayCommands(self.num, commands, user, self.stage2)
+
+    def signedOn(self):
+        if not self.isconnected:
+            self.isconnected = True
+            # log("signed on as a relay: %s" % self.num)
+            sleeptime = 0
+            increment = 0.8
+            for i in self.relayCommands.keys():
+                for x in self.relayCommands[i]:
+                    reactor.callLater(
+                        sleeptime,
+                        self.msg,
+                        main.config["Tweaks"]["ZNC"]["Prefix"] + i,
+                        x,
+                    )
+                sleeptime += increment
+                increment += 0.8
+            if self.stage2 is not None:
+                reactor.callLater(sleeptime, self.sendStage2)
+            reactor.callLater(sleeptime + 5, self.transport.loseConnection)
+            return
+
+
+class IRCRelayFactory(ReconnectingClientFactory):
+    def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
+        if net is None:
+            self.num = num
+            self.net = None
+            self.name = "relay - %i" % num
+            self.relay = True
+        else:
+            self.name = net + str(num)
+            self.num = num
+            self.net = net
+            self.relay = False
+        self.client = None
+        self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
+        self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
+        self.factor = main.config["Tweaks"]["Delays"]["Factor"]
+        self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
+
+        self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
+
+    def buildProtocol(self, addr):
+        entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
+
+        self.client = entry
+        return entry
+
+    def clientConnectionLost(self, connector, reason):
+        if not self.relay:
+            userinfo.delChannels(self.net, self.client.channels)
+        if self.client is not None:
+            self.client.isconnected = False
+            self.client.authenticated = False
+            self.client.channels = []
+        error = reason.getErrorMessage()
+        if not self.relay:
+            log("%s - %i: connection lost: %s" % (self.net, self.num, error))
+            sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
+            ctime = str(datetime.now().isoformat())
+            sendRelayNotification(
+                {
+                    "type": "conn",
+                    "net": self.net,
+                    "num": self.num,
+                    "status": "lost",
+                    "message": error,
+                    "ts": ctime,
+                }
+            )
+            self.retry(connector)
+        # ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
+
+    def clientConnectionFailed(self, connector, reason):
+        if self.client is not None:
+            self.client.isconnected = False
+            self.client.authenticated = False
+            self.client.channels = []
+        error = reason.getErrorMessage()
+        log("%s - %i: connection failed: %s" % (self.net, self.num, error))
+        if not self.relay:
+            sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
+            ctime = str(datetime.now().isoformat())
+            sendRelayNotification(
+                {
+                    "type": "conn",
+                    "net": self.net,
+                    "num": self.num,
+                    "status": "failed",
+                    "message": error,
+                    "ts": ctime,
+                }
+            )
+            self.retry(connector)
+        # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
+
+
+def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
+    keyFN = main.certPath + main.config["Key"]
+    certFN = main.certPath + main.config["Certificate"]
+    contextFactory = DefaultOpenSSLContextFactory(
+        keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
+    )
+    bot = IRCRelayFactory(
+        net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2
+    )
+    host, port = getRelay(num)
+    reactor.connectSSL(host, port, bot, contextFactory)

legacy/utils/get.py

@@ -1,10 +1,11 @@
 import main
 
+
 def getRelay(num):
     host = main.config["Relay"]["Host"].replace("x", str(num))
-    port = int(str(main.config["Relay"]["Port"]).replace("x", str(num)))
+    port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2)))
-    user = main.config["Relay"]["User"]
+    # user = main.config["Relay"]["User"]
-    password = main.config["Relay"]["Password"]
+    # password = main.config["Relay"]["Password"]
     try:
         port = int(port)
     except ValueError:

legacy/utils/loaders/command_loader.py

@@ -0,0 +1,29 @@
+from os import listdir
+
+from main import CommandMap
+from utils.logging.debug import debug
+from utils.logging.log import error
+
+
+def loadCommands(allowDup=False):
+    for filename in listdir("commands"):
+        if filename.endswith(".py") and filename != "__init__.py":
+            commandName = filename[0:-3]
+            className = commandName.capitalize() + "Command"
+            # try:
+            module = __import__("commands.%s" % commandName)
+            if commandName not in CommandMap:
+                CommandMap[commandName] = getattr(
+                    getattr(module, commandName), className
+                )
+                debug("Registered command: %s" % commandName)
+            else:
+                if allowDup:
+                    CommandMap[commandName] = getattr(
+                        getattr(module, commandName), className
+                    )
+                    debug("Registered command: %s" % commandName)
+
+                error("Duplicate command: %s" % (commandName))
+            # except Exception as err:
+            #    error("Exception while loading command %s:\n%s" % (commandName, err))

legacy/utils/loaders/single_loader.py

@@ -1,23 +1,23 @@
-from os import listdir
-from importlib import reload
 import sys
-
+from importlib import reload
-from utils.logging.debug import debug
+from os import listdir
-from utils.logging.log import *
-import commands
 
 from main import CommandMap
+from utils.logging.debug import debug
+
 
 def loadSingle(commandName):
-    if commandName+".py" in listdir("commands"):
+    if commandName + ".py" in listdir("commands"):
-        className = commandName.capitalize()+"Command"
+        className = commandName.capitalize() + "Command"
         try:
             if commandName in CommandMap.keys():
-                reload(sys.modules["commands."+commandName])
+                reload(sys.modules["commands." + commandName])
-                CommandMap[commandName] = getattr(sys.modules["commands."+commandName], className)
+                CommandMap[commandName] = getattr(
+                    sys.modules["commands." + commandName], className
+                )
                 debug("Reloaded command: %s" % commandName)
                 return "RELOAD"
-            module = __import__('commands.%s' % commandName)
+            module = __import__("commands.%s" % commandName)
             CommandMap[commandName] = getattr(getattr(module, commandName), className)
             debug("Registered command: %s" % commandName)
             return True

legacy/utils/logging/debug.py

@@ -1,7 +1,13 @@
 import main
+
+
 # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main
 #  to read the setting
 def debug(*data):
     if main.config["Debug"]:
         print("[DEBUG]", *data)
 
+
+def trace(*data):
+    if main.config["Trace"]:
+        print("[TRACE]", *data)

legacy/utils/logging/log.py

@@ -1,8 +1,10 @@
 def log(*data):
     print("[LOG]", *data)
 
+
 def warn(*data):
     print("[WARNING]", *data)
 
+
 def error(*data):
     print("[ERROR]", *data)

legacy/utils/logging/send.py

@@ -1,31 +1,38 @@
 import main
 
+
 def sendData(addr, data):
     main.connections[addr].send(data)
 
+
 def sendWithPrefix(addr, data, prefix):
     toSend = ""
     for i in data.split("\n"):
         toSend += prefix + " " + i + "\n"
     sendData(addr, toSend)
 
+
 def sendSuccess(addr, data):
     sendWithPrefix(addr, data, "[y]")
 
+
 def sendFailure(addr, data):
     sendWithPrefix(addr, data, "[n]")
 
+
 def sendInfo(addr, data):
     sendWithPrefix(addr, data, "[i]")
 
+
 def sendAll(data):
     for i in main.connections:
         if main.connections[i].authed:
             main.connections[i].send(data)
     return
 
+
 def incorrectUsage(addr, mode):
-    if mode == None:
+    if mode is None:
         sendFailure(addr, "Incorrect usage")
         return
     if mode in main.help.keys():

main.py

@@ -1,94 +0,0 @@
-import json
-import pickle
-from redis import StrictRedis
-from string import digits
-from os import urandom
-
-from utils.logging.log import *
-
-ZNCErrors = ["Error:",
-             "Unable to load",
-             "does not exist",
-             "doesn't exist"]
-
-configPath = "conf/"
-certPath = "cert/"
-
-filemap = {
-    # JSON configs
-    "config": ["config.json", "configuration", "json"],
-    "help": ["help.json", "command help", "json"],
-    "counters": ["counters.json", "counters file", "json"],
-    "monitor": ["monitor.json", "monitoring database", "json"],
-    "tokens": ["tokens.json", "authentication tokens", "json"],
-    "aliasdata": ["aliasdata.json", "data for alias generation", "json"],
-    "alias": ["alias.json", "provisioned alias data", "json"],
-
-    # Binary (pickle) configs
-    "network": ["network.dat", "network list", "pickle"]
-}
-
-connections = {}
-relayConnections = {}
-IRCPool = {}
-ReactorPool = {}
-FactoryPool = {}
-TempChan = {}
-
-MonitorPool = []
-
-CommandMap = {}
-
-runningSample = 0
-lastMinuteSample = 0
-
-# Generate 16-byte hex key for message checksums
-hashKey = urandom(16)
-lastEvents = {}
-
-def liveNets():
-    networks = set()
-    for i in IRCPool.keys():
-        networks.add("".join([x for x in i if not x in digits]))
-    return networks
-
-def saveConf(var):
-    if filemap[var][2] == "json":
-        with open(configPath+filemap[var][0], "w") as f:
-            json.dump(globals()[var], f, indent=4)
-    elif filemap[var][2] == "pickle":
-        with open(configPath+filemap[var][0], "wb") as f:
-            pickle.dump(globals()[var], f)
-    else:
-        raise Exception("invalid format")
-
-def loadConf(var):
-    if filemap[var][2] == "json":
-        with open(configPath+filemap[var][0], "r") as f:
-            globals()[var] = json.load(f)
-        if var == "alias":
-            # This is a hack to convert all the keys into integers since JSON
-            #  turns them into strings...
-            #  Dammit Jason!
-            global alias
-            alias = {int(x):y for x, y in alias.items()}
-    elif filemap[var][2] == "pickle":
-        try:
-            with open(configPath+filemap[var][0], "rb") as f:
-                globals()[var] = pickle.load(f)
-        except FileNotFoundError:
-            globals()[var] = {}
-    else:
-        raise Exception("invalid format")
-
-def initConf():
-    for i in filemap.keys():
-        loadConf(i)
-
-def initMain():
-    global r, g
-    initConf()
-    r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit
-    g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent
-
-

modules/chankeep.py

@@ -1,174 +0,0 @@
-import main
-from utils.logging.log import *
-from utils.logging.debug import *
-from copy import deepcopy
-from math import ceil
-from modules.provision import provisionMultipleRelays
-from twisted.internet.threads import deferToThread
-
-def allRelaysActive(net):
-    relayNum = len(main.network[net].relays.keys())
-    existNum = 0
-    for i in main.network[net].relays.keys():
-        name = net+str(i)
-        if name in main.IRCPool.keys():
-            if main.IRCPool[name].isconnected:
-                existNum += 1
-    if existNum == relayNum:
-        return True
-    return False
-
-def getChanFree(net, new):
-    chanfree = {}
-    chanlimits = set()
-    for i in main.network[net].relays.keys():
-        if i in new:
-            continue
-        name = net+str(i)
-        chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels)
-        chanlimits.add(main.IRCPool[name].chanlimit)
-    if not len(chanlimits) == 1:
-        error("Network %s has servers with different CHANLIMIT values" % net)
-        return False
-    return (chanfree, chanlimits.pop())
-
-def emptyChanAllocate(net, flist, relay, new):
-    chanfree = getChanFree(net, new)
-    if not chanfree:
-        return
-    for i in new:
-        chanfree[0][i] = chanfree[1]
-    allocated = {}
-    toalloc = len(flist)
-    if toalloc > sum(chanfree[0].values()):
-        error("Too many channels to allocate for %s - this is probably a bug" % net)
-        return False
-    for i in chanfree[0].keys():
-        for x in range(chanfree[0][i]):
-            if not len(flist):
-                break
-            if i in allocated.keys():
-                allocated[i].append(flist.pop())
-            else:
-                allocated[i] = [flist.pop()]
-    return allocated
-
-def populateChans(net, clist, relay, new):
-    #divided = array_split(clist, relay)
-    allocated = emptyChanAllocate(net, clist, relay, new)
-    if not allocated:
-        return
-    for i in allocated.keys():
-        if net in main.TempChan.keys():
-            main.TempChan[net][i] = allocated[i]
-        else:
-            main.TempChan[net] = {i: allocated[i]}
-
-def notifyJoin(net):
-    for i in main.network[net].relays.keys():
-        name = net+str(i)
-        if name in main.IRCPool.keys():
-            main.IRCPool[name].checkChannels()
-
-def minifyChans(net, listinfo):
-    if not allRelaysActive(net):
-        error("All relays for %s are not active, cannot minify list" % net)
-        return False
-    for i in main.network[net].relays.keys():
-        name = net+str(i)
-        for x in main.IRCPool[name].channels:
-            for y in listinfo:
-                if y[0] == x:
-                    listinfo.remove(y)
-    if not listinfo:
-        log("We're on all the channels we want to be on, dropping LIST")
-        return False
-    return listinfo
-
-def keepChannels(net, listinfo, mean, sigrelay, relay):
-    listinfo = minifyChans(net, listinfo)
-    if not listinfo:
-        return
-    if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
-        coverAll = True
-    elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
-        coverAll = False
-        if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
-            error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
-            return
-    if coverAll:
-        needed = relay-len(main.network[net].relays.keys())
-        newNums = provisionMultipleRelays(net, needed)
-        flist = [i[0] for i in listinfo]
-        populateChans(net, flist, relay, newNums)
-    else:
-        needed = sigrelay-len(main.network[net].relays.keys())
-        newNums = provisionMultipleRelays(net, needed)
-        siglist = [i[0] for i in listinfo if int(i[1]) > mean]
-        populateChans(net, siglist, sigrelay, newNums)
-    notifyJoin(net)
-
-def nukeNetwork(net):
-    #purgeRecords(net)
-    #p = main.g.pipeline()
-    main.g.delete("analytics.list."+net)
-    #p.delete("list."+net)
-    #p.execute()
-
-#def nukeNetwork(net):
-#    deferToThread(_nukeNetwork, net)
-
-
-def _initialList(net, num, listinfo, chanlimit):
-    listlength = len(listinfo)
-    cumul = 0
-    try:
-        cumul += sum(int(i[1]) for i in listinfo)
-    except TypeError:
-        warn("Bad LIST data received from %s - %i" % (net, num))
-        return
-    mean = round(cumul/listlength, 2)
-    siglength = 0
-    insiglength = 0
-    sigcumul = 0
-    insigcumul = 0
-    for i in listinfo:
-        if int(i[1]) > mean:
-            siglength += 1
-            sigcumul += int(i[1])
-        elif int(i[1]) < mean:
-            insiglength += 1
-            insigcumul += int(i[1])
-
-    sigrelay = ceil(siglength/chanlimit)
-    relay = ceil(listlength/chanlimit)
-    netbase = "list.%s" % net
-    abase = "analytics.list.%s" % net
-    p = main.g.pipeline()
-    p.hset(abase, "mean", mean)
-    p.hset(abase, "total", listlength)
-    p.hset(abase, "sigtotal", siglength)
-    p.hset(abase, "insigtotal", insiglength)
-    p.hset(abase, "sigperc", round(siglength/listlength*100, 2))
-    p.hset(abase, "insigperc", round(insiglength/listlength*100, 2))
-    p.hset(abase, "cumul", cumul)
-    p.hset(abase, "sigcumul", sigcumul)
-    p.hset(abase, "insigcumul", insigcumul)
-    p.hset(abase, "relay", relay)
-    p.hset(abase, "sigrelay", sigrelay)
-    p.hset(abase, "insigrelay", ceil(insiglength/chanlimit))
-
-    # Purge existing records before writing
-    #purgeRecords(net)
-    #for i in listinfo:
-    #    p.rpush(netbase+"."+i[0], i[1])
-    #    p.rpush(netbase+"."+i[0], i[2])
-    #    p.sadd(netbase, i[0])
-
-    p.execute()
-    debug("List parsing completed on %s" % net)
-    keepChannels(net, listinfo, mean, sigrelay, relay)
-
-def initialList(net, num, listinfo, chanlimit):
-    deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit)
-

modules/monitor.py

@@ -1,109 +0,0 @@
-from copy import deepcopy
-from json import dumps
-from datetime import datetime
-
-import main
-from core.relay import sendRelayNotification
-from modules import userinfo
-from utils.dedup import dedup
-
-order = ["type", "net", "num", "channel", "msg", "nick",
-        "ident", "host", "mtype", "user", "modes", "modeargs"
-        "realname", "server", "status"]
-
-def testNetTarget(name, target):
-    called = False
-    for i in main.monitor.keys():
-        if "sources" in main.monitor[i].keys():
-            if name in main.monitor[i]["sources"]:
-                if main.monitor[i]["sources"][name] == True:
-                    yield i
-                    called = True
-                elif target in main.monitor[i]["sources"][name]:
-                    yield i
-                    called = True
-        else:
-            yield i
-            called = True
-    if not called:
-        return False
-
-def contained_in(x, y):
-    if x is None or y is None:
-        return False
-    elif x == y:
-        return True
-    else:
-        return y in x
-
-def is_in(k, vs, A):
-    return any(contained_in(A.get(k), vp) for vp in vs)
-
-def matches(A, B):
-    return all(is_in(k, vs, A) for (k, vs) in B.items())
-
-def magicFunction(A, B):
-    isInside = False
-    if "send" in B.keys():
-        del B["send"]
-    if "sources" in B.keys():
-        del B["sources"]
-    if "inside" in B.keys():
-        if B["inside"] == True:
-            isInside = True
-        del B["inside"]
-    if isInside:
-        return matches(A, B)
-    else:
-        return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A)
-
-def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
-    if not "channel" in c.keys():
-        c["channel"] = None
-
-    if dedup(numName, c):
-        return
-    # metadata scraping
-    # need to check if this was received from a relay
-    #  in which case, do not do this
-    if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
-        userinfo.editUser(c["net"], c["muser"])
-    elif c["type"] == "nick":
-        userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"])
-    elif c["type"] == "kick":
-        userinfo.editUser(c["net"], c["muser"])
-        userinfo.delUserByNick(c["net"], c["channel"], c["user"])
-    elif c["type"] == "quit":
-        userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"])
-    elif c["type"] == "join":
-        userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"])
-    elif c["type"] == "part":
-        userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"])
-
-    if "mtype" in c.keys():
-        if c["mtype"] == "nick":
-            userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"])
-
-    if "muser" in c.keys():
-        del c["muser"]
-    sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order
-
-    # only monitors below
-    monitorGroups = testNetTarget(c["net"], c["channel"])
-    if monitorGroups == False:
-        return
-    for monitorGroup in monitorGroups:
-        matcher = magicFunction(deepcopy(c), deepcopy(main.monitor[monitorGroup]))
-        if matcher == True:
-            c["monitor"] = True
-            if "send" in main.monitor[monitorGroup].keys():
-                for i in main.monitor[monitorGroup]["send"].keys():
-                    if isinstance(main.monitor[monitorGroup]["send"][i], bool):
-                        sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "errdeliv"})
-                        continue
-                    if not i in main.pool.keys():
-                        sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noname"})
-                    if not i in main.IRCPool.keys():
-                        sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noinstance"})
-                    for x in main.monitor[monitorGroup]["send"][i]:
-                        main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["net"], c))