Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
pull from: druid
merge into: Pathogen:master
Conversation 1 Commits 263 Files Changed 128 +7321 -1922
128 changed files with 7321 additions and 1922 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
.gitignore vendored
View File

@ -1,13 +1,161 @@
*.pyc
*.pem
*.swp
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
conf/config.json
conf/wholist.json
conf/counters.json
conf/monitor.json
conf/tokens.json
conf/network.dat
conf/alias.json
conf/dist.sh
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
# lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
env-glibc/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
.bash_history
.vscode/
docker/data
*.pem
legacy/conf/live/
legacy/conf/cert/

15
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,15 @@
repos:
- repo: https://github.com/psf/black
rev: 22.6.0
hooks:
- id: black
- repo: https://github.com/PyCQA/isort
rev: 5.10.1
hooks:
- id: isort
args: ["--profile", "black"]
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
hooks:
- id: flake8
args: [--max-line-length=88]

26
commands/list.py
View File

@ -1,26 +0,0 @@
import main
class ListCommand:
def __init__(self, *args):
self.list(*args)
def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
if authed:
if length == 2:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
if not 1 in main.network[spl[1]].relays.keys():
failure("Network has no first instance")
return
if not spl[1]+"1" in main.IRCPool.keys():
failure("No IRC instance: %s - 1" % spl[1])
return
main.IRCPool[spl[1]+"1"].list()
success("Requested list with first instance of %s" % spl[1])
return
else:
incUsage("list")
return
else:
incUsage(None)

280
commands/mon.py
View File

@ -1,280 +0,0 @@
import main
import argparse
import sys
from io import StringIO
from yaml import dump
class MonCommand:
def __init__(self, *args):
self.mon(*args)
def setup_arguments(self, ArgumentParser):
self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.")
group1 = self.parser.add_mutually_exclusive_group(required=True)
group1.add_argument("-a", "--add", metavar="entry", dest="addEntry", help="Add an entry")
group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry")
group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries")
group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry")
group2 = self.parser.add_mutually_exclusive_group()
group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing")
group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing")
self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values")
self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who")
self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching")
self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching")
self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching")
self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching")
self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who")
self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)")
self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action")
self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick")
self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode")
self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode")
self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who")
self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who")
self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)")
def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
# We need to override the ArgumentParser class because
# it's only meant for CLI applications and exits
# after running, so we override the exit function
# to do nothing. We also need to do it here in order
# to catch the error messages and show them to the user.
# It sucks. I know.
if authed:
class ArgumentParser(argparse.ArgumentParser):
def exit(self, status=0, message=None):
if message:
failure(message)
self.setup_arguments(ArgumentParser)
if length == 1:
info("Incorrect usage, use mon -h for help")
return
old_stdout = sys.stdout
old_stderr = sys.stderr
my_stdout = sys.stdout = StringIO()
my_stderr = sys.stderr = StringIO()
try:
parsed = self.parser.parse_args(spl[1:])
except:
return
sys.stdout = old_stdout
sys.stderr = old_stderr
stdout = my_stdout.getvalue()
stderr = my_stdout.getvalue()
if not stdout == "":
info(stdout)
elif not stderr == "":
failure(my_stdout.getvalue())
return
my_stdout.close()
my_stderr.close()
if parsed.addEntry:
if parsed.addEntry in main.monitor.keys():
failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry)
return
cast = self.makeCast(parsed, failure, info)
if cast == False:
return
main.monitor[parsed.addEntry] = cast
main.saveConf("monitor")
success("Successfully created monitor group %s" % parsed.addEntry)
return
elif parsed.delEntry:
if not parsed.delEntry in main.monitor.keys():
failure("No such monitor group: %s" % parsed.delEntry)
return
del main.monitor[parsed.delEntry]
main.saveConf("monitor")
success("Successfully removed monitor group: %s" % parsed.delEntry)
return
elif parsed.modEntry:
if not parsed.doAppend and not parsed.doRemove:
failure("Specify --append or --remove with --mod")
return
if not parsed.modEntry in main.monitor.keys():
failure("No such monitor group: %s" % parsed.modEntry)
return
cast = self.makeCast(parsed, failure, info)
if cast == False:
return
if parsed.doAppend:
merged = self.addCast(main.monitor[parsed.modEntry], cast, info)
main.monitor[parsed.modEntry] = merged
elif parsed.doRemove:
merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info)
if merged == {}:
del main.monitor[parsed.modEntry]
info("Group %s deleted due to having no attributes" % parsed.modEntry)
main.saveConf("monitor")
return
main.monitor[parsed.modEntry] = merged
main.saveConf("monitor")
success("Successfully updated entry %s" % parsed.modEntry)
return
elif parsed.listEntry:
info(dump(main.monitor))
return
else:
incUsage(None)
def dedup(self, data):
if not isinstance(data, bool):
return list(set(data))
return data
def parseNetworkFormat(self, lst, failure, info):
cast = {}
if lst == None:
return "nil"
if len(lst) == 0:
failure("List has no entries")
return False
elif len(lst) > 0:
if len(lst[0]) == 0:
failure("Nested list has no entries")
return False
for i in lst:
if not i[0] in main.pool.keys():
failure("Name does not exist: %s" % i[0])
return False
if i[0] in main.IRCPool.keys():
for x in i[1:]:
if not x in main.IRCPool[i[0]].channels:
info("%s: Bot not on channel: %s" % (i[0], x))
if len(i) == 1:
cast[i[0]] = True
else:
if i[0] in cast.keys():
if not cast[i[0]] == True:
for x in self.dedup(i[1:]):
cast[i[0]].append(x)
else:
cast[i[0]] = self.dedup(i[1:])
else:
cast[i[0]] = self.dedup(i[1:])
for i in cast.keys():
deduped = self.dedup(cast[i])
cast[i] = deduped
return cast
# Create or modify a monitor group magically
def makeCast(self, obj, failure, info):
validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"]
cast = {}
if not obj.specType == None:
types = self.dedup(obj.specType)
for i in types:
if not i in validTypes:
failure("Invalid type: %s" % i)
info("Available types: %s" % ", ".join(validTypes))
return False
cast["type"] = types
if not obj.source == None:
sourceParse = self.parseNetworkFormat(obj.source, failure, info)
if not sourceParse:
return False
if not sourceParse == "nil":
cast["sources"] = sourceParse
if not obj.send == None:
sendParse = self.parseNetworkFormat(obj.send, failure, info)
if not sendParse:
return False
if not sendParse == "nil":
cast["send"] = sendParse
if not obj.message == None:
cast["message"] = []
for i in obj.message:
cast["message"].append(" ".join(i))
if not obj.user == None:
cast["user"] = obj.user
if not obj.modes == None:
cast["modes"] = obj.modes
if not obj.modeargs == None:
cast["modeargs"] = obj.modeargs
if not obj.server == None:
cast["server"] = obj.server
if not obj.status == None:
cast["status"] = obj.status
if not obj.free == None:
cast["free"] = obj.free
if not obj.nick == None:
cast["nick"] = obj.nick
if not obj.ident == None:
cast["ident"] = obj.ident
if not obj.host == None:
cast["host"] = obj.host
if not obj.real == None:
cast["real"] = []
for i in obj.real:
cast["real"].append(" ".join(i))
if not obj.inside == None:
if obj.inside.lower() in ["yes", "true", "on"]:
cast["inside"] = True
elif obj.inside.lower() in ["no", "false", "off"]:
cast["inside"] = False
else:
failure("inside: Unknown operand: %s" % obj.inside)
return
return cast
def subtractCast(self, source, patch, info):
for i in patch.keys():
if i in source.keys():
if isinstance(source[i], dict):
result = self.subtractCast(source[i], patch[i], info)
if result == {}:
info("Removing upper element: %s" % i)
del source[i]
continue
source[i] = result
continue
if isinstance(patch[i], bool):
del source[i]
info("Removing entire element: %s" % i)
continue
for x in patch[i]:
if isinstance(source[i], bool):
info("Attempt to remove %s from network-wide definition" % x)
continue
if x in source[i]:
source[i].remove(x)
if source[i] == []:
del source[i]
else:
info("Element %s not in source %s" % (x, i))
else:
info("Non-matched key: %s" % i)
return source
def addCast(self, source, patch, info):
for i in patch.keys():
if i in source.keys():
if isinstance(source[i], dict):
result = self.addCast(source[i], patch[i], info)
source[i] = result
continue
if isinstance(patch[i], bool):
source[i] = patch[i]
info("Overriding local element %s with network-wide definition" % i)
continue
for x in patch[i]:
if isinstance(source[i], bool):
source[i] = []
info("Overriding element %s, previously set network-wide" % i)
if x in source[i]:
info("Element %s already in source %s" % (x, i))
else:
source[i].append(x)
else:
source[i] = patch[i]
return source

1
conf/example/alias.json
View File

@ -1 +0,0 @@
{}

1
conf/example/counters.json
View File

@ -1 +0,0 @@
{}

1
conf/example/masterbuf.json
View File

@ -1 +0,0 @@
[]

1
conf/example/monitor.json
View File

@ -1 +0,0 @@
{}

1
conf/example/tokens.json
View File

@ -1 +0,0 @@
{}

706
core/bot.py
View File

@ -1,706 +0,0 @@
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.internet import reactor, task
from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage
import sys
from string import digits
from random import randint
from copy import deepcopy
from modules import userinfo
from modules import counters
from modules import monitor
from modules import chankeep
from core.relay import sendRelayNotification
from utils.dedup import dedup
from utils.get import getRelay
import main
from utils.logging.log import *
from utils.logging.debug import *
from utils.logging.send import *
from utils.parsing import parsen
from twisted.internet.ssl import DefaultOpenSSLContextFactory
def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
keyFN = main.certPath+main.config["Key"]
certFN = main.certPath+main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"),
certFN.encode("utf-8", "replace"))
bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
host, port = getRelay(num)
rct = reactor.connectSSL(host,
port,
bot, contextFactory)
def parsemsg(s):
"""
Breaks a message from an IRC server into its prefix, command, and
arguments.
@param s: The message to break.
@type s: L{bytes}
@return: A tuple of (prefix, command, args).
@rtype: L{tuple}
"""
prefix = ''
trailing = []
if not s:
raise IRCBadMessage("Empty line.")
if s[0:1] == ':':
prefix, s = s[1:].split(' ', 1)
if s.find(' :') != -1:
s, trailing = s.split(' :', 1)
args = s.split(' ')
args.append(trailing)
else:
args = s.split(' ')
command = args.pop(0)
return prefix, command, args
class IRCRelay(IRCClient):
def __init__(self, num, relayCommands, user, stage2):
self.isconnected = False
self.buffer = ""
if user == None:
self.user = main.config["Relay"]["User"]
else:
self.user = user
password = main.config["Relay"]["Password"]
self.nickname = "relay"
self.realname = "relay"
self.username = self.user
self.password = self.user+":"+password
self.relayCommands = relayCommands
self.num = num
self.stage2 = stage2
self.loop = None
def privmsg(self, user, channel, msg):
nick, ident, host = parsen(user)
for i in main.ZNCErrors:
if i in msg:
error("ZNC issue:", msg)
if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
nick = nick[1:]
if nick in self.relayCommands.keys():
sendAll("[%s] %s -> %s" % (self.num, nick, msg))
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
log("%s: relay password mismatch" % self.num)
sendAll("%s: relay password mismatch" % self.num)
def sendStage2(self):
if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []]
if not len(self.stage2) == 0:
user = self.stage2[0].pop(0)
commands = self.stage2[0].pop(0)
del self.stage2[0]
deliverRelayCommands(self.num, commands, user, self.stage2)
def signedOn(self):
if not self.isconnected:
self.isconnected = True
log("signed on as a relay: %s" % self.num)
#sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares
sleeptime = 0
increment = 0.8
for i in self.relayCommands.keys():
for x in self.relayCommands[i]:
reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x)
sleeptime += increment
increment += 0.8
reactor.callLater(sleeptime, self.sendStage2)
reactor.callLater(sleeptime+5, self.transport.loseConnection)
return
class IRCBot(IRCClient):
def __init__(self, net, num):
self.isconnected = False
self.channels = []
self.net = net
self.num = num
self.buffer = ""
self.name = net + str(num)
alias = main.alias[num]
relay = main.network[self.net].relays[num]
self.nickname = alias["nick"]
self.realname = alias["realname"]
self.username = alias["nick"]+"/"+relay["net"]
self.password = main.config["Relay"]["Password"]
self.userinfo = None #
self.fingerReply = None #
self.versionName = None # don't tell anyone information about us
self.versionNum = None #
self.versionEnv = None #
self.sourceURL = None #
self._getWho = {} # LoopingCall objects -- needed to be able to stop them
self._tempWho = {} # temporary storage for gathering WHO info
self._tempNames = {} # temporary storage for gathering NAMES info
self._tempList = ([], []) # temporary storage for gathering LIST info
self.listOngoing = False # we are currently receiving a LIST
self.listRetried = False # we asked and got nothing so asked again
self.listAttempted = False # we asked for a list
self.listSimple = False # after asking again we got the list, so use the simple
# syntax from now on
self.wantList = False # we want to send a LIST, but not all relays are active yet
self.chanlimit = 0
self.servername = None
def lineReceived(self, line):
if bytes != str and isinstance(line, bytes):
# decode bytes from transport to unicode
line = line.decode("utf-8", "replace")
line = lowDequote(line)
try:
prefix, command, params = parsemsg(line)
if command in numeric_to_symbolic:
command = numeric_to_symbolic[command]
self.handleCommand(command, prefix, params)
except IRCBadMessage:
self.badMessage(line, *sys.exc_info())
def joinChannels(self, channels):
sleeptime = 0.0
increment = 0.8
for i in channels:
if not i in self.channels:
debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds")
reactor.callLater(sleeptime, self.join, i)
sleeptime += increment
if sleeptime == 10:
sleeptime = 0.0
increment = 0.7
increment += 0.1
else:
error("%s - %i - Cannot join channel we are already on - %s" % (self.net, self.num, i))
def checkChannels(self):
if self.net in main.TempChan.keys():
if self.num in main.TempChan[self.net].keys():
self.joinChannels(main.TempChan[self.net][self.num])
del main.TempChan[self.net][self.num]
if not main.TempChan[self.net]:
del main.TempChan[self.net]
def event(self, **cast):
for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over
if cast[i] == "": # a dictionary that changes length with each iteration
del cast[i]
if "muser" in cast.keys():
if cast["muser"] == self.servername:
return
if "channel" in cast.keys():
if cast["channel"] == "*":
return
if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
if set(["nick", "ident", "host", "msg"]).issubset(set(cast)):
if "msg" in cast.keys():
if cast["ident"] == "znc" and cast["host"] == "znc.in":
cast["type"] = "znc"
cast["num"] = self.num
del cast["nick"]
del cast["ident"]
del cast["host"]
del cast["channel"]
if "Disconnected from IRC" in cast["msg"]:
log("ZNC disconnected on %s - %i" % (self.net, self.num))
self.isconnected = False
if "Connected!" in cast["msg"]:
log("ZNC connected on %s - %i" % (self.net, self.num))
self.isconnected = True
if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here
if cast["channel"].lower() == self.nickname.lower(): # as they are channel == nickname
#castDup = deepcopy(cast) # however modes are not queries!
cast["mtype"] = cast["type"]
cast["type"] = "query"
cast["num"] = self.num
#self.event(**castDup)
# Don't call self.event for this one because queries are not events on a
# channel, but we still want to see them
if "user" in cast.keys():
if cast["user"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
cast["num"] = self.num
self.event(**castDup)
if "nick" in cast.keys():
if cast["nick"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
cast["num"] = self.num
if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly
self.event(**castDup) # don't tell anyone else
if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
if not cast["msg"] == None:
if self.nickname.lower() in cast["msg"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "highlight"
cast["num"] = self.num
self.event(**castDup)
if not "net" in cast.keys():
cast["net"] = self.net
counters.event(self.net, cast["type"])
monitor.event(self.net, cast)
def privmsg(self, user, channel, msg):
self.event(type="msg", muser=user, channel=channel, msg=msg)
def noticed(self, user, channel, msg):
self.event(type="notice", muser=user, channel=channel, msg=msg)
def action(self, user, channel, msg):
self.event(type="action", muser=user, channel=channel, msg=msg)
def get(self, var):
try:
result = getattr(self, var)
except AttributeError:
result = None
return result
def setNick(self, nickname):
self._attemptedNick = nickname
self.sendLine("NICK %s" % nickname)
self.nickname = nickname
def alterCollidedNick(self, nickname):
newnick = nickname + "_"
return newnick
def nickChanged(self, olduser, newnick):
self.nickname = newnick
self.event(type="self", mtype="nick", muser=olduser, user=newnick)
def irc_ERR_NICKNAMEINUSE(self, prefix, params):
self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
self.setNick(self._attemptedNick)
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
log("%s - %i: password mismatch" % (self.net, self.num))
sendAll("%s - %i: password mismatch" % (self.net, self.num))
def _who(self, channel):
d = Deferred()
if channel not in self._tempWho:
self._tempWho[channel] = ([], [])
self._tempWho[channel][0].append(d)
self.sendLine("WHO %s" % channel)
return d
def who(self, channel):
self._who(channel).addCallback(self.got_who)
def irc_RPL_WHOREPLY(self, prefix, params):
channel = params[1]
ident = params[2]
host = params[3]
server = params[4]
nick = params[5]
status = params[6]
realname = params[7]
if channel not in self._tempWho:
return
n = self._tempWho[channel][1]
n.append([nick, nick, host, server, status, realname])
self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status)
def irc_RPL_ENDOFWHO(self, prefix, params):
channel = params[1]
if channel not in self._tempWho:
return
callbacks, info = self._tempWho[channel]
for cb in callbacks:
cb.callback((channel, info))
del self._tempWho[channel]
def got_who(self, whoinfo):
userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
def sanit(self, data):
if len(data) >= 1:
if data[0] in ["!", "~", "&", "@", "%", "+"]:
data = data[1:]
return data
return data
else:
return False
def names(self, channel):
d = Deferred()
if channel not in self._tempNames:
self._tempNames[channel] = ([], [])
self._tempNames[channel][0].append(d)
self.sendLine("NAMES %s" % channel)
return d
def irc_RPL_NAMREPLY(self, prefix, params):
channel = params[2]
nicklist = params[3].split(' ')
if channel not in self._tempNames:
return
n = self._tempNames[channel][1]
n.append(nicklist)
def irc_RPL_ENDOFNAMES(self, prefix, params):
channel = params[1]
if channel not in self._tempNames:
return
callbacks, namelist = self._tempNames[channel]
for cb in callbacks:
cb.callback((channel, namelist))
del self._tempNames[channel]
def got_names(self, nicklist):
newNicklist = []
for i in nicklist[1]:
for x in i:
f = self.sanit(x) # need to store this as well, or potentially just do not remove it...
if f:
newNicklist.append(f)
userinfo.initialNames(self.net, nicklist[0], newNicklist)
def myInfo(self, servername, version, umodes, cmodes):
self.servername = servername
def _list(self, noargs):
d = Deferred()
self._tempList = ([], [])
self._tempList[0].append(d)
if self.listSimple:
self.sendLine("LIST")
return d # return early if we know what to do
if noargs:
self.sendLine("LIST")
else:
self.sendLine("LIST >0")
return d
def list(self, noargs=False, nocheck=False):
if self.listAttempted:
debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num))
return
else:
self.listAttempted = True
if self.listOngoing:
debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num))
return
else:
if nocheck:
allRelays = True # override the system - if this is
else: # specified, we already did this
allRelays = chankeep.allRelaysActive(self.net)
if not allRelays:
self.wantList = True
debug("Not all relays were active for LIST request")
return
self._list(noargs).addCallback(self.got_list)
def irc_RPL_LISTSTART(self, prefix, params):
self.listAttempted = False
self.listOngoing = True
self.wantList = False
def irc_RPL_LIST(self, prefix, params):
channel = params[1]
users = params[2]
topic = params[3]
self._tempList[1].append([channel, users, topic])
def irc_RPL_LISTEND(self, prefix, params):
if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there
debug("We didn't ask for this LIST, discarding")
self._tempList[0].clear()
self._tempList[1].clear()
return
callbacks, info = self._tempList
self.listOngoing = False
for cb in callbacks:
cb.callback((info))
noResults = False
if len(self._tempList[1]) == 0:
noResults = True
self._tempList[0].clear()
self._tempList[1].clear()
if noResults:
if self.listRetried:
warn("LIST still empty after retry: %s - %i" % (net, num))
self.listRetried = False
return
else:
self.list(True)
self.listRetried = True
else:
if self.listRetried:
self.listRetried = False
debug("List received after retry - defaulting to simple list syntax")
self.listSimple = True
def got_list(self, listinfo):
if len(listinfo) == 0: # probably ngircd not supporting LIST >0
return
chankeep.initialList(self.net, self.num, listinfo, self.chanlimit)
def isupport(self, options):
interested = ("CHANLIMIT", "MAXCHANNELS")
if not any((x for x in options if any(y in x for y in interested))):
return # check if any of interested is in any of options, some networks
chanlimit = None # call isupport() more than once, so discard swiftly anything
if not self.isconnected: # we don't care about
log("endpoint connected: %s - %i" % (self.net, self.num))
self.isconnected = True
for i in options:
if i.startswith("CHANLIMIT"):
if ":" in i:
split = i.split(":")
if len(split) >= 2:
chanlimit = split[1]
break
elif i.startswith("MAXCHANNELS"):
if "=" in i:
split = i.split("=")
if len(split) == 2:
chanlimit = split[1]
break
print("chanlimit", chanlimit)
try:
self.chanlimit = int(chanlimit)
except TypeError:
warn("Invalid chanlimit: %s" % i)
if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's unlimited
allRelays = chankeep.allRelaysActive(self.net)
print(self.net, self.num, allRelays)
if allRelays:
for i in main.network.keys():
for x in main.network[i].relays.keys():
name = i+str(x)
if main.IRCPool[name].wantList == True:
main.IRCPool[name].list(nocheck=True)
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
if self.num == 1: # Only one instance should do a list
if self.chanlimit:
if allRelays:
self.list()
else:
self.wantList = True
else:
debug("Aborting LIST due to bad chanlimit")
self.checkChannels()
#twisted sucks so i have to do this to actually get the user info
def irc_JOIN(self, prefix, params):
"""
Called when a user joins a channel.
"""
nick = prefix.split('!')[0]
channel = params[-1]
if nick == self.nickname:
self.joined(channel)
else:
self.userJoined(prefix, channel)
def irc_PART(self, prefix, params):
"""
Called when a user leaves a channel.
"""
nick = prefix.split('!')[0]
channel = params[0]
if len(params) >= 2:
message = params[1]
else:
message = None
if nick == self.nickname:
self.left(prefix, channel, message)
else:
self.userLeft(prefix, channel, message)
def irc_QUIT(self, prefix, params):
"""
Called when a user has quit.
"""
nick = prefix.split('!')[0]
#if nick == self.nickname:
#self.botQuit(prefix, params[0])
#else:
self.userQuit(prefix, params[0])
def irc_NICK(self, prefix, params):
"""
Called when a user changes their nickname.
"""
nick = prefix.split('!', 1)[0]
if nick == self.nickname:
self.nickChanged(prefix, params[0])
else:
self.userRenamed(prefix, params[0])
def irc_KICK(self, prefix, params):
"""
Called when a user is kicked from a channel.
"""
#kicker = prefix.split('!')[0]
channel = params[0]
kicked = params[1]
message = params[-1]
# Checks on whether it was us that was kicked are done in userKicked
self.userKicked(kicked, channel, prefix, message)
def irc_TOPIC(self, prefix, params):
"""
Someone in the channel set the topic.
"""
#user = prefix.split('!')[0]
channel = params[0]
newtopic = params[1]
self.topicUpdated(prefix, channel, newtopic)
#END hacks
def signedOn(self):
log("signed on: %s - %i" % (self.net, self.num))
#self.event(type="conn", status="connected")
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"})
def joined(self, channel):
if not channel in self.channels:
self.channels.append(channel)
self.names(channel).addCallback(self.got_names)
if main.config["Toggles"]["Who"]:
#self.who(channel).addCallback(self.got_who)
#self.who(channel)
lc = LoopingCall(self.who, channel)
self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
interval = randint(minint, minint+intrange)
lc.start(interval)
def botLeft(self, channel):
if channel in self.channels:
self.channels.remove(channel)
if channel in self._getWho.keys():
lc = self._getWho[channel]
lc.stop()
del self._getWho[channel]
userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this
#log("Can no longer cover %s, removing records" % channel)# as it will only be matched once --
# other bots have different nicknames so
def left(self, user, channel, message): # even if they saw it, they wouldn't react
self.event(type="part", muser=user, channel=channel, message=message)
self.botLeft(channel)
def userJoined(self, user, channel):
self.event(type="join", muser=user, channel=channel)
def userLeft(self, user, channel, message):
self.event(type="part", muser=user, channel=channel, message=message)
def userQuit(self, user, quitMessage):
self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage})
def userKicked(self, kickee, channel, kicker, message):
if kickee.lower() == self.nickname.lower():
self.botLeft(channel)
self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee)
def chanlessEvent(self, cast):
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
if dedup(self.name, cast): # Needs to be kept self.name until the dedup
# function is converted to the new net, num
# format
return # stop right there sir!
chans = userinfo.getChanList(self.net, cast["nick"])
if chans == None:
error("No channels returned for chanless event: %s" % cast)
# self.event(**cast) -- no, should NEVER happen
return
# getChansSingle returns all channels of the user, we only want to use
# ones we have common with them
realChans = set(chans).intersection(set(self.channels))
for i in realChans:
cast["channel"] = i
self.event(**cast)
def userRenamed(self, oldname, newname):
self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
def topicUpdated(self, user, channel, newTopic):
self.event(type="topic", muser=user, channel=channel, message= newTopic)
def modeChanged(self, user, channel, toset, modes, args):
argList = list(args)
modeList = [i for i in modes]
for a, m in zip(argList, modeList):
self.event(type="mode", muser=user, channel=channel, modes=m, status=toset, modeargs=a)
class IRCBotFactory(ReconnectingClientFactory):
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
if net == None:
self.num = num
self.net = None
self.name = "relay - %i" % num
self.relay = True
else:
self.name = net+str(num)
self.num = num
self.net = net
self.relay = False
self.client = None
self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
self.factor = main.config["Tweaks"]["Delays"]["Factor"]
self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
def buildProtocol(self, addr):
if self.relay == False:
entry = IRCBot(self.net, self.num)
main.IRCPool[self.name] = entry
else:
entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
self.client = entry
return entry
def clientConnectionLost(self, connector, reason):
if not self.relay:
userinfo.delChannels(self.net, self.client.channels)
if not self.client == None:
self.client.isconnected = False
self.client.channels = []
error = reason.getErrorMessage()
log("%s - %i: connection lost: %s" % (self.net, self.num, error))
if not self.relay:
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error})
self.retry(connector)
#ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
if not self.client == None:
self.client.isconnected = False
self.client.channels = []
error = reason.getErrorMessage()
log("%s - %i: connection failed: %s" % (self.net, self.num, error))
if not self.relay:
sendAll("%s -%s: connection failed: %s" % (self.net, self.num, error))
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error})
self.retry(connector)
#ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

29
core/parser.py
View File

@ -1,29 +0,0 @@
import main
from utils.logging.log import *
from utils.logging.send import *
def parseCommand(addr, authed, data):
#call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
spl = data.split()
if addr in main.connections.keys():
obj = main.connections[addr]
else:
warn("Got connection object with no instance in the address pool")
return
success = lambda data: sendSuccess(addr, data)
failure = lambda data: sendFailure(addr, data)
info = lambda data: sendInfo(addr, data)
incUsage = lambda mode: incorrectUsage(addr, mode)
length = len(spl)
if len(spl) > 0:
cmd = spl[0]
else:
failure("No text was sent")
return
if spl[0] in main.CommandMap.keys():
main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length)
return
incUsage(None)
return

110
db.py Normal file
View File

@ -0,0 +1,110 @@
import random
import aioredis
import orjson
# Kafka
from aiokafka import AIOKafkaProducer
from redis import StrictRedis
import util
# KAFKA_TOPIC = "msg"
log = util.get_logger("db")
# Redis (legacy)
r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
# AIORedis
ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
TYPES_MAIN = [
"msg",
"notice",
"action",
"part",
"join",
"kick",
"quit",
"nick",
"mode",
"topic",
"update",
]
TYPES_META = ["who"]
TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
KEYPREFIX = "queue."
async def store_kafka_batch(data):
log.debug(f"Storing Kafka batch of {len(data)} messages")
producer = AIOKafkaProducer(bootstrap_servers="kafka:9092")
await producer.start()
batch = producer.create_batch()
for msg in data:
if msg["type"] in TYPES_MAIN:
index = "main"
# schema = mc_s.schema_main
elif msg["type"] in TYPES_META:
index = "meta"
# schema = mc_s.schema_meta
elif msg["type"] in TYPES_INT:
index = "internal"
# schema = mc_s.schema_int
KAFKA_TOPIC = index
# normalise fields
for key, value in list(msg.items()):
if value is None:
del msg[key]
# if key in schema:
# if isinstance(value, int):
# if schema[key].startswith("string") or schema[key].startswith(
# "text"
# ):
# msg[key] = str(value)
body = orjson.dumps(msg)
# orjson returns bytes
# body = str.encode(message)
if "ts" not in msg:
raise Exception("No TS in msg")
metadata = batch.append(key=None, value=body, timestamp=msg["ts"])
if metadata is None:
partitions = await producer.partitions_for(KAFKA_TOPIC)
partition = random.choice(tuple(partitions))
await producer.send_batch(batch, KAFKA_TOPIC, partition=partition)
log.debug(f"{batch.record_count()} messages sent to partition {partition}")
batch = producer.create_batch()
continue
partitions = await producer.partitions_for(KAFKA_TOPIC)
partition = random.choice(tuple(partitions))
await producer.send_batch(batch, KAFKA_TOPIC, partition=partition)
log.debug(f"{batch.record_count()} messages sent to partition {partition}")
await producer.stop()
async def queue_message(msg):
"""
Queue a message on the Redis buffer.
"""
src = msg["src"]
message = orjson.dumps(msg)
key = f"{KEYPREFIX}{src}"
# log.debug(f"Queueing single message of string length {len(message)}")
await ar.sadd(key, message)
async def queue_message_bulk(data):
"""
Queue multiple messages on the Redis buffer.
"""
# log.debug(f"Queueing message batch of length {len(data)}")
for msg in data:
src = msg["src"]
message = orjson.dumps(msg)
key = f"{KEYPREFIX}{src}"
await ar.sadd(key, message)

259
docker-compose.yml Normal file
View File

@ -0,0 +1,259 @@
version: "2.2"
volumes:
metadata_data: {}
middle_var: {}
historical_var: {}
broker_var: {}
coordinator_var: {}
router_var: {}
druid_shared: {}
services:
app:
image: pathogen/monolith:latest
build: ./docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
env_file:
- .env
volumes_from:
- tmp
depends_on:
broker:
condition: service_started
kafka:
condition: service_healthy
tmp:
condition: service_started
redis:
condition: service_healthy
# - db
threshold:
image: pathogen/threshold:latest
build: ./legacy/docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
env_file:
- .env
# for development
extra_hosts:
- "host.docker.internal:host-gateway"
volumes_from:
- tmp
depends_on:
tmp:
condition: service_started
redis:
condition: service_healthy
turnilo:
container_name: turnilo
image: uchhatre/turnilo:latest
ports:
- 9093:9090
environment:
- DRUID_BROKER_URL=http://broker:8082
depends_on:
- broker
metabase:
container_name: metabase
image: metabase/metabase:latest
ports:
- 3001:3000
depends_on:
- broker
postgres:
container_name: postgres
image: postgres:latest
volumes:
- metadata_data:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD=FoolishPassword
- POSTGRES_USER=druid
- POSTGRES_DB=druid
# Need 3.5 or later for container nodes
zookeeper:
container_name: zookeeper
image: zookeeper:3.5
ports:
- "2181:2181"
environment:
- ZOO_MY_ID=1
kafka:
image: bitnami/kafka
depends_on:
- zookeeper
- broker
ports:
- 29092:29092
- 9092:9092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
#KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
ALLOW_PLAINTEXT_LISTENER: yes
# healthcheck:
# test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"]
# interval: 2s
# timeout: 2s
# retries: 15
healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
start_period: 15s
interval: 2s
timeout: 5s
retries: 30
coordinator:
image: apache/druid:0.23.0
container_name: coordinator
volumes:
- druid_shared:/opt/shared
- coordinator_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
ports:
- "8081:8081"
command:
- coordinator
env_file:
- environment
broker:
image: apache/druid:0.23.0
container_name: broker
volumes:
- broker_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
- coordinator
ports:
- "8082:8082"
command:
- broker
env_file:
- environment
historical:
image: apache/druid:0.23.0
container_name: historical
volumes:
- druid_shared:/opt/shared
- historical_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
- coordinator
ports:
- "8083:8083"
command:
- historical
env_file:
- environment
middlemanager:
image: apache/druid:0.23.0
container_name: middlemanager
volumes:
- druid_shared:/opt/shared
- middle_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
- coordinator
ports:
- "8091:8091"
- "8100-8105:8100-8105"
command:
- middleManager
env_file:
- environment
router:
image: apache/druid:0.23.0
container_name: router
volumes:
- router_var:/opt/druid/var
depends_on:
- zookeeper
- postgres
- coordinator
ports:
- "8888:8888"
command:
- router
env_file:
- environment
# db:
# #image: pathogen/manticore:kibana
# image: manticoresearch/manticore:dev
# #build:
# # context: ./docker/manticore
# # args:
# # DEV: 1
# restart: always
# ports:
# - 9308
# - 9312
# - 9306
# ulimits:
# nproc: 65535
# nofile:
# soft: 65535
# hard: 65535
# memlock:
# soft: -1
# hard: -1
# environment:
# - MCL=1
# volumes:
# - ./docker/data:/var/lib/manticore
# - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
tmp:
image: busybox
command: chmod -R 777 /var/run/redis
volumes:
- /var/run/redis
redis:
image: redis
command: redis-server /etc/redis.conf
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
volumes:
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
volumes_from:
- tmp
healthcheck:
test: "redis-cli -s /var/run/redis/redis.sock ping"
interval: 2s
timeout: 2s
retries: 15
networks:
default:
external:
name: pathogen

23
docker/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
# syntax=docker/dockerfile:1
FROM python:3
RUN useradd -d /code pathogen
RUN mkdir /code
RUN chown pathogen:pathogen /code
RUN mkdir /venv
RUN chown pathogen:pathogen /venv
USER pathogen
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
WORKDIR /code
COPY requirements.txt /code/
COPY discord-patched.tgz /code/
RUN python -m venv /venv
RUN . /venv/bin/activate && pip install -r requirements.txt && python -m spacy download en_core_web_sm
RUN tar xf /code/discord-patched.tgz -C /venv/lib/python3.10/site-packages
CMD . /venv/bin/activate && exec python monolith.py

BIN
docker/discord-patched.tgz Normal file
View File

Binary file not shown.

77
docker/docker-compose.prod.yml Normal file
View File

@ -0,0 +1,77 @@
version: "2"
services:
app:
image: pathogen/monolith:latest
build: ./docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
env_file:
- ../stack.env
volumes_from:
- tmp
depends_on:
- db
threshold:
image: pathogen/threshold:latest
build: ./legacy/docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
env_file:
- ../stack.env
volumes_from:
- tmp
db:
image: manticoresearch/manticore:latest
restart: always
ports:
- 9308
- 9312
- 9306
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
volumes:
- /opt/docker/pathogen/monolith/prod:/var/lib/manticore
- ${PORTAINER_GIT_DIR}/docker/manticore.conf:/etc/manticoresearch/manticore.conf
tmp:
image: busybox
command: chmod -R 777 /var/run/redis
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
volumes:
- /var/run/redis
redis:
image: redis
command: redis-server /etc/redis.conf
volumes:
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
volumes_from:
- tmp
networks:
default:
external:
name: pathogen

265
docker/manticore.conf Normal file
View File

@ -0,0 +1,265 @@
#!/bin/sh
ip=`hostname -i|rev|cut -d\ -f 1|rev`
cat << EOF
searchd {
# https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
# access_plain_attrs = mmap_preread
# https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
# access_blob_attrs = mmap_preread
# https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
# access_doclists = file
# https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
# access_hitlists = file
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
# agent_connect_timeout =
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
# agent_query_timeout =
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
# agent_retry_count = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
# agent_retry_delay = 500
# https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
# attr_flush_period = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
# binlog_flush = 2
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
# binlog_max_log_size = 268435456
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
# binlog_path =
# https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
# client_timeout = 300
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
# collation_libc_locale = C
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
# collation_server = libc_ci
# https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
data_dir = /var/lib/manticore
# https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
# docstore_cache_size = 16m
# https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
# expansion_limit = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
# grouping_in_utc = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
# ha_period_karma = 60
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
# ha_ping_interval = 1000
# https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
# hostname_lookup =
# https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
# jobs_queue_size =
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
# listen_backlog = 5
# https://manual.manticoresearch.com/Server_settings/Searchd#listen
# listen_env = this directive allows to append listeners from environment variables
listen = 9306:mysql41
listen = /var/run/mysqld/mysqld.sock:mysql41
listen = $ip:9312
listen = 9308:http
listen = $ip:9315-9325:replication
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
# listen_tfo = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#log
log = /var/log/manticore/searchd.log
# https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
# max_batch_queries = 32
# https://manual.manticoresearch.com/Server_settings/Searchd#threads
# threads =
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
# max_filters = 256
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
# max_filter_values = 4096
# https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
# max_open_files = max
# https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
max_packet_size = 128M
# https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
# mysql_version_string =
# https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
# net_workers = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
# net_wait_tm = -1
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
# net_throttle_accept = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
# net_throttle_action = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#node_address
# node_address =
# https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
# ondisk_attrs_default = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
# persistent_connections_limit =
# https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
pid_file = /var/run/manticore/searchd.pid
# https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
# predicted_time_costs = doc=64, hit=48, skip=2048, match=64
# https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
# preopen_indexes = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
qcache_max_bytes = 128Mb
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
qcache_thresh_msec = 150
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
qcache_ttl_sec = 120
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
query_log_format = sphinxql
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
# query_log_min_msec = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log
# query_log = /var/log/manticore/query.log
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
# query_log_mode = 600
# https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
# max_connections =
# https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
# network_timeout = 5
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
# read_buffer = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
# read_buffer_docs = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
# read_buffer_hits = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
# read_unhinted 32K
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
# rt_flush_period =
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
# rt_merge_iops = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
# rt_merge_maxiosize = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
# seamless_rotate = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#server_id
# server_id =
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
# shutdown_timeout = 3
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
# shutdown_token =
# https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
# snippets_file_prefix =
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
# sphinxql_state =
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
# sphinxql_timeout = 900
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
# ssl_ca =
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
# ssl_cert =
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
# ssl_key =
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
# subtree_docs_cache = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
# subtree_hits_cache = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
# thread_stack =
# https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
# unlink_old = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
# watchdog = 1
}
common {
# https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
# lemmatizer_base = /usr/local/share
# https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
# progressive_merge =
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
# json_autoconv_keynames =
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
# json_autoconv_numbers = 0
# https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
# on_json_attr_error = ignore_attr
# https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
# plugin_dir =
}
# indexer {
# lemmatizer_cache = 1024M
# max_iops = 0
# max_iosize = 0
# mem_limit = 1024M
# }
EOF

12
docker/manticore/.mysql_history Normal file
View File

@ -0,0 +1,12 @@
_HiStOrY_V2_
SELECT * FROM films WHERE MATCH('"shark monkey boy robot"/2') AND release_year IN(2006,2007) AND rental_rate BETWEEN 2.0 and 3.0;
SELECT title, HIGHLIGHT({},'description') FROM films WHERE MATCH('"shark monkey boy robot"/2');
SELECT * FROM films WHERE MATCH('" shark monkey boy robot "/2');
SELECT * FROM films WHERE MATCH('Emotional drama') FACET release_year FACET category_id;
SELECT * FROM films WHERE MATCH('Emotional drama') GROUP BY release_year;
SELECT * FROM films WHERE MATCH('Emotional drama -dog -shark');
SELECT * FROM films WHERE MATCH('Emotional drama');
SELECT * FROM films;
DESCRIBE films;
SHOW TABLES;
SOURCE /sandbox.sql

76
docker/manticore/Dockerfile Normal file
View File

@ -0,0 +1,76 @@
FROM ubuntu:focal
ARG DEV
ARG DAEMON_URL
ARG MCL_URL
RUN groupadd -r manticore && useradd -r -g manticore manticore
ENV GOSU_VERSION 1.11
ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"}
RUN set -x \
&& apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
&& export GNUPGHOME="$(mktemp -d)" \
&& gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
&& rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc \
&& chmod +x /usr/local/bin/gosu \
&& gosu nobody true && \
if [ "${DEV}" = "1" ]; then \
echo "DEV IS ONE" && \
exit && \
wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
&& dpkg -i manticore-dev-repo.noarch.deb \
&& apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
&& apt-get update \
&& echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
elif [ "${DEV}" = "2" ]; then \
echo "DEV IS TWO" && \
wget $BETA_URL && unzip focal.zip && rm focal.zip && \
dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\
else \
echo "DEV NOT EITHER" && \
exit && \
wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
fi \
&& mkdir -p /var/run/manticore && mkdir -p /var/lib/manticore/replication \
&& apt-get update && apt-get -y install libexpat1 libodbc1 libpq5 openssl libcrypto++6 libmysqlclient21 mysql-client \
&& apt-get -y purge --auto-remove \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /usr/bin/mariabackup /usr/bin/mysqldump /usr/bin/mysqlslap /usr/bin/mysqladmin /usr/bin/mysqlimport \
/usr/bin/mysqlshow /usr/bin/mbstream /usr/bin/mysql_waitpid /usr/bin/innotop /usr/bin/mysqlaccess /usr/bin/mytop \
/usr/bin/mysqlreport /usr/bin/mysqldumpslow /usr/bin/mysql_find_rows /usr/bin/mysql_fix_extensions \
/usr/bin/mysql_embedded /usr/bin/mysqlcheck \
&& rm -f /usr/bin/spelldump /usr/bin/wordbreaker \
&& mkdir -p /var/run/mysqld/ && chown manticore:manticore /var/run/mysqld/ \
&& echo "\n[mysql]\nsilent\nwait\ntable\n" >> /etc/mysql/my.cnf && \
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/en.pak.tgz && \
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/de.pak.tgz && \
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/ru.pak.tgz && \
tar -xf /tmp/en.pak.tgz -C /usr/share/manticore/ && \
tar -xf /tmp/de.pak.tgz -C /usr/share/manticore/ && \
tar -xf /tmp/ru.pak.tgz -C /usr/share/manticore/
COPY manticore.conf /etc/manticoresearch/
COPY sandbox.sql /sandbox.sql
COPY .mysql_history /root/.mysql_history
COPY docker-entrypoint.sh /usr/local/bin/
RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh
WORKDIR /var/lib/manticore
ENTRYPOINT ["docker-entrypoint.sh"]
EXPOSE 9306
EXPOSE 9308
EXPOSE 9312
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"]

278
docker/manticore/README.md Normal file
View File

@ -0,0 +1,278 @@
# Manticore Search Docker image
This is the git repo of official [Docker image](https://hub.docker.com/r/manticoresearch/manticore/) for [Manticore Search](https://github.com/manticoresoftware/manticoresearch).
Manticore Search is an easy to use open source fast database for search. It helps thousands of companies from small to large, such as Craigslist, to search and filter petabytes of text data on a single or hundreds of nodes, do stream full-text filtering, add auto-complete, spell correction, more-like-this, faceting and other search-related technologies to their websites and applications.
The default configuration includes a sample Real-Time index and listens on the default ports:
* `9306` for connections from a MySQL client
* `9308` for connections via HTTP
* `9312` for connections via a binary protocol (e.g. in case you run a cluster)
The image comes with libraries for easy indexing data from MySQL, PostgreSQL XML and CSV files.
# How to run Manticore Search Docker image
## Quick usage
The below is the simplest way to start Manticore in a container and log in to it via mysql client:
```bash
docker run --name manticore --rm -d manticoresearch/manticore && sleep 3 && docker exec -it manticore mysql && docker stop manticore
```
When you exit from the mysql client it stops and removes the container, so **use it only for testing / sandboxing purposes**. See below how to use it in production.
The image comes with a sample index which can be loaded like this:
```mysql
mysql> source /sandbox.sql
```
Also the mysql client has in history several sample queries that you can run on the above index, just use Up/Down keys in the client to see and run them.
## Production use
### Ports and mounting points
For data persistence `/var/lib/manticore/` should be mounted to local storage or other desired storage engine.
```bash
docker run --name manticore -v $(pwd)/data:/var/lib/manticore -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
```
Configuration file inside the instance is located at `/etc/manticoresearch/manticore.conf`. For custom settings, this file should be mounted to your own configuration file.
The ports are 9306/9308/9312 for SQL/HTTP/Binary, expose them depending on how you are going to use Manticore. For example:
```bash
docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data:/var/lib/manticore/ -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
```
Make sure to remove `127.0.0.1:` if you want the ports to be available for external hosts.
### Manticore Columnar Library
The docker image doesn't include [Manticore Columnar Library](https://github.com/manticoresoftware/columnar) which has to be used if you need:
* columnar storage
* secondary indexes
but you can easily enable it in runtime by using environment variable `MCL=1`, i.e. `docker run -e MCL=1 ... manticoresearch/manticore`. It will then download and install the library and put it to the data dir (which is normally mapped as a volume in production). Next time you run the container the library will be already there, hence it won't be downloaded again unless you change the Manticore Search version.
### Docker-compose
In many cases you might want to use Manticore together with other images specified in a docker-compose YAML file. Here is the minimal recommended specification for Manticore Search in docker-compose.yml:
```yaml
version: '2.2'
services:
manticore:
container_name: manticore
image: manticoresearch/manticore
restart: always
ports:
- 127.0.0.1:9306:9306
- 127.0.0.1:9308:9308
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
volumes:
- ./data:/var/lib/manticore
# - ./manticore.conf:/etc/manticoresearch/manticore.conf # uncommment if you use a custom config
```
Besides using the exposed ports 9306 and 9308 you can log into the instance by running `docker-compose exec manticore mysql`.
### HTTP protocol
Manticore is accessible via HTTP on ports 9308 and 9312. You can map either of them locally and connect with curl:
```bash
docker run --name manticore -p 9308:9308 -d manticoresearch/manticore
```
Create a table:
```bash
curl -X POST 'http://127.0.0.1:9308/sql' -d 'mode=raw&query=CREATE TABLE testrt ( title text, content text, gid integer)'
```
Insert a document:
```bash
curl -X POST 'http://127.0.0.1:9308/json/insert' -d'{"index":"testrt","id":1,"doc":{"title":"Hello","content":"world","gid":1}}'
```
Perform a simple search:
```bash
curl -X POST 'http://127.0.0.1:9308/json/search' -d '{"index":"testrt","query":{"match":{"*":"hello world"}}}'
```
### Logging
By default, Manticore logs to `/dev/stdout`, so you can watch the log on the host with:
```bash
docker logs manticore
```
If you want to get log of your queries the same way you can do it by passing environment variable `QUERY_LOG_TO_STDOUT=true`.
### Multi-node cluster with replication
Here is a simple `docker-compose.yml` for defining a two node cluster:
```yaml
version: '2.2'
services:
manticore-1:
image: manticoresearch/manticore
restart: always
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
networks:
- manticore
manticore-2:
image: manticoresearch/manticore
restart: always
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
memlock:
soft: -1
hard: -1
environment:
- MCL=1
networks:
- manticore
networks:
manticore:
driver: bridge
```
* Start it: `docker-compose up`
* Create a cluster with a table:
```mysql
$ docker-compose exec manticore-1 mysql
mysql> CREATE TABLE testrt ( title text, content text, gid integer);
mysql> CREATE CLUSTER posts;
Query OK, 0 rows affected (0.24 sec)
mysql> ALTER CLUSTER posts ADD testrt;
Query OK, 0 rows affected (0.07 sec)
MySQL [(none)]> exit
Bye
```
* Join to the the cluster on the 2nd instance and insert smth to the table:
```mysql
$ docker-compose exec manticore-2 mysql
mysql> JOIN CLUSTER posts AT 'manticore-1:9312';
mysql> INSERT INTO posts:testrt(title,content,gid) VALUES('hello','world',1);
Query OK, 1 row affected (0.00 sec)
MySQL [(none)]> exit
Bye
```
* If you now go back to the first instance you'll see the new record:
```mysql
$ docker-compose exec manticore-1 mysql
MySQL [(none)]> select * from testrt;
+---------------------+------+-------+---------+
| id | gid | title | content |
+---------------------+------+-------+---------+
| 3891565839006040065 | 1 | hello | world |
+---------------------+------+-------+---------+
1 row in set (0.00 sec)
MySQL [(none)]> exit
Bye
```
## Memory locking and limits
It's recommended to overwrite the default ulimits of docker for the Manticore instance:
```bash
--ulimit nofile=65536:65536
```
For the best performance, Manticore tables' components can be locked into memory. When Manticore is run under Docker, the instance requires additional privileges to allow memory locking. The following options must be added when running the instance:
```bash
--cap-add=IPC_LOCK --ulimit memlock=-1:-1
```
## Configuring Manticore Search with Docker
If you want to run Manticore with your custom config containing indexes definition you will need to mount the configuration to the instance:
```bash
docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data/:/var/lib/manticore -p 127.0.0.1:9306:9306 -d manticoresearch/manticore
```
Take into account that Manticore search inside the container is run under user `manticore`. Performing operations with tables (like creating or rotating plain indexes) should be also done under `manticore`. Otherwise the files will be created under `root` and the search daemon won't have rights to open them. For example here is how you can rotate all plain indexes:
```bash
docker exec -it manticore gosu manticore indexer --all --rotate
```
### Environment variables
You can also set individual `searchd` and `common` configuration settings using Docker environment variables.
The settings must be prefixed with their section name, for example to change value of setting `mysql_version_string` in section `searchd` the variable must be named `searchd_mysql_version_string`:
```bash
docker run --name manticore -p 127.0.0.1:9306:9306 -e searchd_mysql_version_string='5.5.0' -d manticoresearch/manticore
```
In case of `listen` directive, you can pass using Docker variable `searchd_listen` new listening interfaces in addition to the default ones. Multiple interfaces can be declared separated by semi-colon ("|").
For listening only on network address, the `$ip` (retrieved internally from `hostname -i`) can be used as address alias.
For example `-e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip'` will add an additional SQL interface on port 9307, a SQL VIP on 5443 running only on the instance IP and HTTP on port 9316, beside the defaults on 9306 and 9308, respectively.
```bash
$ docker run --rm -p 1188:9307 -e searchd_mysql_version_string='5.5.0' -e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip' manticore
[Mon Aug 17 07:31:58.719 2020] [1] using config file '/etc/manticoresearch/manticore.conf' (9130 chars)...
listening on all interfaces for http, port=9316
listening on all interfaces for mysql, port=9307
listening on 172.17.0.17:5443 for VIP mysql
listening on all interfaces for mysql, port=9306
listening on UNIX socket /var/run/mysqld/mysqld.sock
listening on 172.17.0.17:9312 for sphinx
listening on all interfaces for http, port=9308
prereading 0 indexes
prereaded 0 indexes in 0.000 sec
accepting connections
```
# Issues
For reporting issues, please use the [issue tracker](https://github.com/manticoresoftware/docker/issues).

118
docker/manticore/docker-entrypoint.sh Executable file
View File

@ -0,0 +1,118 @@
#!/bin/bash
set -eo pipefail
echo "RUNNING ENTRYPOINT"
# check to see if this file is being run or sourced from another script
_is_sourced() {
# https://unix.stackexchange.com/a/215279
[ "${#FUNCNAME[@]}" -ge 2 ] &&
[ "${FUNCNAME[0]}" = '_is_sourced' ] &&
[ "${FUNCNAME[1]}" = 'source' ]
}
_searchd_want_help() {
local arg
for arg; do
case "$arg" in
-'?' | --help | -h | -v)
return 0
;;
esac
done
return 1
}
docker_setup_env() {
if [ -n "$QUERY_LOG_TO_STDOUT" ]; then
export searchd_query_log=/var/log/manticore/query.log
[ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
fi
if [[ "${MCL}" == "1" ]]; then
LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
[ -L /usr/share/manticore/modules/lib_manticore_columnar.so ] || ln -s $LIB_MANTICORE_COLUMNAR /usr/share/manticore/modules/lib_manticore_columnar.so
[ -L /usr/share/manticore/modules/lib_manticore_secondary.so ] || ln -s $LIB_MANTICORE_SECONDARY /usr/share/manticore/modules/lib_manticore_secondary.so
searchd -v|grep -i error|egrep "trying to load" \
&& rm $LIB_MANTICORE_COLUMNAR $LIB_MANTICORE_SECONDARY \
&& echo "WARNING: wrong MCL version was removed, installing the correct one"
if [[ ! -f "$LIB_MANTICORE_COLUMNAR" || ! -f "$LIB_MANTICORE_SECONDARY" ]]; then
if ! mkdir -p /var/lib/manticore/.mcl/ ; then
echo "ERROR: Manticore Columnar Library is inaccessible: couldn't create /var/lib/manticore/.mcl/."
exit
fi
MCL_URL=$(cat /mcl.url)
wget -P /tmp $MCL_URL
LAST_PATH=$(pwd)
cd /tmp
PACKAGE_NAME=$(ls | grep manticore-columnar | head -n 1)
ar -x $PACKAGE_NAME
tar -xf data.tar.gz
find . -name '*.so' -exec cp {} /var/lib/manticore/.mcl/ \;
cd $LAST_PATH
fi
fi
}
_main() {
# first arg is `h` or some `--option`
if [ "${1#-}" != "$1" ]; then
set -- searchd "$@"
fi
# Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options
if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then
docker_setup_env "$@"
# allow the container to be started with `--user`
if [ "$(id -u)" = '0' ]; then
find /var/lib/manticore /var/log/manticore /var/run/manticore /etc/manticoresearch \! -user manticore -exec chown manticore '{}' +
exec gosu manticore "$0" "$@"
fi
fi
_replace_conf_from_env
exec "$@"
}
_replace_conf_from_env() {
sed_query=""
while IFS='=' read -r oldname value; do
if [[ $oldname == 'searchd_'* || $oldname == 'common_'* ]]; then
value=$(echo ${!oldname} | sed 's/\//\\\//g')
oldname=$(echo $oldname | sed "s/searchd_//g;s/common_//g;")
newname=$oldname
if [[ $newname == 'listen' ]]; then
oldname="listen_env"
IFS='|' read -ra ADDR <<<"$value"
count=0
for i in "${ADDR[@]}"; do
if [[ $count == 0 ]]; then
value=$i
else
value="$value\n listen = $i"
fi
count=$((count + 1))
done
fi
if [[ -z $sed_query ]]; then
sed_query="s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
else
sed_query="$sed_query;s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
fi
fi
done < <(env)
if [[ ! -z $sed_query ]]; then
sed -i -E "$sed_query" /etc/manticoresearch/manticore.conf
fi
}
# If we are sourced from elsewhere, don't perform any further actions
if ! _is_sourced; then
_main "$@"
fi

259
docker/manticore/manticore.conf Normal file
View File

@ -0,0 +1,259 @@
#!/bin/sh
ip=`hostname -i|rev|cut -d\ -f 1|rev`
cat << EOF
searchd {
# https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
# access_plain_attrs = mmap_preread
# https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
# access_blob_attrs = mmap_preread
# https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
# access_doclists = file
# https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
# access_hitlists = file
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
# agent_connect_timeout =
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
# agent_query_timeout =
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
# agent_retry_count = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
# agent_retry_delay = 500
# https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
# attr_flush_period = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
# binlog_flush = 2
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
# binlog_max_log_size = 268435456
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
# binlog_path =
# https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
# client_timeout = 300
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
# collation_libc_locale = C
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
# collation_server = libc_ci
# https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
data_dir = /var/lib/manticore
# https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
# docstore_cache_size = 16m
# https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
# expansion_limit = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
# grouping_in_utc = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
# ha_period_karma = 60
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
# ha_ping_interval = 1000
# https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
# hostname_lookup =
# https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
# jobs_queue_size =
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
# listen_backlog = 5
# https://manual.manticoresearch.com/Server_settings/Searchd#listen
# listen_env = this directive allows to append listeners from environment variables
listen = 9306:mysql41
listen = /var/run/mysqld/mysqld.sock:mysql41
listen = $ip:9312
listen = 9308:http
listen = $ip:9315-9325:replication
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
# listen_tfo = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#log
log = /var/log/manticore/searchd.log
# https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
# max_batch_queries = 32
# https://manual.manticoresearch.com/Server_settings/Searchd#threads
# threads =
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
# max_filters = 256
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
# max_filter_values = 4096
# https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
# max_open_files =
# https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
max_packet_size = 128M
# https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
# mysql_version_string =
# https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
# net_workers = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
# net_wait_tm = -1
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
# net_throttle_accept = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
# net_throttle_action = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#node_address
# node_address =
# https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
# ondisk_attrs_default = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
# persistent_connections_limit =
# https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
pid_file = /var/run/manticore/searchd.pid
# https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
# predicted_time_costs = doc=64, hit=48, skip=2048, match=64
# https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
# preopen_indexes = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
# qcache_max_bytes = 16Mb
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
# qcache_thresh_msec = 3000
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
# qcache_ttl_sec = 60
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
query_log_format = sphinxql
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
query_log_min_msec = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log
query_log = /var/log/manticore/query.log
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
# query_log_mode = 600
# https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
# max_connections =
# https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
# network_timeout = 5
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
# read_buffer = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
# read_buffer_docs = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
# read_buffer_hits = 256K
# https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
# read_unhinted 32K
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
# rt_flush_period =
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
# rt_merge_iops = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
# rt_merge_maxiosize = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
# seamless_rotate = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#server_id
# server_id =
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
# shutdown_timeout = 3
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
# shutdown_token =
# https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
# snippets_file_prefix =
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
# sphinxql_state =
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
# sphinxql_timeout = 900
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
# ssl_ca =
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
# ssl_cert =
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
# ssl_key =
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
# subtree_docs_cache = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
# subtree_hits_cache = 0
# https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
# thread_stack =
# https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
# unlink_old = 1
# https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
# watchdog = 1
}
common {
# https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
# lemmatizer_base = /usr/local/share
# https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
# progressive_merge =
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
# json_autoconv_keynames =
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
# json_autoconv_numbers = 0
# https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
# on_json_attr_error = ignore_attr
# https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
# plugin_dir =
}
EOF

9
docker/manticore/sandbox.sql Normal file
View File

File diff suppressed because one or more lines are too long

2
docker/redis.conf Normal file
View File

@ -0,0 +1,2 @@
unixsocket /var/run/redis/redis.sock
unixsocketperm 777

21
docker/requirements.txt Normal file
View File

@ -0,0 +1,21 @@
wheel
beautifulsoup4
redis
siphashc
aiohttp[speedups]
python-dotenv
#manticoresearch
numpy
ujson
aioredis[hiredis]
aiokafka
vaderSentiment
polyglot
pyicu
pycld2
morfessor
six
nltk
spacy
python-Levenshtein
orjson

52
environment Normal file
View File

@ -0,0 +1,52 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Java tuning
DRUID_XMX=1g
DRUID_XMS=1g
DRUID_MAXNEWSIZE=250m
DRUID_NEWSIZE=250m
DRUID_MAXDIRECTMEMORYSIZE=6172m
druid_emitter_logging_logLevel=debug
druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
druid_zk_service_host=zookeeper
druid_metadata_storage_host=
druid_metadata_storage_type=postgresql
druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
druid_metadata_storage_connector_user=druid
druid_metadata_storage_connector_password=FoolishPassword
druid_coordinator_balancer_strategy=cachingCost
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
druid_storage_type=local
druid_storage_storageDirectory=/opt/shared/segments
druid_indexer_logs_type=file
druid_indexer_logs_directory=/opt/shared/indexing-logs
druid_processing_numThreads=2
druid_processing_numMergeBuffers=2
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

0
commands/__init__.py → event_log.txt
View File

13
legacy/.env.example Normal file
View File

@ -0,0 +1,13 @@
THRESHOLD_LISTENER_HOST=0.0.0.0
THRESHOLD_LISTENER_PORT=13867
THRESHOLD_LISTENER_SSL=1
THRESHOLD_RELAY_ENABLED=1
THRESHOLD_RELAY_HOST=0.0.0.0
THRESHOLD_RELAY_PORT=13868
THRESHOLD_RELAY_SSL=1
THRESHOLD_API_ENABLED=1
THRESHOLD_API_HOST=0.0.0.0
THRESHOLD_API_PORT=13869
PORTAINER_GIT_DIR=.

11
legacy/.gitignore vendored Normal file
View File

@ -0,0 +1,11 @@
*.pyc
*.pem
*.swp
__pycache__/
conf/live/
conf/cert/
env/
venv/
.idea/
.env
.bash_history

17
legacy/.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,17 @@
repos:
- repo: https://github.com/psf/black
rev: 22.6.0
hooks:
- id: black
args:
- --line-length=120
- repo: https://github.com/PyCQA/isort
rev: 5.10.1
hooks:
- id: isort
args: ["--profile", "black"]
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
hooks:
- id: flake8
args: [--max-line-length=120]

746
legacy/api/views.py Normal file
View File

@ -0,0 +1,746 @@
import functools
from json import JSONDecodeError, dumps, loads
from string import digits
import main
from klein import Klein
from modules import chankeep, helpers, provision, regproc, userinfo
from modules.network import Network
from twisted.web.server import Request
from utils.logging.log import warn
def login_required(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if isinstance(args[0], Request):
request = args[0]
apikey = request.getHeader("ApiKey")
token = request.getHeader("Token")
if not apikey:
return "No API key provided"
if not token:
return "No token provided"
if apikey not in main.tokens:
return "No such API key"
config_token = main.tokens[apikey]
if not token == config_token["hello"]:
return "Invalid token"
counter = config_token["counter"]
request.setHeader("Counter", counter)
return func(self, *args, **kwargs)
return wrapper
class API(object):
"""
Our API webapp.
"""
app = Klein()
@app.route("/", methods=["GET"])
@login_required
def hello(self, request):
return "Hello"
@app.route("/who/<query>/", methods=["POST"])
@login_required
def who(self, request, query):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "query" not in data:
return "No query provided"
result = userinfo.getWho(data["query"])
# Expand the generator
return dumps({k: [x for x in v] for k, v in result.items()})
@app.route("/chans/", methods=["POST"])
@login_required
def chans(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "net" not in data:
return "No net provided"
if "query" not in data:
return "No query provided"
if not data["query"]:
warn(f"No query provided: for chans {data}")
return dumps({})
result = userinfo.getChansSingle(data["net"], data["query"])
if not result:
return dumps({})
return dumps({"chans": [x for x in result]})
@app.route("/users/", methods=["POST"])
@login_required
def users(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "net" not in data:
return "No net provided"
if "query" not in data:
return "No query provided"
if not data["query"]:
warn(f"No query provided for users: {data}")
return dumps({})
result = userinfo.getUsersSingle(data["net"], data["query"])
if not result:
return dumps({})
return dumps({"users": [x for x in result]})
@app.route("/online/", methods=["POST"])
@login_required
def online(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "net" not in data:
return "No net provided"
if "query" not in data:
return "No users provided"
if not data["query"]:
warn(f"No users provided: for online {data}")
return dumps({})
net = data["net"]
usermap = {}
for user in data["query"]:
channels = userinfo.getChansSingle(net, [user])
if channels:
usermap[user] = True
else:
usermap[user] = False
return dumps(usermap)
@app.route("/num_users/", methods=["POST"])
@login_required
def num_users(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "net" not in data:
return "No net provided"
if "query" not in data:
return "No users provided"
if not data["query"]:
warn(f"No chans provided: for num_users {data}")
return dumps({})
net = data["net"]
results = userinfo.getUserNum(net, data["query"])
return dumps(results)
@app.route("/num_chans/", methods=["POST"])
@login_required
def num_chans(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "net" not in data:
return "No net provided"
if "query" not in data:
return "No users provided"
if not data["query"]:
warn(f"No users provided: for num_chans {data}")
return dumps({})
net = data["net"]
results = userinfo.getChanNum(net, data["query"])
return dumps(results)
@app.route("/irc/stats/", methods=["POST"])
@login_required
def irc_stats(self, request):
stats = {}
numChannels = 0
numWhoEntries = 0
for i in main.IRCPool.keys():
numChannels += len(main.IRCPool[i].channels)
numWhoEntries += userinfo.getNumTotalWhoEntries()
numRelays = 0
for net in main.network.keys():
numRelays += len(main.network[net].relays)
stats["servers_total_total"] = numRelays
stats["servers_total_unique"] = len(main.network.keys())
stats["servers_online_total"] = len(main.IRCPool.keys())
stats["servers_online_unique"] = len(main.liveNets())
stats["channels"] = numChannels
stats["records"] = numWhoEntries
stats["eventrate"] = main.lastMinuteSample
return dumps(stats)
@app.route("/irc/networks/", methods=["POST"])
@login_required
def irc_networks(self, request):
networks = {}
for net in main.network.keys():
networks[net] = {
"active": chankeep.allRelaysActive(net),
"relays": len(main.network[net].relays),
"channels": userinfo.getTotalChanNum(net),
"records": userinfo.getNumWhoEntries(net),
}
return dumps(networks)
@app.route("/irc/auth/", methods=["POST"])
@login_required
def irc_network_recheckauth(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "func" not in data:
return dumps({"success": False, "reason": "no function specified."})
func = data["func"]
if "net" not in data:
return dumps({"success": False, "reason": "no net specified."})
net = data["net"]
if not net or net == "None":
nets = main.network.keys()
else:
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
nets = [net]
for net_name in nets:
conns = helpers.get_connected_relays(net_name)
if not conns:
return dumps(
{
"success": False,
"reason": f"failed to get instances for {net_name}.",
}
)
if func == "recheckauth":
for conn in conns:
conn.regPing(reset=True)
elif func == "resetauth":
for conn in conns:
conn.authenticated = False
conn.regPing(reset=True)
elif func == "register":
for conn in conns:
regproc.registerAccount(conn.net, conn.num)
return dumps({"success": True})
@app.route("/irc/network/<net>/", methods=["POST"])
@login_required
def irc_network(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
inst = main.network[net]
network = {}
network["net"] = inst.net
network["auth"] = inst.auth
network["host"] = inst.host
network["last"] = inst.last
network["port"] = inst.port
network["security"] = inst.security
network["relays"] = len(inst.relays)
network["chanlimit"] = inst.chanlimit
network["channels"] = userinfo.getTotalChanNum(net)
network["records"] = userinfo.getNumWhoEntries(net)
return dumps(network)
@app.route("/irc/network/<net>/", methods=["DELETE"])
@login_required
def irc_network_delete(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
main.network[net].seppuku()
del main.network[net]
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/<net>/edit/", methods=["POST"])
@login_required
def irc_network_edit(self, request, net):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
inst = main.network[net]
for item in data:
if item == "auth":
auth = data[item][0]
if auth not in ["sasl", "ns", "none"]:
return dumps({"success": False, "reason": "invalid auth."})
elif item == "host":
host = data[item][0]
elif item == "last":
last = data[item][0]
if not last.isdigit():
return dumps(
{"success": False, "reason": "invalid last: not a number."}
)
elif item == "port":
port = data[item][0]
if not port.isdigit():
return dumps(
{"success": False, "reason": "invalid port: not a number."}
)
port = int(port)
elif item == "chanlimit":
chanlimit = data[item][0]
if chanlimit == "None":
chanlimit = None
elif not chanlimit.isdigit():
return dumps(
{"success": False, "reason": "invalid chanlimit: not a number."}
)
else:
chanlimit = int(chanlimit)
online_relays = helpers.get_connected_relays(net)
for r in online_relays:
r.chanlimit = chanlimit
elif item == "security":
security = data[item][0]
if security not in ["ssl", "plain"]:
return dumps({"success": False, "reason": "invalid security."})
inst.auth = auth
inst.host = host
inst.last = last
inst.port = port
inst.security = security
inst.chanlimit = chanlimit
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/create/", methods=["PUT"])
@login_required
def irc_network_create(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
fields = ["net", "auth", "host", "port", "security"]
if not set(fields).issubset(set(data)):
return dumps({"success": False, "reason": "not enough fields."})
for item in data:
if item == "net":
net = data[item]
if net in main.network.keys():
return dumps(
{"success": False, "reason": "network already exists."}
)
if set(net).intersection(set(digits)):
return dumps(
{
"success": False,
"reason": "network name cannot contain numbers.",
}
)
elif item == "auth":
auth = data[item]
if auth not in ["sasl", "ns", "none"]:
return dumps({"success": False, "reason": "invalid auth."})
elif item == "host":
host = data[item]
elif item == "port":
port = data[item]
if not port.isdigit():
return dumps(
{"success": False, "reason": "invalid port: not a number."}
)
port = int(port)
elif item == "security":
security = data[item]
if security not in ["ssl", "plain"]:
return dumps({"success": False, "reason": "invalid security."})
main.network[net] = Network(net, host, int(port), security, auth)
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/<net>/relays/", methods=["POST"])
@login_required
def irc_network_relays(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
relays_inst = main.network[net].relays
relays = []
for num in relays_inst.keys():
to_append = relays_inst[num]
name = f"{net}{num}"
if name in main.IRCPool.keys():
to_append["chans"] = len(main.IRCPool[name].channels)
to_append["nick"] = main.IRCPool[name].nickname
to_append["conn"] = main.IRCPool[name].isconnected
to_append["limit"] = main.IRCPool[name].chanlimit
to_append["authed"] = main.IRCPool[name].authenticated
else:
to_append["chans"] = 0
to_append["nick"] = None
to_append["conn"] = False
to_append["limit"] = None
to_append["authed"] = None
relays.append(to_append)
return dumps({"relays": relays})
@app.route("/irc/network/<net>/<num>/", methods=["POST"])
@login_required
def irc_network_relay(self, request, net, num):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
net_inst = main.network[net]
if num not in net_inst.relays:
return dumps({"success": False, "reason": "network has no such relay."})
if "status" in data:
if not type(data["status"]) == int:
return dumps({"success": False, "reason": "invalid type for enabled."})
enabled = data["status"]
if enabled:
net_inst.enable_relay(num)
else:
net_inst.disable_relay(num)
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/<net>/<num>/", methods=["PUT"])
@login_required
def irc_network_relay_add(self, request, net, num):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
net_inst = main.network[net]
if num in net_inst.relays:
return dumps(
{"success": False, "reason": "network already has this relay."}
)
id, alias = net_inst.add_relay(num)
main.saveConf("network")
return dumps({"success": True, "id": id, "alias": alias})
@app.route("/irc/network/<net>/", methods=["PUT"])
@login_required
def irc_network_relay_add_next(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
net_inst = main.network[net]
id, alias = net_inst.add_relay()
main.saveConf("network")
return dumps({"success": True, "id": id, "alias": alias})
@app.route("/irc/network/<net>/<num>/", methods=["DELETE"])
@login_required
def irc_network_relay_del(self, request, net, num):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
net_inst = main.network[net]
if num not in net_inst.relays:
return dumps(
{"success": False, "reason": "network does not have this relay."}
)
net_inst.delete_relay(num)
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/<net>/<num>/provision/", methods=["POST"])
@login_required
def irc_network_relay_provision(self, request, net, num):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
net_inst = main.network[net]
if num not in net_inst.relays:
return dumps(
{"success": False, "reason": "network does not have this relay."}
)
provision.provisionRelay(num, net)
return dumps({"success": True})
@app.route("/irc/network/<net>/channels/", methods=["POST"])
@login_required
def irc_network_channels(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
relays_inst = main.network[net].relays
channels = []
for num in relays_inst.keys():
name = f"{net}{num}"
if name in main.IRCPool.keys():
net_chans = main.IRCPool[name].channels
channels_annotated = userinfo.getUserNum(net, net_chans)
for channel in net_chans:
channel_inst = {
"name": channel,
"users": channels_annotated[channel],
"num": num,
}
channels.append(channel_inst)
# channels[channel] = channels_annotated[channel]
return dumps({"channels": channels})
# API version
@app.route("/irc/network/<net>/channel/", methods=["DELETE"])
@login_required
def irc_network_channel_part_api(self, request, net):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "channel" not in data:
return dumps({"success": False, "reason": "no channel specified."})
channel = data["channel"]
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
parted = chankeep.partSingle(net, channel)
if not parted:
dumps({"success": False, "reason": "no channels matched."})
return dumps({"success": True, "relays": parted})
# API version
@app.route("/irc/network/<net>/channel/", methods=["PUT"])
@login_required
def irc_network_channel_join_api(self, request, net):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "channel" not in data:
return dumps({"success": False, "reason": "no channel specified."})
channel = data["channel"]
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
joined = chankeep.joinSingle(net, channel)
if not joined:
return dumps(
{"success": False, "reason": "could not allocate channel to relay."}
)
return dumps({"success": True, "relays": joined})
@app.route("/irc/network/<net>/channel/<channel>/", methods=["PUT"])
@login_required
def irc_network_channel_join(self, request, net, channel):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
joined = chankeep.joinSingle(net, channel)
if not joined:
return dumps(
{"success": False, "reason": "could not allocate channel to relay."}
)
return dumps({"success": True, "relays": joined})
@app.route("/aliases/", methods=["GET"])
@login_required
def aliases(self, request):
alias_list = []
for num, alias in main.alias.items():
alias_dup = dict(alias)
alias_dup["num"] = num
alias_list.append(alias_dup)
return dumps({"aliases": alias_list})
@app.route("/aliases/", methods=["POST"])
@login_required
def aliases_update(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
for alias, fields in data.items():
if not alias.isdigit():
return dumps({"success": False, "reason": "alias not a number."})
alias = int(alias)
if alias not in main.alias.keys():
return dumps({"success": False, "reason": "alias does not exist."})
if fields:
for field in fields:
if field in main.alias[alias]:
main.alias[alias][field] = fields[field]
if "emails" in fields:
if not fields["emails"]:
main.alias[alias]["emails"] = []
main.saveConf("alias")
return dumps({"success": True})
@app.route("/irc/auto/<net>/", methods=["POST"])
@login_required
def irc_auto_network(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if 1 in main.network[net].relays.keys():
return dumps({"success": False, "reason": f"first relay exists on {net}"})
num, alias = main.network[net].add_relay(1)
provision.provisionRelay(num, net)
main.saveConf("network")
return dumps(
{
"success": True,
"message": f"created relay {num} with alias {alias} on {net}",
}
)
@app.route("/irc/list/<net>/", methods=["POST"])
@login_required
def irc_list_network(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
first_relay = helpers.get_first_relay(net)
if not first_relay:
return dumps(
{"success": False, "reason": f"could not get first relay for {net}"}
)
first_relay.list()
return dumps(
{
"success": True,
"message": f"requested list with instance {first_relay.num} of {net}",
}
)
@app.route("/irc/list/<net>/", methods=["GET"])
@login_required
def get_irc_list_info(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
listinfo = chankeep.getListInfo(net)
return dumps({"success": True, "listinfo": listinfo})
@app.route("/irc/msg/<net>/<num>/", methods=["PUT"])
@login_required
def irc_send_message(self, request, net, num):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if "msg" not in data:
return dumps({"success": False, "reason": "no msg."})
if "channel" not in data:
return dumps({"success": False, "reason": "no msg."})
msg = data["msg"]
channel = data["channel"]
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
if num not in main.network[net].relays.keys():
return dumps({"success": False, "reason": f"no relay {num} on {net}"})
name = f"{net}{num}"
if name not in main.IRCPool.keys():
return dumps({"success": False, "reason": f"relay {num} not on {net}"})
# We are in a query
in_query = False
if "nick" in data:
nick = data["nick"]
if nick == channel:
in_query = True
else:
nick = None
if channel == main.IRCPool[name].nickname or nick == channel:
in_query = True
if not nick:
return dumps({"success": False, "reason": "no nick specified to query"})
else:
main.IRCPool[name].sendmsg(nick, msg, in_query=in_query)
else:
main.IRCPool[name].sendmsg(channel, msg, in_query=in_query)
return dumps(
{"success": True, "message": f"sent message to {channel} on {name}"}
)
@app.route("/irc/nick/<net>/<num>/", methods=["GET"])
@login_required
def irc_get_nick(self, request, net, num):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
if num not in main.network[net].relays.keys():
return dumps({"success": False, "reason": f"no relay {num} on {net}"})
name = f"{net}{num}"
if name not in main.IRCPool.keys():
return dumps({"success": False, "reason": f"relay {num} not on {net}"})
return dumps({"nickname": main.IRCPool[name].nickname})
@app.route("/irc/reg/<net>/", methods=["GET"])
@login_required
def irc_get_unreg_net(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
unreg = regproc.get_unregistered_relays(net)
return dumps({"success": True, "unreg": unreg})
@app.route("/irc/reg/", methods=["GET"])
@login_required
def irc_get_unreg(self, request):
unreg = regproc.get_unregistered_relays()
return dumps({"success": True, "unreg": unreg})
@app.route("/irc/reg/", methods=["PUT"])
@login_required
def irc_confirm_accounts(self, request):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
for item, token in data.items():
if "|" not in item:
return dumps({"success": False, "reason": f"malformed item: {item}"})
spl = item.split("|")
if not len(spl) == 2:
return dumps({"success": False, "reason": f"malformed item: {item}"})
net, num = spl
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
if token:
regproc.confirmAccount(net, num, token)
return dumps({"success": True})
@app.route("/irc/network/<net>/<num>/auth/", methods=["POST"])
@login_required
def irc_enable_auth(self, request, net, num):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
if num not in main.network[net].relays.keys():
return dumps({"success": False, "reason": f"no relay {num} on {net}"})
regproc.enableAuthentication(net, num, jump=False, run_now=True)
return dumps({"success": True})
@app.route("/irc/sinst/<net>/", methods=["GET"])
@login_required
def irc_get_authentity(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
auth = regproc.selectInst(net)
if not auth:
return dumps({"success": False, "reason": "error getting results."})
return dumps({"success": True, "sinst": auth})

0
legacy/commands/__init__.py Normal file
View File

7
commands/admall.py → legacy/commands/admall.py
View File

@ -1,11 +1,14 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class AdmallCommand:
def __init__(self, *args):
self.admall(*args)
def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def admall(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length > 2:
for i in main.network.keys():

11
commands/alias.py → legacy/commands/alias.py
View File

@ -1,12 +1,15 @@
import main
from yaml import dump
from modules import alias
from yaml import dump
class AliasCommand:
def __init__(self, *args):
self.alias(*args)
def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def alias(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
info(dump(main.alias))
@ -21,7 +24,7 @@ class AliasCommand:
if len(main.alias.keys()) == 0:
nextNum = 1
else:
nextNum = max(main.alias.keys())+1
nextNum = max(main.alias.keys()) + 1
main.alias[nextNum] = alias.generate_alias()
success("Generated new alias: %i" % nextNum)
main.saveConf("alias")
@ -31,7 +34,7 @@ class AliasCommand:
failure("Must be a number, not %s" % spl[2])
return
num = int(spl[2])
if not num in main.alias.keys():
if num not in main.alias.keys():
failure("No such alias: %i" % num)
return
failed = False

14
commands/all.py → legacy/commands/all.py
View File

@ -1,11 +1,14 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class AllCommand:
def __init__(self, *args):
self.all(*args)
def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def all(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length > 2:
for i in main.network.keys():
@ -14,8 +17,11 @@ class AllCommand:
net = main.network[i].relays[x]["net"]
alias = main.alias[x]["nick"]
commands = {spl[1]: [" ".join(spl[2:])]}
success("Sending commands to relay %s as user %s" % (num, alias+"/"+net))
deliverRelayCommands(num, commands, user=alias+"/"+net)
success(
"Sending commands to relay %s as user %s"
% (num, alias + "/" + net)
)
deliverRelayCommands(num, commands, user=alias + "/" + net)
return
else:
incUsage("all")

23
commands/allc.py → legacy/commands/allc.py
View File

@ -1,11 +1,14 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class AllcCommand:
def __init__(self, *args):
self.allc(*args)
def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def allc(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length > 4:
targets = []
@ -16,9 +19,12 @@ class AllcCommand:
targets.append((i, x))
elif spl[1] == "alias":
for i in main.network.keys():
[targets.append((i, x)) for x in main.alias.keys() if
main.alias[x]["nick"] == spl[2] and
x in main.network[i].aliases.keys()]
[
targets.append((i, x))
for x in main.alias.keys()
if main.alias[x]["nick"] == spl[2]
and x in main.network[i].aliases.keys()
]
else:
incUsage("allc")
return
@ -30,8 +36,11 @@ class AllcCommand:
num = i[1]
alias = main.alias[num]["nick"]
commands = {spl[3]: [" ".join(spl[4:])]}
success("Sending commands to relay %i as user %s" % (num, alias+"/"+net))
deliverRelayCommands(num, commands, user=alias+"/"+net)
success(
"Sending commands to relay %i as user %s"
% (num, alias + "/" + net)
)
deliverRelayCommands(num, commands, user=alias + "/" + net)
return
else:
incUsage("allc")

43
legacy/commands/authcheck.py Normal file
View File

@ -0,0 +1,43 @@
import main
class AuthcheckCommand:
def __init__(self, *args):
self.authcheck(*args)
def authcheck(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
results = []
for i in main.IRCPool.keys():
num = main.IRCPool[i].num
net = main.IRCPool[i].net
if not main.IRCPool[i].authenticated:
results.append(
"%s - %s: %s" % (net, num, main.alias[num]["nick"])
)
info("\n".join(results))
return
elif length == 2:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
results = []
for i in main.IRCPool.keys():
num = main.IRCPool[i].num
net = main.IRCPool[i].net
if not net == spl[1]:
continue
if not main.IRCPool[i].authenticated:
results.append(
"%s - %s: %s" % (net, num, main.alias[num]["nick"])
)
info("\n".join(results))
return
else:
incUsage("authcheck")
return
else:
incUsage(None)

25
commands/auto.py → legacy/commands/auto.py
View File

@ -1,11 +1,14 @@
import main
from modules import provision
class AutoCommand:
def __init__(self, *args):
self.auto(*args)
def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def auto(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
for i in main.network.keys():
@ -13,9 +16,15 @@ class AutoCommand:
info("Skipping %s - first relay exists" % i)
else:
num, alias = main.network[i].add_relay(1)
success("Successfully created first relay on network %s with alias %s" % (i, alias))
success(
"Successfully created first relay on network %s with alias %s"
% (i, alias)
)
provision.provisionRelay(num, i)
success("Started provisioning network %s on first relay for alias %s" % (i, alias))
success(
"Started provisioning network %s on first relay for alias %s"
% (i, alias)
)
main.saveConf("network")
return
elif length == 2:
@ -26,9 +35,15 @@ class AutoCommand:
failure("First relay exists on %s" % spl[1])
return
num, alias = main.network[spl[1]].add_relay(1)
success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias))
success(
"Successfully created relay %i on network %s with alias %s"
% (num, spl[1], alias)
)
provision.provisionRelay(num, spl[1])
success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias))
success(
"Started provisioning network %s on relay %s for alias %s"
% (spl[1], num, alias)
)
main.saveConf("network")
return
else:

44
legacy/commands/blacklist.py Normal file
View File

@ -0,0 +1,44 @@
import main
from yaml import dump
class BlacklistCommand:
def __init__(self, *args):
self.blacklist(*args)
def blacklist(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
info(dump(main.blacklist))
return
elif length == 4:
if spl[1] == "add":
if spl[2] in main.blacklist.keys():
main.blacklist[spl[2]].append(spl[3])
else:
main.blacklist[spl[2]] = [spl[3]]
success("Blacklisted %s on %s" % (spl[3], spl[2]))
main.saveConf("blacklist")
return
elif spl[1] == "del":
if spl[2] in main.blacklist.keys():
if spl[3] in main.blacklist[spl[2]]:
main.blacklist[spl[2]].remove(spl[3])
if len(main.blacklist[spl[2]]) == 0:
del main.blacklist[spl[2]]
else:
failure("Not in list: %s" % spl[3])
return
else:
failure("No such entry: %s" % spl[2])
return
success("Removed blacklist %s on %s" % (spl[3], spl[2]))
main.saveConf("blacklist")
return
else:
incUsage("blacklist")
return
else:
incUsage(None)

8
commands/chans.py → legacy/commands/chans.py
View File

@ -1,11 +1,13 @@
import main
import modules.userinfo as userinfo
class ChansCommand:
def __init__(self, *args):
self.chans(*args)
def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def chans(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if len(spl) < 2:
incUsage("chans")
@ -16,7 +18,7 @@ class ChansCommand:
rtrn += "Matches from: %s" % i
rtrn += "\n"
for x in result[i]:
rtrn += (x)
rtrn += x
rtrn += "\n"
rtrn += "\n"
info(rtrn)

8
commands/cmd.py → legacy/commands/cmd.py
View File

@ -1,11 +1,13 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class CmdCommand:
def __init__(self, *args):
self.cmd(*args)
def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def cmd(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length > 4:
if not spl[1].isdigit():

33
legacy/commands/confirm.py Normal file
View File

@ -0,0 +1,33 @@
import main
from modules import regproc
class ConfirmCommand:
def __init__(self, *args):
self.confirm(*args)
def confirm(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 4:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
if not spl[2].isdigit():
failure("Must be a number, not %s" % spl[2])
return
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("No such relay on %s: %s" % (spl[1], spl[2]))
return
regproc.confirmAccount(spl[1], int(spl[2]), spl[3])
success(
"Requested confirmation on %s - %s with token %s"
% (spl[1], spl[2], spl[3])
)
return
else:
incUsage("confirm")
return
else:
incUsage(None)

27
legacy/commands/dedup.py Normal file
View File

@ -0,0 +1,27 @@
from json import dumps
import main
from modules import chankeep
class DedupCommand:
def __init__(self, *args):
self.dedup(*args)
def dedup(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
dupes = chankeep.getDuplicateChannels()
chankeep.partChannels(dupes)
info(dumps(dupes))
return
elif length == 2:
if spl[1] not in main.network.keys():
failure("No such network: %s" % spl[1])
return
dupes = chankeep.getDuplicateChannels(spl[1])
chankeep.partChannels(dupes)
info(dumps(dupes))
return

15
commands/disable.py → legacy/commands/disable.py
View File

@ -1,11 +1,14 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class DisableCommand:
def __init__(self, *args):
self.disable(*args)
def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def disable(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 3:
if not spl[1] in main.network.keys():
@ -15,18 +18,18 @@ class DisableCommand:
failure("Must be a number, not %s" % spl[2])
return
relayNum = int(spl[2])
name = spl[1]+spl[2]
name = spl[1] + spl[2]
if not spl[1] in main.IRCPool.keys():
info("Note - instance not running, proceeding anyway")
if not relayNum in main.network[spl[1]].relays.keys():
if relayNum not in main.network[spl[1]].relays.keys():
failure("No such relay: %s in network %s" % (spl[2], spl[1]))
return
main.network[spl[1]].relays[relayNum]["enabled"] = False
user = main.alias[relayNum]["nick"]
network = spl[1]
relay = main.network[spl[1]].relays[relayNum]
# relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]}
deliverRelayCommands(relayNum, commands, user=user+"/"+network)
deliverRelayCommands(relayNum, commands, user=user + "/" + network)
main.saveConf("network")
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():

8
commands/dist.py → legacy/commands/dist.py
View File

@ -1,11 +1,15 @@
from subprocess import PIPE, run
import main
from subprocess import run, PIPE
class DistCommand:
def __init__(self, *args):
self.dist(*args)
def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def dist(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if main.config["Dist"]["Enabled"]:
rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE)

119
legacy/commands/email.py Normal file
View File

@ -0,0 +1,119 @@
import main
from yaml import dump
class EmailCommand:
def __init__(self, *args):
self.email(*args)
def email(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 4:
if spl[1] == "add":
if spl[2].isdigit():
num = int(spl[2])
if num not in main.alias.keys():
failure("No such alias: %i" % num)
return
if not spl[3] in main.alias[num]["emails"]:
main.alias[num]["emails"].append(spl[3])
main.saveConf("alias")
success(
"Successfully added email %s to alias %i"
% (spl[3], num)
)
return
else:
failure(
"Email already exists in alias %i: %s" % (num, spl[3])
)
return
else:
failure("Must be a number, not %s" % spl[2])
if spl[2] == "domain":
domain = spl[3]
if "@" in domain:
failure("Not a domain: %s" % domain)
return
if domain not in main.irc["_"]["domains"]:
main.irc["_"]["domains"].append(domain)
success(
"Successfully added domain %s to default config"
% domain
)
main.saveConf("irc")
else:
failure(
"Domain already exists in default config: %s"
% domain
)
return
elif spl[1] == "del":
if not spl[2].isdigit():
# failure("Must be a number, not %s" % spl[2])
if spl[2] == "domain":
domain = spl[3]
if domain in main.irc["_"]["domains"]:
main.irc["_"]["domains"].remove(domain)
success(
"Successfully removed domain %s to default config"
% domain
)
main.saveConf("irc")
else:
failure(
"Domain does not exist in default config: %s"
% domain
)
return
elif spl[1] == "del":
if num not in main.alias.keys():
failure("No such alias: %i" % num)
return
if spl[3] in main.alias[num]["emails"]:
main.alias[num]["emails"].remove(spl[3])
main.saveConf("alias")
success(
"Successfully removed email %s from alias %i"
% (spl[3], num)
)
return
else:
failure(
"Email does not exist in alias %i: %s" % (spl[3], num)
)
return
elif length == 2:
if spl[1] == "list":
info(dump(main.alias))
return
else:
incUsage("email")
return
elif length == 3:
if spl[1] == "list":
if spl[2] == "domain":
filtered = {
f"{k}:{k2}": v2
for k, v in main.irc.items()
for k2, v2 in v.items()
if k2 == "domains"
}
info(dump(filtered))
return
else:
incUsage("email")
return
else:
incUsage("email")
return
else:
incUsage("email")
return
else:
incUsage(None)

11
commands/enable.py → legacy/commands/enable.py
View File

@ -1,11 +1,14 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class EnableCommand:
def __init__(self, *args):
self.enable(*args)
def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def enable(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 3:
if not spl[1] in main.network.keys():
@ -22,9 +25,9 @@ class EnableCommand:
user = main.alias[int(spl[2])]["nick"]
network = spl[1]
commands = {"status": ["Connect"]}
deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network)
deliverRelayCommands(int(spl[2]), commands, user=user + "/" + network)
main.saveConf("network")
if not spl[1]+spl[2] in main.IRCPool.keys():
if not spl[1] + spl[2] in main.IRCPool.keys():
main.network[spl[1]].start_bot(int(spl[2]))
else:
pass

6
commands/exec.py → legacy/commands/exec.py
View File

@ -1,10 +1,10 @@
import main
class ExecCommand:
def __init__(self, *args):
self.exec(*args)
def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def exec(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length > 1:
try:

35
legacy/commands/getstr.py Normal file
View File

@ -0,0 +1,35 @@
import main
from utils.get import getRelay
class GetstrCommand:
def __init__(self, *args):
self.getstr(*args)
def getstr(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 3:
net = spl[1]
num = spl[2]
if net not in main.network.keys():
failure("Network does not exist: %s" % net)
return
if not num.isdigit():
failure("Must be a number, not %s" % num)
return
num = int(num)
alias = main.alias[num]["nick"].lower()
host, port = getRelay(num)
password = main.config["Relay"]["Password"]
connstr = f"/connect -ssl {host} {port}"
authstr = f"/quote PASS {alias}/{net}:{password}"
obj.send(connstr)
obj.send(authstr)
else:
incUsage("getstr")
return
else:
incUsage(None)

5
commands/help.py → legacy/commands/help.py
View File

@ -1,10 +1,13 @@
import main
class HelpCommand:
def __init__(self, *args):
self.help(*args)
def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def help(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
helpMap = []
for i in main.help.keys():

23
commands/join.py → legacy/commands/join.py
View File

@ -1,22 +1,33 @@
import main
import modules.chankeep
class JoinCommand:
def __init__(self, *args):
self.join(*args)
def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def join(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 4:
if length == 3:
if not spl[1] in main.network.keys():
failure("Network does not exist: %s" % spl[1])
return
modules.chankeep.joinSingle(spl[1], spl[2])
success("Joined %s" % spl[2])
elif length == 4:
if not spl[1] in main.network.keys():
failure("Network does not exist: %s" % spl[1])
return
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
return
if not spl[1]+spl[2] in main.IRCPool.keys():
if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1])
return
main.IRCPool[spl[1]+spl[2]].join(spl[3])
main.IRCPool[spl[1] + spl[2]].join(spl[3])
success("Joined %s" % spl[3])
return
elif length == 5:
@ -26,10 +37,10 @@ class JoinCommand:
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %", (spl[2], spl[1]))
return
if not spl[1]+spl[2] in main.IRCPool.keys():
if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1])
return
main.IRCPool[spl[1]+spl[2]].join(spl[3], spl[4])
main.IRCPool[spl[1] + spl[2]].join(spl[3], spl[4])
success("Joined %s with key %s" % (spl[3], spl[4]))
return
else:

38
legacy/commands/list.py Normal file
View File

@ -0,0 +1,38 @@
import main
from modules import helpers
class ListCommand:
def __init__(self, *args):
self.list(*args)
def list(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
for i in main.network.keys():
first_relay = helpers.get_first_relay(i)
####
if not first_relay:
info("Network has no first instance: %s" % i)
continue
first_relay.list()
success(f"Requested list with instance {first_relay.num} of {i}")
return
elif length == 2:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
first_relay = helpers.get_first_relay(spl[1])
if not first_relay:
failure("Could not get first instance")
return
first_relay.list()
success(f"Requested list with instance {first_relay.num} of {spl[1]}")
return
else:
incUsage("list")
return
else:
incUsage(None)

5
commands/load.py → legacy/commands/load.py
View File

@ -1,10 +1,13 @@
import main
class LoadCommand:
def __init__(self, *args):
self.load(*args)
def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def load(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
if spl[1] in main.filemap.keys():

8
commands/loadmod.py → legacy/commands/loadmod.py
View File

@ -1,15 +1,17 @@
import main
from utils.loaders.single_loader import loadSingle
class LoadmodCommand:
def __init__(self, *args):
self.loadmod(*args)
def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def loadmod(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
rtrn = loadSingle(spl[1])
if rtrn == True:
if rtrn is True:
success("Loaded module: %s" % spl[1])
return
elif rtrn == "RELOAD":

8
commands/logout.py → legacy/commands/logout.py
View File

@ -1,14 +1,12 @@
import main
class LogoutCommand:
def __init__(self, *args):
self.logout(*args)
def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def logout(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
obj.authed = False
if obj.addr in main.MonitorPool:
main.MonitorPool.remove(obj.addr)
success("Logged out")
return
else:

16
commands/mod.py → legacy/commands/mod.py
View File

@ -1,12 +1,14 @@
import main
from yaml import dump
class ModCommand:
# This could be greatly improved, but not really important right now
def __init__(self, *args):
self.mod(*args)
def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def mod(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 4:
if not spl[1] in main.network.keys():
@ -15,15 +17,17 @@ class ModCommand:
try:
setattr(main.network[spl[1]], spl[2], spl[3])
except e:
failure("Something went wrong.")
except Exception as e:
failure(f"Something went wrong: {e}")
return
main.saveConf("network")
success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]))
success(
"Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])
)
return
# Find a better way to do this
#elif length == 6:
# elif length == 6:
# if not spl[1] in main.network.keys():
# failure("Network does not exist: %s" % spl[1])
# return

18
commands/msg.py → legacy/commands/msg.py
View File

@ -1,25 +1,31 @@
import main
class MsgCommand:
def __init__(self, *args):
self.msg(*args)
def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def msg(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length >= 5:
if not spl[1] in main.network.keys():
failure("Network does not exist: %s" % spl[1])
return
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %" % (spl[2], spl[1]))
failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
return
if not spl[1]+spl[2] in main.IRCPool.keys():
if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1])
return
if not spl[3] in main.IRCPool[spl[1]+spl[2]].channels:
if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels:
info("Bot not on channel: %s" % spl[3])
main.IRCPool[spl[1]+spl[2]].msg(spl[3], " ".join(spl[4:]))
success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1]))
main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:]))
success(
"Sent %s to %s on relay %s on network %s"
% (" ".join(spl[4:]), spl[3], spl[2], spl[1])
)
return
else:
incUsage("msg")

18
commands/network.py → legacy/commands/network.py
View File

@ -1,13 +1,17 @@
import main
from yaml import dump
from modules.network import Network
from string import digits
import main
from modules.network import Network
from yaml import dump
class NetworkCommand:
def __init__(self, *args):
self.network(*args)
def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def network(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 7:
if spl[1] == "add":
@ -27,7 +31,9 @@ class NetworkCommand:
failure("Auth must be sasl, ns or none, not %s" % spl[5])
return
else:
main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower())
main.network[spl[2]] = Network(
spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()
)
success("Successfully created network: %s" % spl[2])
main.saveConf("network")
return
@ -38,7 +44,7 @@ class NetworkCommand:
elif length == 3:
if spl[1] == "del":
if spl[2] in main.network.keys():
main.network[spl[2]].seppuku() # ;(
main.network[spl[2]].seppuku() # ;(
del main.network[spl[2]]
success("Successfully removed network: %s" % spl[2])
main.saveConf("network")

11
commands/part.py → legacy/commands/part.py
View File

@ -1,10 +1,13 @@
import main
class PartCommand:
def __init__(self, *args):
self.part(*args)
def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def part(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 4:
if not spl[1] in main.network.keys():
@ -13,10 +16,10 @@ class PartCommand:
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %", (spl[2], spl[1]))
return
if not spl[1]+spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1]+spl[2])
if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1] + spl[2])
return
main.IRCPool[spl[1]+spl[2]].part(spl[3])
main.IRCPool[spl[1] + spl[2]].part(spl[3])
success("Left %s" % spl[3])
return
else:

5
commands/pass.py → legacy/commands/pass.py
View File

@ -1,10 +1,13 @@
import main
class PassCommand:
def __init__(self, *args):
self.password(*args)
def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def password(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
info("You are already authenticated")
return

40
legacy/commands/pending.py Normal file
View File

@ -0,0 +1,40 @@
import main
class PendingCommand:
def __init__(self, *args):
self.pending(*args)
def pending(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
results = []
for i in main.network.keys():
for x in main.network[i].relays.keys():
if not main.network[i].relays[x]["registered"]:
results.append(
"%s: confirm %s %s [code]"
% (main.alias[x]["nick"], i, x)
)
info("\n".join(results))
return
elif length == 2:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
results = []
for x in main.network[spl[1]].relays.keys():
if not main.network[spl[1]].relays[x]["registered"]:
results.append(
"%s: confirm %s %s [code]"
% (main.alias[x]["nick"], spl[1], x)
)
info("\n".join(results))
return
else:
incUsage("pending")
return
else:
incUsage(None)

36
legacy/commands/recheckauth.py Normal file
View File

@ -0,0 +1,36 @@
import main
class RecheckauthCommand:
def __init__(self, *args):
self.recheckauth(*args)
def recheckauth(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
for i in main.IRCPool.keys():
net = main.IRCPool[i].net
main.IRCPool[i].authenticated = False
main.IRCPool[i].regPing()
success("Successfully reset authentication status")
return
elif length == 2:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
for i in main.IRCPool.keys():
# num = main.IRCPool[i].num
net = main.IRCPool[i].net
if not net == spl[1]:
continue
main.IRCPool[i].authenticated = False
main.IRCPool[i].regPing()
success("Successfully reset authentication status on %s" % spl[1])
return
else:
incUsage("recheckauth")
return
else:
incUsage(None)

38
legacy/commands/reg.py Normal file
View File

@ -0,0 +1,38 @@
import main
from modules import regproc
class RegCommand:
def __init__(self, *args):
self.reg(*args)
def reg(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
for i in main.network[spl[1]].relays.keys():
regproc.registerAccount(spl[1], i)
success("Requested registration for all relays on %s" % spl[1])
return
elif length == 3:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
if not spl[2].isdigit():
failure("Must be a number, not %s" % spl[2])
return
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("No such relay on %s: %s" % (spl[2], spl[1]))
return
regproc.registerAccount(spl[1], int(spl[2]))
success("Requested registration on %s - %s" % (spl[1], spl[2]))
return
else:
incUsage("reg")
return
else:
incUsage(None)

19
commands/relay.py → legacy/commands/relay.py
View File

@ -1,17 +1,23 @@
import main
from yaml import dump
class RelayCommand:
def __init__(self, *args):
self.relay(*args)
def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def relay(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 3:
if spl[1] == "add":
if spl[2] in main.network.keys():
id, alias = main.network[spl[2]].add_relay()
success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
success(
"Successfully created relay %s on network %s with alias %s"
% (str(id), spl[2], alias)
)
main.saveConf("network")
return
else:
@ -34,7 +40,10 @@ class RelayCommand:
failure("Must be a number, not %s" % spl[3])
return
id, alias = main.network[spl[2]].add_relay(int(spl[3]))
success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
success(
"Successfully created relay %s on network %s with alias %s"
% (str(id), spl[2], alias)
)
main.saveConf("network")
return
else:
@ -51,7 +60,9 @@ class RelayCommand:
failure("No such relay: %s on network %s" % (spl[3], spl[2]))
return
main.network[spl[2]].delete_relay(int(spl[3]))
success("Successfully deleted relay %s on network %s" % (spl[3], spl[2]))
success(
"Successfully deleted relay %s on network %s" % (spl[3], spl[2])
)
main.saveConf("network")
return
else:

5
commands/save.py → legacy/commands/save.py
View File

@ -1,10 +1,13 @@
import main
class SaveCommand:
def __init__(self, *args):
self.save(*args)
def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def save(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
if spl[1] in main.filemap.keys():

14
commands/stats.py → legacy/commands/stats.py
View File

@ -1,13 +1,17 @@
from string import digits
import main
import modules.counters as count
import modules.userinfo as userinfo
from string import digits
class StatsCommand:
def __init__(self, *args):
self.stats(*args)
def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def stats(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 1:
stats = []
@ -25,7 +29,7 @@ class StatsCommand:
stats.append("User records: %s" % numWhoEntries)
stats.append("Events/min: %s" % main.lastMinuteSample)
counterEvents = count.getEvents()
if counterEvents == None:
if counterEvents is None:
stats.append("No counters records")
else:
stats.append("Counters:")
@ -42,7 +46,7 @@ class StatsCommand:
numNodes = 0
for i in main.IRCPool.keys():
if "".join([x for x in i if not x in digits]) == spl[1]:
if "".join([x for x in i if x not in digits]) == spl[1]:
numChannels += len(main.IRCPool[i].channels)
found = True
numNodes += 1
@ -53,7 +57,7 @@ class StatsCommand:
stats.append("User records: %s" % numWhoEntries)
stats.append("Endpoints: %s" % numNodes)
counterEvents = count.getEvents(spl[1])
if counterEvents == None:
if counterEvents is None:
stats.append("No counters records")
else:
stats.append("Counters:")

5
commands/swho.py → legacy/commands/swho.py
View File

@ -1,10 +1,13 @@
import main
class SwhoCommand:
def __init__(self, *args):
self.swho(*args)
def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def swho(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
if not spl[1] in main.network.keys():

19
commands/token.py → legacy/commands/token.py
View File

@ -1,12 +1,16 @@
from uuid import uuid4
import main
from yaml import dump
from uuid import uuid4
class TokenCommand:
def __init__(self, *args):
self.token(*args)
def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def token(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
if spl[1] == "list":
@ -31,11 +35,12 @@ class TokenCommand:
elif length == 4:
if spl[1] == "add":
if not spl[2] in main.tokens.keys():
if spl[3] in ["relay"]: # more to come!
main.tokens[spl[2]] = {"hello": str(uuid4()),
"usage": spl[3],
"counter": str(uuid4()),
}
if spl[3] in ["relay", "api"]: # more to come!
main.tokens[spl[2]] = {
"hello": str(uuid4()),
"usage": spl[3],
"counter": str(uuid4()),
}
main.saveConf("tokens")
success("Successfully created token %s:" % spl[2])
info(dump(main.tokens[spl[2]]))

8
commands/users.py → legacy/commands/users.py
View File

@ -1,11 +1,13 @@
import main
import modules.userinfo as userinfo
class UsersCommand:
def __init__(self, *args):
self.users(*args)
def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def users(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if len(spl) < 2:
incUsage("users")
@ -16,7 +18,7 @@ class UsersCommand:
rtrn += "Matches from: %s" % i
rtrn += "\n"
for x in result[i]:
rtrn += (x)
rtrn += x
rtrn += "\n"
rtrn += "\n"
info(rtrn)

8
commands/who.py → legacy/commands/who.py
View File

@ -1,11 +1,13 @@
import main
import modules.userinfo as userinfo
class WhoCommand:
def __init__(self, *args):
self.who(*args)
def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
def who(
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
):
if authed:
if length == 2:
result = userinfo.getWho(spl[1])
@ -14,7 +16,7 @@ class WhoCommand:
rtrn += "Matches from: %s" % i
rtrn += "\n"
for x in result[i]:
rtrn += (x)
rtrn += x
rtrn += "\n"
rtrn += "\n"
info(rtrn)

0
conf/aliasdata.json → legacy/conf/templates/aliasdata.json
View File

30
conf/example/config.json → legacy/conf/templates/config.json
View File

@ -10,19 +10,34 @@
"Address": "127.0.0.1",
"UseSSL": true
},
"API": {
"Enabled": true,
"Port": 13869,
"Address": "127.0.0.1"
},
"Key": "key.pem",
"Certificate": "cert.pem",
"RedisSocket": "/tmp/redis.sock",
"UsePassword": true,
"RedisSocket": "/var/run/redis/redis.sock",
"RedisDBEphemeral": 1,
"RedisDBPersistent": 0,
"UsePassword": false,
"ConnectOnCreate": false,
"AutoReg": false,
"Debug": false,
"Trace": false,
"Relay": {
"Host": "127.0.0.1",
"Port": "201x",
"User": "sir",
"Password": "sir"
"Port": "2001",
"User": "x",
"Password": "x"
},
"Ingest": {
"Key": "queue.irc",
"Enabled": true
},
"ChanKeep": {
"Enabled": false,
"Provision": false,
"MaxRelay": 30,
"SigSwitch": 20
},
@ -32,10 +47,10 @@
"File": "conf/dist.sh"
},
"Toggles": {
"Who": false,
"Who": true,
"CycleChans": true
},
"Password": "s",
"Password": "x",
"Tweaks": {
"MaxHash": 10,
"DedupPrecision": 9,
@ -43,6 +58,7 @@
"Prefix": "*"
},
"Delays": {
"WhoDelay": 3600,
"WhoLoop": 600,
"WhoRange": 1800,
"Timeout": 30,

18
conf/help.json → legacy/conf/templates/help.json
View File

@ -7,14 +7,12 @@
"part": "part <name> <num> <channel>",
"enable": "enable <name> <num>",
"disable": "disable <name> <num>",
"list": "list",
"stats": "stats [<name>]",
"save": "save <(file)|list|all>",
"load": "load <(file)|list|all>",
"dist": "dist",
"loadmod": "loadmod <module>",
"msg": "msg <name> <target> <message...>",
"mon": "mon -h",
"msg": "msg <network> <num> <target> <message...>",
"chans": "chans <nick> [<nick> ...]",
"users": "users <channel> [<channel> ...]",
"relay": "relay <add|del|list> [<network>] [<num>]",
@ -22,11 +20,19 @@
"alias": "alias [<add|del>] [<num>]",
"auto": "auto [<network>]",
"cmd": "cmd <relay> <user> <entity> <text ...>",
"token": "token <add|del|list> [<key>] [<relay>]",
"token": "token <add|del|list> [<key>] [relay|api]",
"all": "all <entity> <text ...>",
"allc": "allc <network|alias> <(network)|(alias)> <entity> <text ...>",
"admall": "admall <entity> <text ...>",
"swho": "swho <network> [<channel>]",
"list": "list <network>",
"exec": "exec <expr ...>"
"list": "list [<network>]",
"exec": "exec <expr ...>",
"reg": "reg <network> [<num>]",
"confirm": "confirm <network> <num> <token>",
"pending": "pending [<network>]",
"authcheck": "authcheck [<network>]",
"recheckauth": "recheckauth [<network>]",
"blacklist": "blacklist <network> <channel>",
"email": "email <add|del|list> [(domain)|<num>] [<email>]",
"getstr": "getstr <net> <num>"
}

34
legacy/conf/templates/irc.json Normal file
View File

@ -0,0 +1,34 @@
{
"_": {
"register": true,
"entity": "NickServ",
"domains": [],
"registermsg": "REGISTER {password} {email}",
"identifymsg": "IDENTIFY {password}",
"confirm": "CONFIRM {token}",
"check": true,
"ping": true,
"negative": true,
"pingmsg": "STATUS",
"negativemsg": "INFO {nickname}",
"checktype": "mode",
"checkmode": "R",
"checkmsg": "Password accepted - you are now recognized.",
"checkmsg2": "You are logged in as",
"checknegativemsg": "has \u0002NOT COMPLETED\u0002 registration verification",
"checkendnegative": "End of Info"
},
"freenode": {
"confirm": "VERIFY REGISTER {nickname} {token}"
},
"libera": {
"confirm": "VERIFY REGISTER {nickname} {token}"
},
"cyberia": {
"setmode": "I"
},
"ircnow": {
"negative": false,
"ping": false
}
}

946
legacy/core/bot.py Normal file
View File

@ -0,0 +1,946 @@
import sys
from copy import deepcopy
from datetime import datetime
from random import randint
import main
from core.relay import sendRelayNotification
from modules import chankeep, counters, helpers, monitor, regproc, userinfo
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.task import LoopingCall
from twisted.words.protocols.irc import (
IRCBadMessage,
IRCClient,
lowDequote,
numeric_to_symbolic,
)
from utils.dedup import dedup
from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn
from utils.logging.send import sendAll
from utils.parsing import parsen
# Copied from the Twisted source so we can fix a bug
def parsemsg(s):
"""
Breaks a message from an IRC server into its prefix, command, and
arguments.
@param s: The message to break.
@type s: L{bytes}
@return: A tuple of (prefix, command, args).
@rtype: L{tuple}
"""
prefix = ""
trailing = []
if not s:
raise IRCBadMessage("Empty line.")
if s[0:1] == ":":
prefix, s = s[1:].split(" ", 1)
if s.find(" :") != -1:
s, trailing = s.split(" :", 1)
args = s.split(" ") # Twisted bug fixed by adding an argument to split()
args.append(trailing)
else:
args = s.split(" ") # And again
command = args.pop(0)
return prefix, command, args
class IRCBot(IRCClient):
def __init__(self, net, num):
self.isconnected = False
self.channels = []
self.net = net
self.authenticated = not regproc.needToAuth(self.net)
self.num = num
self.buffer = ""
self.name = net + str(num)
alias = main.alias[num]
relay = main.network[self.net].relays[num]
self.netinst = main.network[self.net]
self.nickname = alias["nick"]
self.realname = alias["realname"]
self.username = alias["nick"].lower() + "/" + relay["net"]
self.password = main.config["Relay"]["Password"]
self.userinfo = None #
self.fingerReply = None #
self.versionName = None # Don't give out information
self.versionNum = None #
self.versionEnv = None #
self.sourceURL = None #
self._getWho = {} # LoopingCall objects -- needed to be able to stop them
self._tempWho = {} # temporary storage for gathering WHO info
self._tempNames = {} # temporary storage for gathering NAMES info
self._tempList = ([], []) # temporary storage for gathering LIST info
self.listOngoing = False # we are currently receiving a LIST
self.listRetried = False # we asked and got nothing so asked again
self.listAttempted = False # we asked for a list
self.listSimple = False # after asking again we got the list, so use the simple
# syntax from now on
self.wantList = (
False # we want to send a LIST, but not all relays are active yet
)
self.chanlimit = 0
self.prefix = {}
self.servername = None
self._regAttempt = None
self._negativePass = None
def lineReceived(self, line):
if bytes != str and isinstance(line, bytes):
# decode bytes from transport to unicode
line = line.decode("utf-8", "replace")
line = lowDequote(line)
try:
prefix, command, params = parsemsg(line)
if command in numeric_to_symbolic:
command = numeric_to_symbolic[command]
try:
self.handleCommand(command, prefix, params)
except Exception as err:
error(err)
except IRCBadMessage:
self.badMessage(line, *sys.exc_info())
def joinChannels(self, channels):
sleeptime = 0.0
increment = 0.8
for i in channels:
if i not in self.channels:
if self.net in main.blacklist.keys():
if i in main.blacklist[self.net]:
debug(
"Not joining blacklisted channel %s on %s - %i"
% (i, self.net, self.num)
)
continue
trace(
self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds"
)
reactor.callLater(sleeptime, self.join, i)
sleeptime += increment
if sleeptime == 10:
sleeptime = 0.0
increment = 0.7
increment += 0.1
else:
error(
"%s - Cannot join channel we are already on: %s - %i"
% (i, self.net, self.num)
)
def checkChannels(self):
if chankeep.allRelaysActive(self.net):
debug(f"checkChannels() all relays active for {self.net}")
else:
debug(
"checkChannels() skipping channel check as we have inactive relays: %s - %i"
% (self.net, self.num)
)
return
if self.net in main.TempChan.keys():
if self.num in main.TempChan[self.net].keys():
self.joinChannels(main.TempChan[self.net][self.num])
del main.TempChan[self.net][self.num]
if not main.TempChan[self.net]:
del main.TempChan[self.net]
def event(self, **cast):
if "ts" not in cast.keys():
cast["ts"] = int(datetime.now().timestamp())
# remove odd stuff
for i in list(
cast.keys()
): # Make a copy of the .keys() as Python 3 cannot handle iterating over
if cast[i] == "": # a dictionary that changes length with each iteration
del cast[i]
# remove server stuff
if "muser" in cast.keys():
if cast["muser"] == self.servername:
cast["type"] = "conn"
if "channel" in cast.keys():
if cast["channel"] == "*":
cast["type"] = "conn"
##
# expand out the hostmask
if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
if "muser" in cast.keys():
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
# handle ZNC stuff
if {"nick", "ident", "host", "msg"}.issubset(set(cast)):
if cast["ident"] == "znc" and cast["host"] == "znc.in":
cast["type"] = "znc"
cast["num"] = self.num
del cast["nick"]
del cast["ident"]
del cast["host"]
del cast["channel"]
del cast["muser"]
if "Disconnected from IRC" in cast["msg"]:
log("ZNC disconnected on %s - %i" % (self.net, self.num))
self.isconnected = False
self.authenticated = False
if "Connected!" in cast["msg"]:
log("ZNC connected on %s - %i" % (self.net, self.num))
self.isconnected = True
if "could not be joined, disabling it" in cast["msg"]:
error(cast["msg"])
#
# don't reprocess the same message twice
# if the type is in that list, it's already been here, don't run it again
if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
cast["num"] = self.num
if "channel" in cast.keys():
if cast["type"] == "mode":
if cast["channel"].lower() == self.nickname.lower():
# castDup = deepcopy(cast)
cast["mtype"] = cast["type"]
cast["type"] = "self"
# self.event(**castDup)
if cast["modearg"]: # check if modearg is non-NoneType
if self.nickname.lower() == cast["modearg"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "highlight"
self.event(**castDup)
else:
if cast["channel"].lower() == self.nickname.lower() or not cast[
"channel"
].startswith("#"):
cast["mtype"] = cast["type"]
cast["type"] = "query"
# self.event(**castDup)
# Don't call self.event for this one because queries are not events on a
# channel, but we still want to see them
if cast["channel"] == "AUTH":
cast["type"] = "conn"
cast["mtype"] = cast["type"]
# TODO: better way to do this
# as we changed the types above, check again
if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
# we have been kicked
if "user" in cast.keys():
if cast["user"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
if self.net in main.blacklist.keys():
if cast["channel"] not in main.blacklist[self.net]:
main.blacklist[self.net].append(cast["channel"])
else:
main.blacklist[self.net] = [cast["channel"]]
main.saveConf("blacklist")
self.event(**castDup)
# we sent a message/left/joined/kick someone/quit
if "nick" in cast.keys():
if cast["nick"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
# we have been mentioned in a msg/notice/action/part/quit/topic message
if "msg" in cast.keys(): # Don't highlight queries
if cast["msg"] is not None:
if self.nickname.lower() in cast["msg"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "highlight"
self.event(**castDup)
if "net" not in cast.keys():
cast["net"] = self.net
if "num" not in cast.keys():
cast["num"] = self.num
if not self.authenticated:
regproc.registerTest(cast)
counters.event(self.net, cast["type"])
monitor.event(self.net, cast)
def privmsg(self, user, channel, msg):
self.event(type="msg", muser=user, channel=channel, msg=msg)
def noticed(self, user, channel, msg):
self.event(type="notice", muser=user, channel=channel, msg=msg)
def action(self, user, channel, msg):
self.event(type="action", muser=user, channel=channel, msg=msg)
def sendmsg(self, channel, msg, in_query=False):
query = f"{self.nickname}!*@*"
res = userinfo.getWhoSingle(self.net, query)
if not res:
res = []
us = list(res)
if len(us) > 0:
hostmask = us[0]
else:
# Close enough...
hostmask = f"{self.nickname}!{self.nickname}@{self.servername}"
warn(f"Could not get a hostname, using {hostmask}")
nick, ident, host = parsen(hostmask)
# We sent someone a query reply
if in_query:
self.event(
type="self",
mtype="msg",
channel=self.nickname,
nick=channel,
ident=ident,
host=host,
msg=msg,
)
else:
self.event(
type="self",
mtype="msg",
channel=channel,
nick=self.nickname,
ident=ident,
host=host,
msg=msg,
)
self.event(
type="msg",
channel=channel,
nick=self.nickname,
ident=ident,
host=host,
msg=msg,
)
self.msg(channel, msg)
def get(self, var):
try:
result = getattr(self, var)
except AttributeError:
result = None
return result
def setNick(self, nickname):
self._attemptedNick = nickname
self.sendLine("NICK %s" % nickname)
self.nickname = nickname
def alterCollidedNick(self, nickname):
newnick = nickname + "_"
return newnick
def nickChanged(self, olduser, newnick):
self.nickname = newnick
self.event(type="self", mtype="nick", muser=olduser, user=newnick)
def irc_ERR_NICKNAMEINUSE(self, prefix, params):
self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
self.setNick(self._attemptedNick)
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username))
sendAll(
"%s - %i: password mismatch as %s" % (self.net, self.num, self.username)
)
def _who(self, channel):
d = Deferred()
if channel not in self._tempWho:
self._tempWho[channel] = ([], [])
self._tempWho[channel][0].append(d)
self.sendLine("WHO %s" % channel)
return d
def who(self, channel):
self._who(channel).addCallback(self.got_who)
def irc_RPL_WHOREPLY(self, prefix, params):
channel = params[1]
ident = params[2]
host = params[3]
server = params[4]
nick = params[5]
status = params[6]
realname = params[7]
if channel not in self._tempWho:
return
n = self._tempWho[channel][1]
n.append([nick, nick, host, server, status, realname])
self.event(
type="who",
nick=nick,
ident=ident,
host=host,
realname=realname,
channel=channel,
server=server,
status=status,
)
def irc_RPL_ENDOFWHO(self, prefix, params):
channel = params[1]
if channel not in self._tempWho:
return
callbacks, info = self._tempWho[channel]
for cb in callbacks:
cb.callback((channel, info))
del self._tempWho[channel]
def got_who(self, whoinfo):
userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
def sanit(self, data):
if len(data) >= 1:
if data[0] in self.prefix.keys():
return (
self.prefix[data[0]],
data[1:],
) # would use a set but it's possible these are the same
return (None, data)
else:
return (None, False)
def names(self, channel):
d = Deferred()
if channel not in self._tempNames:
self._tempNames[channel] = ([], [])
self._tempNames[channel][0].append(d)
self.sendLine("NAMES %s" % channel)
return d
def irc_RPL_NAMREPLY(self, prefix, params):
channel = params[2]
nicklist = params[3].split(" ")
if channel not in self._tempNames:
return
n = self._tempNames[channel][1]
n.append(nicklist)
def irc_RPL_ENDOFNAMES(self, prefix, params):
channel = params[1]
if channel not in self._tempNames:
return
callbacks, namelist = self._tempNames[channel]
for cb in callbacks:
cb.callback((channel, namelist))
del self._tempNames[channel]
def got_names(self, nicklist):
newNicklist = []
for i in nicklist[1]:
for x in i:
mode, nick = self.sanit(x)
if nick:
newNicklist.append((mode, nick))
userinfo.initialNames(self.net, nicklist[0], newNicklist)
def myInfo(self, servername, version, umodes, cmodes):
self.servername = servername
def _list(self, noargs):
d = Deferred()
self._tempList = ([], [])
self._tempList[0].append(d)
if self.listSimple:
self.sendLine("LIST")
return d # return early if we know what to do
if noargs:
self.sendLine("LIST")
else:
self.sendLine("LIST >0")
return d
def list(self, noargs=False, nocheck=False):
if not self.authenticated:
debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num))
return
if self.listAttempted:
debug(
"List request dropped, already asked for LIST - %s - %i"
% (self.net, self.num)
)
return
else:
self.listAttempted = True
if self.listOngoing:
debug(
"LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)
)
return
else:
if nocheck:
allRelays = True # override the system - if this is
else: # specified, we already did this
allRelays = chankeep.allRelaysActive(self.net)
if not allRelays:
self.wantList = True
debug("Not all relays were active for LIST request")
return
self._list(noargs).addCallback(self.got_list)
def irc_RPL_LISTSTART(self, prefix, params):
self.listAttempted = False
self.listOngoing = True
self.wantList = False
def irc_RPL_LIST(self, prefix, params):
channel = params[1]
users = params[2]
topic = params[3]
self._tempList[1].append([channel, users, topic])
def irc_RPL_LISTEND(self, prefix, params):
if (
not len(self._tempList[0]) > 0
): # there are no callbacks, can't do anything there
debug("We didn't ask for this LIST, discarding")
self._tempList[0].clear()
self._tempList[1].clear()
return
callbacks, info = self._tempList
self.listOngoing = False
for cb in callbacks:
cb.callback((info))
noResults = False
if len(self._tempList[1]) == 0:
noResults = True
self._tempList[0].clear()
self._tempList[1].clear()
if noResults:
if self.listRetried:
warn("LIST still empty after retry: %s - %i" % (self.net, self.num))
self.listRetried = False
return
else:
self.list(True)
self.listRetried = True
else:
if self.listRetried:
self.listRetried = False
debug(
"List received after retry - defaulting to simple list syntax: %s - %i"
% (self.net, self.num)
)
self.listSimple = True
def got_list(self, listinfo):
if len(listinfo) == 0: # probably ngircd not supporting LIST >0
return
if main.config["ChanKeep"]["Enabled"]:
chankeep.initialList(self.net, self.num, listinfo)
def recheckList(self):
if not main.config["ChanKeep"]["Enabled"]:
return
allRelays = chankeep.allRelaysActive(self.net)
debug(f"recheckList() all relays for {self.net} {allRelays}")
if allRelays:
debug(f"recheckList() all relays active for {self.net}")
first_relay = helpers.get_first_relay(self.net)
debug(f"recheckList() first relay for {self.net}: {first_relay.num}")
if first_relay:
if first_relay.wantList is True:
first_relay.list(nocheck=True)
debug(
f"recheckList() asking for a list for {self.net} after final relay {self.num} connected"
)
else:
debug(
f"recheckList() first relay wantList is False for {self.net} ({first_relay.num})"
)
# name = self.net + "1"
# if self.num == 1: # Only one instance should do a list
if helpers.is_first_relay(self.net, self.num):
debug(f"recheckList() we are the first relay for {self.net} ({self.num})")
if self.chanlimit:
if allRelays:
self.list()
debug(
f"recheckList() requested a list for {self.net} from {self.num}"
)
else:
debug(f"recheckList() not all relays active for {self.net}")
self.wantList = True
else:
debug("recheckList() aborting LIST due to bad chanlimit")
self.checkChannels()
def seed_chanlimit(self, chanlimit):
if not main.network[self.net].relays[self.num][
"registered"
]: # TODO: add check for register request sent, only send it once
if main.config["AutoReg"]:
if not self.authenticated:
self._regAttempt = reactor.callLater(
5, regproc.registerAccount, self.net, self.num
)
# regproc.registerAccount(self.net, self.num)
try:
self.chanlimit = int(chanlimit)
except TypeError:
warn("Invalid chanlimit: %s" % chanlimit)
if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's not limited
net_inst_chanlimit = self.netinst.chanlimit
if net_inst_chanlimit:
if self.chanlimit > net_inst_chanlimit:
self.chanlimit = net_inst_chanlimit
# warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}")
if not regproc.needToRegister(
self.net
): # if we need to register, only recheck on auth confirmation
if main.config["ChanKeep"]["Enabled"]:
self.recheckList()
def seed_prefix(self, prefix):
prefix = prefix.replace(")", "")
prefix = prefix.replace("(", "")
length = len(prefix)
half = int(length / 2)
prefixToMode = dict(zip(prefix[half:], prefix[:half]))
self.prefix = prefixToMode
def isupport(self, options):
interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"}
newOptions = {x for x in options if any(y in x for y in interested)}
if len(newOptions) == 0:
return
if not self.isconnected:
log("endpoint connected: %s - %i" % (self.net, self.num))
self.isconnected = True
for i in newOptions:
if i.startswith("PREFIX"):
if "=" in i:
split = i.split("=")
if len(split) == 2:
prefix = split[1]
self.seed_prefix(prefix)
elif i.startswith("CHANLIMIT"):
if ":" in i:
split = i.split(":")
if len(split) >= 2:
chanlimit = split[1]
self.seed_chanlimit(chanlimit)
elif i.startswith("MAXCHANNELS"):
if "=" in i:
split = i.split("=")
if len(split) == 2:
chanlimit = split[1]
self.seed_chanlimit(chanlimit)
# We need to override these functions as Twisted discards
# the hostname and other useful information in the functions
# that these call by default
def irc_JOIN(self, prefix, params):
nick = prefix.split("!")[0]
channel = params[-1]
if nick == self.nickname:
self.joined(channel)
else:
self.userJoined(prefix, channel)
def irc_PART(self, prefix, params):
nick = prefix.split("!")[0]
channel = params[0]
if len(params) >= 2:
message = params[1]
else:
message = None
if nick == self.nickname:
self.left(prefix, channel, message)
else:
self.userLeft(prefix, channel, message)
def irc_QUIT(self, prefix, params):
# nick = prefix.split("!")[0]
self.userQuit(prefix, params[0])
def irc_NICK(self, prefix, params):
nick = prefix.split("!", 1)[0]
if nick == self.nickname:
self.nickChanged(prefix, params[0])
else:
self.userRenamed(prefix, params[0])
def irc_KICK(self, prefix, params):
channel = params[0]
kicked = params[1]
message = params[-1]
# Checks on whether it was us that was kicked are done in userKicked
self.userKicked(kicked, channel, prefix, message)
def irc_TOPIC(self, prefix, params):
channel = params[0]
newtopic = params[1]
self.topicUpdated(prefix, channel, newtopic)
# End of Twisted hackery
def regPing(self, negativepass=None, reset=True):
if not main.config["AutoReg"]:
return
if self.authenticated:
return
if not regproc.needToAuth(self.net):
self.authenticated = True
return
sinst = regproc.substitute(self.net, self.num)
if not sinst:
error(f"regPing() {self.net}: registration ping failed for {self.num}")
return
if reset:
self._negativePass = None
debug(
f"regPing() {self.net} - {self.num}: _negativePass:{self._negativePass} negativepass:{negativepass}"
)
if self._negativePass is None:
# We have failed, the blacklisted message has been found
if negativepass is False:
self._negativePass = False
debug(
(
f"regPing() {self.net} - {self.num} not passed negative:checknegativemsg "
f"check, {sinst['checknegativemsg']} present in message"
)
)
return
# End of negative output reached with no blacklisted message
elif negativepass is True:
if self._negativePass is None: # check if it's not failed yet
self._negativePass = True
debug(
(
f"regPing() {self.net} - {self.num} passed negative:checkendnegative "
f"check, {sinst['checkendnegative']} present in message"
)
)
else:
debug(
f"regPing() {self.net}: negative registration check - {self.num}"
)
return
else:
if sinst["negative"]:
self.msg(sinst["entity"], sinst["negativemsg"])
debug(
(
f"regPing() {self.net}: sent negativemsg "
f"'{sinst['negativemsg']}' to {sinst['entity']} - {self.num}"
)
)
return
else:
self._negativePass = (
True # if it's disabled, we still want the next block to run
)
# Only run if negativepass has completed, or exempted as above
if self._negativePass:
if sinst["check"]:
if sinst["ping"]:
self.msg(sinst["entity"], sinst["pingmsg"])
debug(
f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}"
)
return
else:
self.authenticated = True
def signedOn(self):
log("signed on: %s - %i" % (self.net, self.num))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "signedon",
"ts": ctime,
}
)
if not self.authenticated:
reactor.callLater(10, self.regPing)
def setup_who_loop(self, channel):
# if main.config["Toggles"]["Who"]:
lc = LoopingCall(self.who, channel)
self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
interval = randint(minint, minint + intrange)
lc.start(interval)
def joined(self, channel):
if channel not in self.channels:
self.channels.append(channel)
self.names(channel).addCallback(self.got_names)
if main.config["Toggles"]["Who"]:
reactor.callLater(
main.config["Tweaks"]["Delays"]["WhoDelay"],
self.setup_who_loop,
channel,
)
# lc = LoopingCall(self.who, channel)
# self._getWho[channel] = lc
# intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
# minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
# interval = randint(minint, minint + intrange)
# lc.start(interval)
def botLeft(self, channel):
if channel in self.channels:
self.channels.remove(channel)
if channel in self._getWho.keys():
lc = self._getWho[channel]
lc.stop()
del self._getWho[channel]
userinfo.delChannels(
self.net, [channel]
) # < we do not need to deduplicate this
# log("Can no longer cover %s, removing records" % channel)# as it will only be matched once --
# other bots have different nicknames so
def left(self, user, channel, message): # even if they saw it, they wouldn't react
self.event(type="part", muser=user, channel=channel, msg=message)
self.botLeft(channel)
def userJoined(self, user, channel):
self.event(type="join", muser=user, channel=channel)
def userLeft(self, user, channel, message):
self.event(type="part", muser=user, channel=channel, msg=message)
def userQuit(self, user, quitMessage):
self.chanlessEvent({"type": "quit", "muser": user, "msg": quitMessage})
def userKicked(self, kickee, channel, kicker, message):
if kickee.lower() == self.nickname.lower():
self.botLeft(channel)
self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee)
def chanlessEvent(self, cast):
cast["ts"] = int(datetime.now().timestamp())
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
if dedup(self.name, cast): # Needs to be kept self.name until the dedup
# function is converted to the new net, num
# format
return # stop right there sir!
chans = userinfo.getChanList(self.net, cast["nick"])
if chans is None:
error("No channels returned for chanless event: %s" % cast)
# self.event(**cast) -- no, should NEVER happen
return
# getChansSingle returns all channels of the user, we only want to use
# ones we have common with them
realChans = set(chans).intersection(set(self.channels))
for i in realChans:
cast["channel"] = i
self.event(**cast)
def userRenamed(self, oldname, newname):
self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
def topicUpdated(self, user, channel, newTopic):
self.event(type="topic", muser=user, channel=channel, msg=newTopic)
def modeChanged(self, user, channel, toset, modes, args):
argList = list(args)
modeList = [i for i in modes]
for a, m in zip(argList, modeList):
self.event(
type="mode",
muser=user,
channel=channel,
mode=m,
status=toset,
modearg=a,
)
# TODO: strip out relay functionality
class IRCBotFactory(ReconnectingClientFactory):
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
if net is None:
self.num = num
self.net = None
self.name = "relay - %i" % num
self.relay = True
else:
self.name = net + str(num)
self.num = num
self.net = net
self.relay = False
self.client = None
self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
self.factor = main.config["Tweaks"]["Delays"]["Factor"]
self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
def buildProtocol(self, addr):
entry = IRCBot(self.net, self.num)
main.IRCPool[self.name] = entry
self.client = entry
return entry
def clientConnectionLost(self, connector, reason):
if not self.relay:
userinfo.delChannels(self.net, self.client.channels)
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
error = reason.getErrorMessage()
log("%s - %i: connection lost: %s" % (self.net, self.num, error))
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "lost",
"message": error,
"ts": ctime,
}
)
self.retry(connector)
# ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
error = reason.getErrorMessage()
log("%s - %i: connection failed: %s" % (self.net, self.num, error))
if not self.relay:
sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "failed",
"message": error,
"ts": ctime,
}
)
self.retry(connector)
# ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

27
legacy/core/parser.py Normal file
View File

@ -0,0 +1,27 @@
import main
from utils.logging.log import warn
from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess
def parseCommand(addr, authed, data):
# call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
spl = data.split()
if addr in main.connections.keys():
obj = main.connections[addr]
else:
warn("Got connection object with no instance in the address pool")
return
success = lambda data: sendSuccess(addr, data) # noqa: E731
failure = lambda data: sendFailure(addr, data) # noqa: E731
info = lambda data: sendInfo(addr, data) # noqa: E731
incUsage = lambda mode: incorrectUsage(addr, mode) # noqa: E731
length = len(spl)
if spl[0] in main.CommandMap.keys():
main.CommandMap[spl[0]](
addr, authed, data, obj, spl, success, failure, info, incUsage, length
)
return
incUsage(None)
return

58
core/relay.py → legacy/core/relay.py
View File

@ -1,12 +1,28 @@
from twisted.internet.protocol import Protocol, Factory, ClientFactory
from json import dumps, loads
from copy import deepcopy
from datetime import datetime
import main
from utils.logging.log import *
from twisted.internet.protocol import Factory, Protocol
from utils.logging.log import log, warn
validTypes = [
"msg",
"notice",
"action",
"who",
"part",
"join",
"kick",
"quit",
"nick",
"topic",
"mode",
"conn",
"znc",
"query",
"self",
"highlight",
]
validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"]
class Relay(Protocol):
def __init__(self, addr):
@ -30,10 +46,10 @@ class Relay(Protocol):
data = data.decode("utf-8", "replace")
try:
parsed = loads(data)
except:
except: # noqa: E722
self.sendErr("MALFORMED")
return
if not "type" in parsed.keys():
if "type" not in parsed.keys():
self.sendErr("NOTYPE")
return
if parsed["type"] == "hello":
@ -70,7 +86,7 @@ class Relay(Protocol):
self.sendErr("NOTLIST")
return
for i in lst:
if not i in validTypes:
if i not in validTypes:
self.sendErr("NONEXISTANT")
return
if i in self.subscriptions:
@ -85,10 +101,10 @@ class Relay(Protocol):
self.sendErr("NOTLIST")
return
for i in lst:
if not i in validTypes:
if i not in validTypes:
self.sendErr("NONEXISTANT")
return
if not i in self.subscriptions:
if i not in self.subscriptions:
self.sendErr("NOTSUBSCRIBED")
return
del self.subscriptions[i]
@ -97,8 +113,13 @@ class Relay(Protocol):
def handleHello(self, parsed):
if parsed["key"] in main.tokens.keys():
if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay":
self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]})
if (
parsed["hello"] == main.tokens[parsed["key"]]["hello"]
and main.tokens[parsed["key"]]["usage"] == "relay"
):
self.sendMsg(
{"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}
)
self.authed = True
else:
self.transport.loseConnection()
@ -109,16 +130,20 @@ class Relay(Protocol):
def connectionMade(self):
log("Relay connection from %s:%s" % (self.addr.host, self.addr.port))
#self.send("Greetings.")
# self.send("Greetings.")
def connectionLost(self, reason):
self.authed = False
log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
log(
"Relay connection lost from %s:%s -- %s"
% (self.addr.host, self.addr.port, reason.getErrorMessage())
)
if self.addr in main.relayConnections.keys():
del main.relayConnections[self.addr]
else:
warn("Tried to remove a non-existant relay connection.")
class RelayFactory(Factory):
def buildProtocol(self, addr):
entry = Relay(addr)
@ -132,10 +157,9 @@ class RelayFactory(Factory):
else:
return
def sendRelayNotification(cast):
for i in main.relayConnections.keys():
if main.relayConnections[i].authed:
if cast["type"] in main.relayConnections[i].subscriptions:
newCast = deepcopy(cast)
newCast["time"] = str(datetime.now().isoformat())
main.relayConnections[i].send(dumps(newCast))
main.relayConnections[i].send(dumps(cast))

18
core/server.py → legacy/core/server.py
View File

@ -1,14 +1,14 @@
from twisted.internet.protocol import Protocol, Factory, ClientFactory
import main
from utils.logging.log import *
from core.parser import parseCommand
from twisted.internet.protocol import Factory, Protocol
from utils.logging.log import log, warn
class Server(Protocol):
def __init__(self, addr):
self.addr = addr
self.authed = False
if main.config["UsePassword"] == False:
if main.config["UsePassword"] is False:
self.authed = True
def send(self, data):
@ -18,7 +18,7 @@ class Server(Protocol):
def dataReceived(self, data):
data = data.decode("utf-8", "replace")
#log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
# log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
if "\n" in data:
splitData = [x for x in data.split("\n") if x]
if "\n" in data:
@ -33,13 +33,15 @@ class Server(Protocol):
def connectionLost(self, reason):
self.authed = False
log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
log(
"Connection lost from %s:%s -- %s"
% (self.addr.host, self.addr.port, reason.getErrorMessage())
)
if self.addr in main.connections.keys():
del main.connections[self.addr]
else:
warn("Tried to remove a non-existant connection.")
if self.addr in main.MonitorPool:
main.MonitorPool.remove(self.addr)
class ServerFactory(Factory):
def buildProtocol(self, addr):

41
legacy/docker-compose.yml Normal file
View File

@ -0,0 +1,41 @@
version: "2"
services:
app:
image: pathogen/threshold:latest
build: ./docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
env_file:
- .env
# for development
extra_hosts:
- "host.docker.internal:host-gateway"
volumes_from:
- tmp
tmp:
image: busybox
command: chmod -R 777 /var/run/redis
volumes:
- /var/run/redis
redis:
image: redis
command: redis-server /etc/redis.conf
volumes:
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
volumes_from:
- tmp
networks:
default:
external:
name: pathogen

18
legacy/docker/Dockerfile Normal file
View File

@ -0,0 +1,18 @@
# syntax=docker/dockerfile:1
FROM python:3
RUN useradd -d /code pathogen
RUN mkdir /code
RUN chown pathogen:pathogen /code
RUN mkdir /venv
RUN chown pathogen:pathogen /venv
USER pathogen
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
WORKDIR /code/legacy
COPY requirements.prod.txt /code/legacy
RUN python -m venv /venv
RUN . /venv/bin/activate && pip install -r /code/legacy/requirements.prod.txt
CMD . /venv/bin/activate && exec python /code/legacy/threshold

38
legacy/docker/docker-compose.prod.yml Normal file
View File

@ -0,0 +1,38 @@
version: "2"
services:
app:
image: pathogen/threshold:latest
build: ./docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
env_file:
- ../stack.env
volumes_from:
- tmp
tmp:
image: busybox
command: chmod -R 777 /var/run/redis
volumes:
- /var/run/redis
redis:
image: redis
command: redis-server /etc/redis.conf
volumes:
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
volumes_from:
- tmp
networks:
default:
external:
name: pathogen

8
legacy/docker/requirements.prod.txt Normal file
View File

@ -0,0 +1,8 @@
wheel
twisted
pyOpenSSL
redis
pyYaML
service_identity
siphashc
Klein

131
legacy/main.py Normal file
View File

@ -0,0 +1,131 @@
import json
import pickle
from os import urandom
from os.path import exists
from string import digits
from redis import StrictRedis
# List of errors ZNC can give us
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
configPath = "conf/live/"
templateConfigPath = "conf/templates/"
certPath = "conf/cert/"
filemap = {
# JSON configs
"config": ["config.json", "configuration", "json"],
"help": ["help.json", "command help", "json"],
"counters": ["counters.json", "counters file", "json"],
"tokens": ["tokens.json", "authentication tokens", "json"],
"aliasdata": ["aliasdata.json", "data for alias generation", "json"],
"alias": ["alias.json", "provisioned alias data", "json"],
"irc": ["irc.json", "IRC network definitions", "json"],
"blacklist": ["blacklist.json", "IRC channel blacklist", "json"],
# Binary (pickle) configs
"network": ["network.dat", "network list", "pickle"],
}
# Connections to the plain-text interface
connections = {}
# Connections to the JSON interface
relayConnections = {}
# Mapping of network names to Protocol (IRCClient) instances
IRCPool = {}
# Mapping of network names to Reactor instances
# Needed for calling .disconnect()
ReactorPool = {}
# Mapping of network names to Factory instances
# Needed for calling .stopTrying()
FactoryPool = {}
# Temporary store for channels allocated after a LIST
# Will get purged as the instances fire up and pop() from
# their respective keys in here
TempChan = {}
# Mapping of command names to their functions
CommandMap = {}
# Incremented by 1 for each event reaching modules.counters.event()
# and cloned into lastMinuteSample every minute
runningSample = 0
lastMinuteSample = 0
# Generate 16-byte hex key for message checksums
hashKey = urandom(16)
lastEvents = {}
# Get networks that are currently online and dedupliate
def liveNets():
networks = set()
for i in IRCPool.keys():
networks.add("".join([x for x in i if x not in digits]))
return networks
def saveConf(var):
if var in ("help", "aliasdata"):
return # no need to save this
if filemap[var][2] == "json":
with open(configPath + filemap[var][0], "w") as f:
json.dump(globals()[var], f, indent=4)
elif filemap[var][2] == "pickle":
with open(configPath + filemap[var][0], "wb") as f:
pickle.dump(globals()[var], f)
else:
raise Exception("invalid format")
def loadConf(var):
if filemap[var][2] == "json":
filename = configPath + filemap[var][0]
# Only take the help from the templates
if var in ("help", "aliasdata"):
filename = templateConfigPath + filemap[var][0]
if not exists(filename):
# Load the template config
if var == "config":
filename = templateConfigPath + filemap[var][0]
else:
# Everything else should be blank
globals()[var] = {}
return
with open(filename, "r") as f:
globals()[var] = json.load(f)
if var == "alias":
# This is a workaround to convert all the keys into integers since JSON
# turns them into strings...
# Dammit Jason!
global alias
alias = {int(x): y for x, y in alias.items()}
elif filemap[var][2] == "pickle":
try:
with open(configPath + filemap[var][0], "rb") as f:
globals()[var] = pickle.load(f)
except FileNotFoundError:
globals()[var] = {}
else:
raise Exception("invalid format")
def initConf():
for i in filemap.keys():
loadConf(i)
def initMain():
global r, g
initConf()
r = StrictRedis(
unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] # noqa
) # Ephemeral - flushed on quit
g = StrictRedis(
unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]
) # noqa

0
legacy/modules/__init__.py Normal file
View File

21
modules/alias.py → legacy/modules/alias.py
View File

@ -1,10 +1,13 @@
import main
import random
import re
import main
def generate_password():
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
def generate_alias():
nick = random.choice(main.aliasdata["stubs"])
rand = random.randint(1, 2)
@ -12,8 +15,8 @@ def generate_alias():
nick = nick.capitalize()
rand = random.randint(1, 4)
while rand == 1:
split = random.randint(0, len(nick)-1)
nick = nick[:split] + nick[split+1:]
split = random.randint(0, len(nick) - 1)
nick = nick[:split] + nick[split + 1 :] # noqa: E203
rand = random.randint(1, 4)
rand = random.randint(1, 3)
if rand == 1 or rand == 4:
@ -51,7 +54,7 @@ def generate_alias():
ident = namebase.split(" ")[0]
ident = ident[:10]
elif rand == 6:
ident = re.sub("\s", "", namebase).lower()
ident = re.sub("\s", "", namebase).lower() # noqa: W605
ident = ident[:10]
realname = nick
@ -63,6 +66,10 @@ def generate_alias():
if rand == 3 or rand == 4:
realname = realname.capitalize()
password = generate_password()
return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password}
return {
"nick": nick,
"altnick": altnick,
"ident": ident,
"realname": realname,
"emails": [],
}

600
legacy/modules/chankeep.py Normal file
View File

@ -0,0 +1,600 @@
from copy import deepcopy
from math import ceil
import main
from modules import helpers
from twisted.internet.threads import deferToThread
from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn
def getAllChannels(net=None):
"""
Get a list of all channels on all relays.
:return: list of channels
"""
channels = {}
if not net:
nets = main.network.keys()
else:
nets = [net]
for net in nets:
relays = helpers.get_connected_relays(net)
for relay in relays:
if net not in channels:
channels[net] = {}
if relay.num not in channels[net]:
channels[net][relay.num] = []
for channel in relay.channels:
channels[net][relay.num].append(channel)
# debug(f"getAllChannels(): {channels}")
return channels
def getDuplicateChannels(net=None, total=False):
"""
Get a list of duplicate channels.
:return: list of duplicate channels
"""
allChans = getAllChannels(net)
duplicates = {}
for net in allChans.keys():
net_chans = []
inst = {}
# add all the channels from this network to a list
for num in allChans[net].keys():
net_chans.extend(allChans[net][num])
for channel in net_chans:
count_chan = net_chans.count(channel)
# I don't know why but it works
# this is used in userinfo.delChannels
set_min = 1
if total:
set_min = 0
if count_chan > set_min:
inst[channel] = count_chan
if inst:
duplicates[net] = inst
if total:
return duplicates
to_part = {}
for net in allChans:
if net in duplicates:
for num in allChans[net].keys():
for channel in allChans[net][num]:
if channel in duplicates[net].keys():
if duplicates[net][channel] > 1:
if net not in to_part:
to_part[net] = {}
if num not in to_part[net]:
to_part[net][num] = []
to_part[net][num].append(channel)
duplicates[net][channel] -= 1
return to_part
def partChannels(data):
for net in data:
for num in data[net]:
name = f"{net}{num}"
if name in main.IRCPool.keys():
for channel in data[net][num]:
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
log(f"Parted {channel} on {net} - {num}")
def getEnabledRelays(net):
"""
Get a list of enabled relays for a network.
:param net: network
:rtype: list of int
:return: list of enabled relay numbers
"""
enabledRelays = [
x
for x in main.network[net].relays.keys()
if main.network[net].relays[x]["enabled"]
]
# debug(f"getEnabledRelays() {net}: {enabledRelays}")
return enabledRelays
def getConnectedRelays(net):
"""
Get a list of connected relays for a network.
:param net: network
:rtype: list of int
:return: list of relay numbers
"""
enabledRelays = getEnabledRelays(net)
connectedRelays = []
for i in enabledRelays:
name = net + str(i)
if name in main.IRCPool.keys():
if main.IRCPool[name].isconnected:
connectedRelays.append(i)
# debug(f"getConnectedRelays() {net}: {connectedRelays}")
return connectedRelays
def getActiveRelays(net):
"""
Get a list of active relays for a network.
:param net: network
:rtype: list of int
:return: list of relay numbers
"""
enabledRelays = getEnabledRelays(net)
activeRelays = []
for i in enabledRelays:
name = net + str(i)
if name in main.IRCPool.keys():
# debug(
# (
# f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
# f"conn:{main.IRCPool[name].isconnected}"
# )
# )
if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
activeRelays.append(i)
debug(f"getActiveRelays() {net}: {activeRelays}")
return activeRelays
def relayIsActive(net, num):
"""
Check if a relay is active.
:param net: network
:param num: relay number
:rtype: bool
:return: True if relay is active, False otherwise
"""
activeRelays = getActiveRelays(net)
return num in activeRelays
def allRelaysActive(net):
"""
Check if all enabled relays are active and authenticated.
:param net: network
:rtype: bool
:return: True if all relays are active and authenticated, False otherwise
"""
activeRelays = getActiveRelays(net)
enabledRelays = getEnabledRelays(net)
relaysActive = len(activeRelays) == len(enabledRelays)
# debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
return relaysActive
def getAverageChanlimit(net):
"""
Get the average channel limit for a network.
:param net: network
:rtype: int
:return: average channel limit
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += main.IRCPool[name].chanlimit
avg_chanlimit = total / len(getActiveRelays(net))
debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
return avg_chanlimit
def getSumChanlimit(net):
"""
Get the sum of all channel limits for a network.
:param net: network
:rtype: int
:return: sum of channel limits
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += main.IRCPool[name].chanlimit
return total
def getChanFree(net):
"""
Get a dictionary with the free channel spaces for
each relay, and a channel limit.
Example return:
({1: 99}, 100)
:param net: network
:return: ({relay: channel spaces}, channel limit)
"""
chanfree = {}
for i in getActiveRelays(net):
name = net + str(i)
if name not in main.IRCPool.keys():
continue
if not main.IRCPool[name].isconnected:
continue
chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
return chanfree
def getTotalChans(net):
"""
Get the total number of channels on all relays for a network.
:param net: network
:rtype: int
:return: total number of channels
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += len(main.IRCPool[name].channels)
return total
def emptyChanAllocate(net, flist):
"""
Allocate channels to relays.
:param net: network
:param flist: list of channels to allocate
:param new: list of newly provisioned relays to account for
:rtype: dict
:return: dictionary of {relay: list of channels}"""
# Get the free channel spaces for each relay
chanfree = getChanFree(net)
if not chanfree:
return
# Pretend the newly provisioned relays are already on the network
# for i in new:
# chanfree[0][i] = chanfree[1]
allocated = {}
newlist = list(flist)
chan_slots_used = getTotalChans(net)
max_chans = getSumChanlimit(net) - chan_slots_used
trunc_list = newlist[:max_chans]
debug(
f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}"
)
for i in chanfree.keys():
for x in range(chanfree[i]):
if not len(trunc_list):
break
if i in allocated.keys():
allocated[i].append(trunc_list.pop())
else:
allocated[i] = [trunc_list.pop()]
return allocated
def populateChans(net, clist):
"""
Populate channels on relays.
Stores channels to join in a list in main.TempChan[net][num]
:param net: network
:param clist: list of channels to join
:param new: list of newly provisioned relays to account for"""
# divided = array_split(clist, relay)
allocated = emptyChanAllocate(net, clist)
trace(f"populateChans() allocated:{allocated}")
if not allocated:
return
for i in allocated.keys():
if net in main.TempChan.keys():
main.TempChan[net][i] = allocated[i]
else:
main.TempChan[net] = {i: allocated[i]}
trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
def notifyJoin(net):
"""
Notify relays to join channels.
They will pull from main.TempChan and remove channels they join.
:param net: network
"""
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
trace(f"notifyJoin() {name}")
main.IRCPool[name].checkChannels()
def minifyChans(net, listinfo, as_list=False):
"""
Remove channels from listinfo that are already covered by a relay.
:param net: network
:param listinfo: list of channels to check
:type listinfo: list of [channel, num_users]
:return: list of channels with joined channels removed
:rtype: list of [channel, num_users]
"""
# We want to make this reusable for joining a bunch of channels.
if as_list:
channel_list = listinfo
if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net)
return False
for i in getConnectedRelays(net):
name = net + str(i)
for x in main.IRCPool[name].channels:
if as_list:
for y in channel_list:
if y == x:
channel_list.remove(y)
else:
for y in listinfo:
if y[0] == x:
listinfo.remove(y)
if not as_list:
if not listinfo:
log("We're on all the channels we want to be on, dropping LIST")
return False
if as_list:
return channel_list
else:
return listinfo
def keepChannels(net, listinfo, mean, sigrelay, relay):
"""
Minify channels, determine whether we can cover all the channels
on the network, or need to use 'significant' mode.
Truncate the channel list to available channel spaces.
Allocate these channels to relays.
Notify relays that they should pull from TempChan to join.
:param net: network
:param listinfo: list of [channel, num_users] lists
:param mean: mean of channel population
:param sigrelay: number of relays needed to cover significant channels
:param relay: number of relays needed to cover all channels
:param chanlimit: maximum number of channels to allocate to a relay
"""
listinfo = minifyChans(net, listinfo)
if not listinfo:
return
if (
relay <= main.config["ChanKeep"]["SigSwitch"]
): # we can cover all of the channels
coverAll = True
elif (
relay > main.config["ChanKeep"]["SigSwitch"]
): # we cannot cover all of the channels
coverAll = False
# if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
# error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
# return
num_instances = len(getActiveRelays(net))
debug(f"keepChannels() {net} instances:{num_instances}")
chan_slots_used = getTotalChans(net)
debug(f"keepChannels() slots_used:{chan_slots_used}")
# max_chans = (chanlimit * num_instances) - chan_slots_used
max_chans = getSumChanlimit(net) - chan_slots_used
if max_chans < 0:
max_chans = 0
debug(f"keepChannels() max_chans:{max_chans}")
if coverAll:
# needed = relay - len(getActiveRelays(net))
# if needed:
# debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
# newNums = modules.provision.provisionMultipleRelays(net, needed)
# else:
# newNums = []
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
if len(listinfo_sort) > max_chans:
max_chans = len(listinfo_sort) - 1
flist = [i[0] for i in listinfo_sort]
flist = flist[:max_chans]
debug(
f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels"
)
trace(f"keepChannels() {net}: joining:{flist}")
populateChans(net, flist)
else:
# needed = sigrelay - len(getActiveRelays(net))
# if needed:
# debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
# newNums = modules.provision.provisionMultipleRelays(net, needed)
# else:
# newNums = []
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
if len(listinfo_sort) > max_chans:
max_chans = len(listinfo_sort) - 1
debug(f"keepChannels() {net}: new max_chans:{max_chans}")
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
trace(f"keepChannels() {net}: new siglist:{siglist}")
siglist = siglist[:max_chans]
trace(f"keepChannels() {net}: truncated siglist:{siglist}")
trace(
f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}"
)
debug(
f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels"
)
trace(f"keepChannels() {net}: joining:{siglist}")
populateChans(net, siglist)
notifyJoin(net)
def joinSingle(net, channel):
"""
Join a channel on a relay.
Use ECA to determine which relay to join on.
:param net: network
:param channel: channel to join
:return: relay number that joined the channel
:rtype: int
"""
if "," in channel:
channels = channel.split(",")
channels = minifyChans(net, channels, as_list=True)
else:
channels = [channel]
populateChans(net, channels)
notifyJoin(net)
return True
def partSingle(net, channel):
"""
Iterate over all the relays of net and part channels matching channel.
:param net: network
:param channel: channel to part
:return: list of relays that parted the channel
:rtype: list of str
"""
parted = []
for i in getConnectedRelays(net):
name = f"{net}{i}"
if name in main.IRCPool.keys():
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
parted.append(str(i))
return parted
def nukeNetwork(net):
"""
Remove network records.
:param net: network"""
# purgeRecords(net)
# p = main.g.pipeline()
main.g.delete("analytics.list." + net)
# p.delete("list."+net)
# p.execute()
# def nukeNetwork(net):
# deferToThread(_nukeNetwork, net)
def _initialList(net, num, listinfo):
"""
Called when a relay receives a full LIST response.
Run statistics to determine how many channels are significant.
This is done by adding all the numbers of users on the channels together,
then dividing by the number of channels.
* cumul - cumulative sum of all channel membership
* siglength - number of significant channels
* listlength - number of channels in the list
* sigrelay - number of relays needed to cover siglength
* relay - number of relays needed to cover all channels
:param net: network
:param num: relay number
:param listinfo: list of [channel, num_users] lists
:param chanlimit: maximum number of channels the relay can join
"""
listlength = len(listinfo)
cumul = 0
try:
cumul += sum(int(i[1]) for i in listinfo)
except TypeError:
warn("Bad LIST data received from %s - %i" % (net, num))
return
mean = round(cumul / listlength, 2)
siglength = 0
insiglength = 0
sigcumul = 0
insigcumul = 0
for i in listinfo:
if int(i[1]) > mean:
siglength += 1
sigcumul += int(i[1])
elif int(i[1]) < mean:
insiglength += 1
insigcumul += int(i[1])
avg_chanlimit = getAverageChanlimit(net)
sigrelay = ceil(siglength / avg_chanlimit)
relay = ceil(listlength / avg_chanlimit)
cur_relays = len(getActiveRelays(net))
sig_relays_missing = sigrelay - cur_relays
all_relays_missing = relay - cur_relays
abase = "analytics.list.%s" % net
main.g.delete(abase)
p = main.g.pipeline()
# See docstring for meanings
p.hset(abase, "mean", mean)
p.hset(abase, "total_chans", listlength)
p.hset(abase, "big_chans", siglength)
p.hset(abase, "small_chans", insiglength)
p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
p.hset(abase, "total_cumul_mem", cumul)
p.hset(abase, "big_chan_cumul_mem", sigcumul)
p.hset(abase, "small_chan_cumul_mem", insigcumul)
p.hset(abase, "relays_for_all_chans", relay)
p.hset(abase, "relays_for_big_chans", sigrelay)
p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
p.hset(abase, "sig_relays_missing", sig_relays_missing)
p.hset(abase, "all_relays_missing", all_relays_missing)
debug(
(
f"_initialList() net:{net} num:{num} listlength:{listlength} "
f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
)
)
# Purge existing records before writing
# purgeRecords(net)
# for i in listinfo:
# p.rpush(netbase+"."+i[0], i[1])
# p.rpush(netbase+"."+i[0], i[2])
# p.sadd(netbase, i[0])
p.execute()
debug("List parsing completed on %s" % net)
keepChannels(net, listinfo, mean, sigrelay, relay)
# return (listinfo, mean, sigrelay, relay)
def convert(data):
"""
Recursively convert a dictionary.
"""
if isinstance(data, bytes):
return data.decode("ascii")
if isinstance(data, dict):
return dict(map(convert, data.items()))
if isinstance(data, tuple):
return map(convert, data)
if isinstance(data, list):
return list(map(convert, data))
return data
def getListInfo(net):
abase = f"analytics.list.{net}"
info = main.g.hgetall(abase)
return convert(info)
def initialList(net, num, listinfo):
"""
Run _initialList in a thread.
See above docstring.
"""
deferToThread(_initialList, net, num, deepcopy(listinfo))

12
modules/counters.py → legacy/modules/counters.py
View File

@ -1,12 +1,13 @@
import main
from twisted.internet.task import LoopingCall
def event(name, eventType):
if not "local" in main.counters.keys():
if "local" not in main.counters.keys():
main.counters["local"] = {}
if not "global" in main.counters.keys():
if "global" not in main.counters.keys():
main.counters["global"] = {}
if not name in main.counters["local"].keys():
if name not in main.counters["local"].keys():
main.counters["local"][name] = {}
if eventType not in main.counters["local"][name].keys():
main.counters["local"][name][eventType] = 0
@ -18,8 +19,9 @@ def event(name, eventType):
main.counters["global"][eventType] += 1
main.runningSample += 1
def getEvents(name=None):
if name == None:
if name is None:
if "global" in main.counters.keys():
return main.counters["global"]
else:
@ -30,10 +32,12 @@ def getEvents(name=None):
else:
return None
def takeSample():
main.lastMinuteSample = main.runningSample
main.runningSample = 0
def setupCounterLoop():
lc = LoopingCall(takeSample)
lc.start(60)

72
legacy/modules/helpers.py Normal file
View File

@ -0,0 +1,72 @@
import main
from modules import chankeep
from utils.logging.debug import debug
def get_first_relay(net):
"""
Get the first relay in the network.
:param net: the network
:param num: number or relay
:return: IRCPool instance for the IRC bot
"""
cur_relay = 0
max_relay = max(main.network[net].relays.keys())
debug(f"get_first_relay() {net}: max_relay:{max_relay}")
activeRelays = chankeep.getActiveRelays(net)
debug(f"get_first_relay() {net}: activeRelays:{activeRelays}")
while cur_relay != max_relay:
cur_relay += 1
if cur_relay not in activeRelays:
continue
name = net + str(cur_relay)
if name in main.IRCPool.keys():
# debug(f"get_first_relay() {net}: found relay {name}")
return main.IRCPool[name]
return None
def is_first_relay(net, num):
"""
Determine if we are the first relay for the network.
:param net: the network
:param num: number or relay
:return: True if we are the first relay, False otherwise
"""
first_relay = get_first_relay(net)
if not first_relay:
return False
return first_relay.num == num
def get_active_relays(net):
"""
Get all active instances for the network.
:param net: the network
:return: list of active instances
:rtype: list of IRCPool instances
"""
active_nums = chankeep.getActiveRelays(net)
active_insts = []
for num in active_nums:
name = net + str(num)
if name in main.IRCPool.keys():
active_insts.append(main.IRCPool[name])
return active_insts
def get_connected_relays(net):
"""
Get all connected instances for the network.
:param net: the network
:return: list of active instances
:rtype: list of IRCPool instances
"""
active_nums = chankeep.getConnectedRelays(net)
active_insts = []
for num in active_nums:
name = net + str(num)
if name in main.IRCPool.keys():
active_insts.append(main.IRCPool[name])
return active_insts

91
legacy/modules/monitor.py Normal file
View File

@ -0,0 +1,91 @@
import json
import main
from core.relay import sendRelayNotification
from modules import userinfo
from utils.dedup import dedup
order = [
"type",
"net",
"num",
"channel",
"msg",
"nick",
"ident",
"host",
"mtype",
"user",
"mode",
"modearg",
"realname",
"server",
"status",
"ts",
]
def parsemeta(numName, c):
if "channel" not in c.keys():
c["channel"] = None
# metadata scraping
# need to check if this was received from a relay
# in which case, do not do this
if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
if "muser" in c.keys():
userinfo.editUser(c["net"], c["muser"])
# if c["type"] == "mode":
# userinfo.updateMode(c)
elif c["type"] == "nick":
userinfo.renameUser(
c["net"],
c["nick"],
c["muser"],
c["user"],
c["user"] + "!" + c["ident"] + "@" + c["host"],
)
elif c["type"] == "kick":
userinfo.editUser(c["net"], c["muser"])
userinfo.delUserByNick(c["net"], c["channel"], c["user"])
elif c["type"] == "quit":
userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"])
elif c["type"] == "join":
userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"])
elif c["type"] == "part":
userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"])
if "mtype" in c.keys():
if c["mtype"] == "nick":
userinfo.renameUser(
c["net"],
c["nick"],
c["muser"],
c["user"],
c["user"] + "!" + c["ident"] + "@" + c["host"],
)
def queue_message(c):
message = json.dumps(c)
print("APPENDING MESSAGE", message)
print("KEY", main.config["Ingest"]["Key"])
main.g.sadd(main.config["Ingest"]["Key"], message)
def event(
numName, c
): # yes I'm using a short variable because otherwise it goes off the screen
if dedup(numName, c):
return
# make a copy of the object with dict() to prevent sending notifications with channel of None
parsemeta(numName, dict(c))
if "muser" in c.keys():
del c["muser"]
# sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order
# sortedKeys["src"] = "irc"
c["src"] = "irc"
if main.config["Ingest"]["Enabled"]:
queue_message(c)
sendRelayNotification(c)

163
legacy/modules/network.py Normal file
View File

@ -0,0 +1,163 @@
from copy import deepcopy
import main
from core.bot import IRCBotFactory
from modules import alias
from modules.chankeep import nukeNetwork
from modules.provision import provisionRelay
from modules.regproc import needToRegister
from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from utils.deliver_relay_commands import deliverRelayCommands
from utils.get import getRelay
from utils.logging.log import log
def migrate():
existing = deepcopy(main.network)
log("Migrating network configuration")
log(f"Existing network configuration: {existing}")
for net, net_inst in existing.items():
log(f"Migrating network {net}")
net = net_inst.net
host = net_inst.host
port = net_inst.port
security = net_inst.security
auth = net_inst.auth
last = net_inst.last
relays = net_inst.relays
aliases = net_inst.aliases
new_net = Network(net, host, port, security, auth)
log(f"New network for {net}: {new_net}")
new_net.last = last
new_net.relays = relays
new_net.aliases = aliases
main.network[net] = new_net
main.saveConf("network")
log("Finished migrating network configuration")
class Network:
def __init__(self, net, host, port, security, auth):
self.net = net
self.host = host
self.port = port
self.security = security
self.auth = auth
self.chanlimit = None
self.last = 1
self.relays = {}
self.aliases = {}
def add_relay(self, num=None):
# Grrrrrrrrrr
self.last = int(self.last)
if not num:
num = self.last
self.last += 1
elif num == self.last:
self.last += 1
registered = False
if not needToRegister(self.net):
registered = True
# Don't need to register if it's been disabled in definitions,
# so we'll pretend we already did
self.relays[num] = {
"enabled": main.config["ConnectOnCreate"],
"net": self.net,
"id": num,
"registered": registered,
}
password = alias.generate_password()
if num not in main.alias.keys():
main.alias[num] = alias.generate_alias()
main.saveConf("alias")
self.aliases[num] = {"password": password}
# if main.config["ConnectOnCreate"]: -- Done in provision
# self.start_bot(num)
provisionRelay(num, self.net)
return num, main.alias[num]["nick"]
def enable_relay(self, num):
"""
Enable a relay for this network.
Send a command to ZNC to connect.
"""
self.relays[num]["enabled"] = True
user = main.alias[num]["nick"]
commands = {"status": ["Connect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name not in main.IRCPool.keys():
self.start_bot(num)
def disable_relay(self, num):
"""
Disable a relay for this network.
Send a command to ZNC to disconnect.
Stop trying to connect to the relay.
"""
self.relays[num]["enabled"] = False
user = main.alias[num]["nick"]
# relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying()
main.ReactorPool[name].disconnect()
if name in main.IRCPool.keys():
del main.IRCPool[name]
del main.ReactorPool[name]
del main.FactoryPool[name]
def killAliases(self, aliasList):
for i in aliasList:
name = self.net + str(i)
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying()
main.ReactorPool[name].disconnect()
if name in main.IRCPool.keys():
del main.IRCPool[name]
del main.ReactorPool[name]
del main.FactoryPool[name]
def delete_relay(self, id):
del self.relays[id]
del self.aliases[id]
# del main.alias[id] - Aliases are global per num, so don't delete them!
self.killAliases([id])
def seppuku(self):
# Removes all bots in preperation for deletion
self.killAliases(self.relays.keys())
nukeNetwork(self.net)
def start_bot(self, num):
# a single name is given to relays in the backend
# e.g. freenode1 for the first relay on freenode network
keyFN = main.certPath + main.config["Key"]
certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(
keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
)
bot = IRCBotFactory(self.net, num)
# host, port = self.relays[num]["host"], self.relays[num]["port"]
host, port = getRelay(num)
rct = reactor.connectSSL(host, port, bot, contextFactory)
name = self.net + str(num)
main.ReactorPool[name] = rct
main.FactoryPool[name] = bot
log("Started bot on relay %s on %s" % (num, self.host))
def start_bots(self):
for num in self.relays.keys():
if self.relays[num]["enabled"]:
self.start_bot(num)

106
legacy/modules/provision.py Normal file
View File

@ -0,0 +1,106 @@
import main
import modules.regproc
from twisted.internet import reactor
from utils.deliver_relay_commands import deliverRelayCommands
from utils.logging.log import warn
def provisionUserNetworkData(
num, nick, altnick, ident, realname, network, host, port, security
):
commands = {}
stage2commands = {}
stage2commands["status"] = []
commands["controlpanel"] = []
user = nick.lower()
commands["controlpanel"].append(
"AddUser %s %s" % (user, main.config["Relay"]["Password"])
)
commands["controlpanel"].append("AddNetwork %s %s" % (user, network))
commands["controlpanel"].append("Set Nick %s %s" % (user, nick))
commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick))
commands["controlpanel"].append("Set Ident %s %s" % (user, ident))
commands["controlpanel"].append("Set RealName %s %s" % (user, realname))
if security == "ssl":
commands["controlpanel"].append(
"SetNetwork TrustAllCerts %s %s true" % (user, network)
) # Don't judge me
commands["controlpanel"].append(
"AddServer %s %s %s +%s" % (user, network, host, port)
)
elif security == "plain":
commands["controlpanel"].append(
"AddServer %s %s %s %s" % (user, network, host, port)
)
if not main.config["ConnectOnCreate"]:
stage2commands["status"].append("Disconnect")
if main.config["Toggles"]["CycleChans"]:
stage2commands["status"].append("LoadMod disconkick")
stage2commands["status"].append("LoadMod chansaver")
inst = modules.regproc.selectInst(network)
if "setmode" in inst.keys():
stage2commands["status"].append("LoadMod perform")
# stage2commands["perform"].append("add mode %nick% +"+inst["setmode"])
deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]])
def provisionAuthenticationData(num, nick, network, auth, password):
commands = {}
commands["status"] = []
user = nick.lower()
if auth == "sasl":
commands["sasl"] = []
commands["status"].append("UnloadMod nickserv")
commands["status"].append("LoadMod sasl")
commands["sasl"].append("Mechanism plain")
commands["sasl"].append("Set %s %s" % (nick, password))
elif auth == "ns":
commands["nickserv"] = []
commands["status"].append("UnloadMod sasl")
commands["status"].append("LoadMod nickserv")
commands["nickserv"].append("Set %s" % password)
inst = modules.regproc.selectInst(network)
if "setmode" in inst.keys():
# perform is loaded above
# commands["status"].append("LoadMod perform")
commands["perform"] = ["add mode %nick% +" + inst["setmode"]]
deliverRelayCommands(num, commands, user=user + "/" + network)
def provisionRelay(num, network): # provision user and network data
aliasObj = main.alias[num]
# alias = aliasObj["nick"]
nick = aliasObj["nick"]
altnick = aliasObj["altnick"]
ident = aliasObj["ident"]
realname = aliasObj["realname"]
provisionUserNetworkData(
num,
nick,
altnick,
ident,
realname,
network,
main.network[network].host,
main.network[network].port,
main.network[network].security,
)
if main.config["ConnectOnCreate"]:
reactor.callLater(10, main.network[network].start_bot, num)
def provisionMultipleRelays(net, relaysNeeded):
if not relaysNeeded:
return []
if not main.config["ChanKeep"]["Provision"]:
warn(
f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled"
)
return []
numsProvisioned = []
for i in range(relaysNeeded):
num, alias = main.network[net].add_relay()
numsProvisioned.append(num)
provisionRelay(num, net)
main.saveConf("network")
return numsProvisioned

237
legacy/modules/regproc.py Normal file
View File

@ -0,0 +1,237 @@
from copy import deepcopy
from random import choice
import main
from modules import provision
from utils.logging.debug import debug
from utils.logging.log import error
def needToRegister(net):
# Check if the network does not support authentication
networkObj = main.network[net]
if networkObj.auth == "none":
return False
# Check if the IRC network definition has registration disabled
inst = selectInst(net)
if "register" in inst.keys():
if inst["register"]:
return True
else:
return False
def needToAuth(net):
networkObj = main.network[net]
if networkObj.auth == "none":
return False
return True
def selectInst(net):
if net in main.irc.keys():
inst = deepcopy(main.irc[net])
for i in main.irc["_"].keys():
if i not in inst:
inst[i] = main.irc["_"][i]
else:
inst = deepcopy(main.irc["_"])
return inst
def substitute(net, num, token=None):
inst = selectInst(net)
alias = main.alias[num]
gotemail = False
if "emails" in alias:
# First priority is explicit email lists
if alias["emails"]:
email = choice(alias["emails"])
gotemail = True
if "domains" in inst:
if inst["domains"]:
if not gotemail:
domain = choice(inst["domains"])
email = f"{alias['nickname']}@{domain}"
gotemail = True
if not gotemail:
inst["email"] = False
nickname = alias["nick"]
# username = nickname + "/" + net
password = main.network[net].aliases[num]["password"]
# inst["email"] = inst["email"].replace("{nickname}", nickname)
name = f"{net}{num}"
if name in main.IRCPool:
curnick = main.IRCPool[name].nickname
else:
curnick = nickname
for i in inst.keys():
if not isinstance(inst[i], str):
continue
inst[i] = inst[i].replace("{nickname}", nickname)
inst[i] = inst[i].replace("{curnick}", curnick)
inst[i] = inst[i].replace("{password}", password)
if gotemail:
inst[i] = inst[i].replace("{email}", email)
if token:
inst[i] = inst[i].replace("{token}", token)
return inst
def registerAccount(net, num):
debug("Attempting to register: %s - %i" % (net, num))
sinst = substitute(net, num)
if not sinst:
error(f"Register account failed for {net} - {num}")
return
if not sinst["email"]:
error(f"Could not get email for {net} - {num}")
return
if not sinst["register"]:
error("Cannot register for %s: function disabled" % (net))
return False
name = net + str(num)
if not main.IRCPool[name].authenticated:
main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"])
def confirmAccount(net, num, token):
sinst = substitute(net, num, token=token)
name = net + str(num)
if name in main.IRCPool:
main.IRCPool[name].msg(sinst["entity"], sinst["confirm"])
enableAuthentication(net, num)
def confirmRegistration(net, num, negativepass=None):
obj = main.network[net]
name = net + str(num)
if name in main.IRCPool.keys():
if negativepass is not None:
main.IRCPool[name].regPing(negativepass=negativepass)
return
debug("Relay authenticated: %s - %i" % (net, num))
main.IRCPool[name].authenticated = True
main.IRCPool[name].recheckList()
if obj.relays[num]["registered"]:
return
if name in main.IRCPool.keys():
if main.IRCPool[name]._regAttempt:
try:
main.IRCPool[name]._regAttempt.cancel()
except: # noqa
pass
obj.relays[num]["registered"] = True
main.saveConf("network")
def attemptManualAuthentication(net, num):
sinst = substitute(net, num)
identifymsg = sinst["identifymsg"]
entity = sinst["entity"]
name = f"{net}{num}"
if name not in main.IRCPool:
return
main.IRCPool[name].sendmsg(entity, identifymsg, in_query=True)
def enableAuthentication(net, num, jump=True, run_now=False):
obj = main.network[net]
nick = main.alias[num]["nick"]
auth = obj.auth
name = f"{net}{num}"
if name not in main.IRCPool:
return
# uname = main.alias[num]["nick"] + "/" + net
password = main.network[net].aliases[num]["password"]
provision.provisionAuthenticationData(
num, nick, net, auth, password
) # Set up for auth
if jump:
main.IRCPool[name].msg(
main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump"
)
if run_now:
attemptManualAuthentication(net, num)
if selectInst(net)["check"] is False:
confirmRegistration(net, num)
def get_unregistered_relays(net=None):
"""
Get a dict of unregistereed relays, either globally or
for a network.
Returns:
{"net": [["nick1", 1], ["nick2", 2], ...]}
"""
unreg = {}
if net:
nets = [net]
else:
nets = main.network.keys()
for i in nets:
for num in main.network[i].relays.keys():
if not main.network[i].relays[num]["registered"]:
nick = main.alias[num]["nick"]
if i in unreg:
unreg[i].append([nick, num])
else:
unreg[i] = [[nick, num]]
return unreg
def registerTest(c):
sinst = substitute(c["net"], c["num"])
name = c["net"] + str(c["num"])
net = c["net"]
num = c["num"]
if sinst["check"] is False:
return
if "msg" in c.keys() and not c["msg"] is None:
if sinst["negative"]:
if name in main.IRCPool.keys():
if main.IRCPool[name]._negativePass is not True:
if c["type"] == "query" and c["nick"] == sinst["entity"]:
if sinst["checknegativemsg"] in c["msg"]:
confirmRegistration(
c["net"], c["num"], negativepass=False
) # Not passed negative check, report back
return
if sinst["checkendnegative"] in c["msg"]:
confirmRegistration(
c["net"], c["num"], negativepass=True
) # Passed the negative check, report back
return
if sinst["ping"]:
if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]:
confirmRegistration(c["net"], c["num"])
debug(
(
f"registerTest() {net} - {num} passed ping:checkmsg2 "
f"check, {sinst['checkmsg2']} present in message"
)
)
return
if sinst["checktype"] == "msg":
if sinst["checkmsg"] in c["msg"]:
confirmRegistration(c["net"], c["num"])
debug(
(
f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, "
f"{sinst['checkmsg']} present in message"
)
)
return
elif sinst["checktype"] == "mode":
if c["type"] == "self":
if c["mtype"] == "mode":
if sinst["checkmode"] in c["mode"] and c["status"] is True:
confirmRegistration(c["net"], c["num"])
debug(
(
f"registerTest() {net} - {num} passed checktype:mode:checkmost check, "
f"{sinst['checkmode']} present in mode"
)
)
return

182
modules/userinfo.py → legacy/modules/userinfo.py
View File

@ -1,17 +1,18 @@
from twisted.internet.threads import deferToThread
from string import digits
import main
from utils.logging.log import *
from utils.logging.debug import debug
from modules import chankeep
from twisted.internet.threads import deferToThread
from utils.logging.debug import debug, trace
from utils.logging.log import warn
from utils.parsing import parsen
def getWhoSingle(name, query):
result = main.r.sscan("live.who."+name, 0, query, count=999999)
result = main.r.sscan("live.who." + name, 0, query, count=999999)
if result[1] == []:
return None
return (i.decode() for i in result[1])
def getWho(query):
result = {}
for i in main.network.keys():
@ -20,20 +21,53 @@ def getWho(query):
result[i] = f
return result
def getChansSingle(name, nick):
nick = ["live.chan."+name+"."+i for i in nick]
nick = ("live.chan." + name + "." + i for i in nick)
result = main.r.sinter(*nick)
if len(result) == 0:
return None
return (i.decode() for i in result)
def getChanList(name, nick):
chanspace = "live.chan."+name+"."+nick
chanspace = "live.chan." + name + "." + nick
result = main.r.smembers(chanspace)
if len(result) == 0:
return None
return (i.decode() for i in result)
def getTotalChanNum(net):
"""
Get the number of channels a network has.
"""
chans = main.r.keys(f"live.who.{net}.*")
return len(chans)
def getUserNum(name, channels):
"""
Get the number of users on a list of channels.
"""
chanspace = ("live.who." + name + "." + i for i in channels)
results = {}
for channel, space in zip(channels, chanspace):
results[channel] = main.r.scard(space)
return results
def getChanNum(name, nicks):
"""
Get the number of channels a list of users are on.
"""
nickspace = ("live.chan." + name + "." + i for i in nicks)
results = {}
for nick, space in zip(nicks, nickspace):
results[nick] = main.r.scard(space)
return results
def getChans(nick):
result = {}
for i in main.network.keys():
@ -42,13 +76,15 @@ def getChans(nick):
result[i] = f
return result
def getUsersSingle(name, nick):
nick = ("live.who."+name+"."+i for i in nick)
nick = ("live.who." + name + "." + i for i in nick)
result = main.r.sinter(*nick)
if len(result) == 0:
return None
return (i.decode() for i in result)
def getUsers(nick):
result = {}
for i in main.network.keys():
@ -57,8 +93,10 @@ def getUsers(nick):
result[i] = f
return result
def getNumWhoEntries(name):
return main.r.scard("live.who."+name)
return main.r.scard("live.who." + name)
def getNumTotalWhoEntries():
total = 0
@ -66,6 +104,7 @@ def getNumTotalWhoEntries():
total += getNumWhoEntries(i)
return total
def getNamespace(name, channel, nick):
gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel)
@ -73,33 +112,40 @@ def getNamespace(name, channel, nick):
mapspace = "live.map.%s" % name
return (gnamespace, namespace, chanspace, mapspace)
def _initialUsers(name, channel, users):
gnamespace = "live.who.%s" % name
mapspace = "live.map.%s" % name
p = main.r.pipeline()
for i in users:
user = i[0]+"!"+i[1]+"@"+i[2]
user = i[0] + "!" + i[1] + "@" + i[2]
p.hset(mapspace, i[0], user)
p.sadd(gnamespace, user)
p.execute()
def initialUsers(name, channel, users):
debug("Initialising WHO records for %s on %s" % (channel, name))
d = deferToThread(_initialUsers, name, channel, users)
#d.addCallback(testCallback)
trace("Initialising WHO records for %s on %s" % (channel, name))
deferToThread(_initialUsers, name, channel, users)
# d.addCallback(testCallback)
def _initialNames(name, channel, names):
namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline()
for i in names:
p.sadd(namespace, i)
p.sadd("live.chan."+name+"."+i, channel)
for mode, nick in names:
p.sadd(namespace, nick)
p.sadd("live.chan." + name + "." + nick, channel)
if mode:
p.hset("live.prefix." + name + "." + channel, nick, mode)
p.execute()
def initialNames(name, channel, names):
debug("Initialising NAMES records for %s on %s" % (channel, name))
d = deferToThread(_initialNames, name, channel, names)
#d.addCallback(testCallback)
trace("Initialising NAMES records for %s on %s" % (channel, name))
deferToThread(_initialNames, name, channel, names)
# d.addCallback(testCallback)
def editUser(name, user):
gnamespace = "live.who.%s" % name
@ -107,9 +153,10 @@ def editUser(name, user):
parsed = parsen(user)
p = main.r.pipeline()
p.sadd(gnamespace, user)
p.hset(mapspace, parsed[0], user) # add nick -> user mapping
p.hset(mapspace, parsed[0], user) # add nick -> user mapping
p.execute()
def addUser(name, channel, nick, user):
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
@ -119,40 +166,50 @@ def addUser(name, channel, nick, user):
p.hset(mapspace, nick, user)
p.execute()
def delUser(name, channel, nick, user):
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
channels = main.r.smembers(chanspace)
p.srem(namespace, nick)
if channels == {channel.encode()}: # can we only see them on this channel?
p.delete(chanspace) # remove channel tracking entry
p.hdel(mapspace, nick) # remove nick mapping entry
if channels == {channel.encode()}: # can we only see them on this channel?
p.delete(chanspace) # remove channel tracking entry
p.hdel(
"live.prefix." + name + "." + channel, nick
) # remove prefix tracking entry
p.hdel(mapspace, nick) # remove nick mapping entry
if user:
p.srem(gnamespace, user) # remove global userinfo entry
p.srem(gnamespace, user) # remove global userinfo entry
else:
warn("Attempt to delete nonexistent user: %s" % user)
else:
p.srem(chanspace, channel) # keep up - remove the channel from their list
p.srem(chanspace, channel) # keep up - remove the channel from their list
p.execute()
def escape(text):
chars = ["[", "]", "^", "-", "*", "?"]
text = text.replace("\\", "\\\\")
for i in chars:
text = text.replace(i, "\\"+i)
text = text.replace(i, "\\" + i)
return text
def getUserByNick(name, nick):
gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
mapspace = "live.map.%s" % name
if main.r.hexists(mapspace, nick):
return main.r.hget(mapspace, nick)
else:
warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace))
#return Falsedd
warn(
"Entry doesn't exist: %s on %s - attempting auxiliary lookup"
% (nick, mapspace)
)
# return False
# legacy code below - remove when map is reliable
usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1)
usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999)
if usermatch[1] == []:
warn("No matches found for user query: %s on %s" % (nick, name))
return False
else:
if len(usermatch[1]) == 1:
@ -162,6 +219,7 @@ def getUserByNick(name, nick):
warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
return False
def renameUser(name, oldnick, olduser, newnick, newuser):
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, oldnick)
@ -172,32 +230,51 @@ def renameUser(name, oldnick, olduser, newnick, newuser):
p.sadd(gnamespace, newuser)
for i in main.r.smembers(chanspace):
i = i.decode()
p.srem("live.who."+name+"."+i, oldnick)
p.sadd("live.who."+name+"."+i, newnick)
p.srem("live.who." + name + "." + i, oldnick)
p.sadd("live.who." + name + "." + i, newnick)
p.hdel(mapspace, oldnick)
p.hset(mapspace, newnick, newuser)
if main.r.exists(
"live.prefix." + name + "." + i
): # if there's a prefix entry for the channel
if main.r.hexists(
"live.prefix." + name + "." + i, oldnick
): # if the old nick is in it
mode = main.r.hget(
"live.prefix." + name + "." + i, oldnick
) # retrieve old modes
p.hset(
"live.prefix." + name + "." + i, newnick, mode
) # set old modes to new nickname
if main.r.exists(chanspace):
p.rename(chanspace, newchanspace)
else:
warn("Key doesn't exist: %s" % chanspace)
p.execute()
def delUserByNick(name, channel, nick): # kick
def delUserByNick(name, channel, nick): # kick
user = getUserByNick(name, nick)
if not user:
return
delUser(name, channel, nick, user)
def delUserByNetwork(name, nick, user): # quit
def delUserByNetwork(name, nick, user): # quit
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, nick)
mapspace = "live.chan.%s" % name
p = main.r.pipeline()
p.srem(gnamespace, user)
for i in main.r.smembers(chanspace):
p.srem("live.who."+name+"."+i.decode(), nick)
p.srem("live.who." + name + "." + i.decode(), nick)
p.hdel("live.prefix." + name + "." + i.decode(), nick)
p.delete(chanspace)
p.hdel(mapspace, nick)
p.execute()
def _delChannels(net, channels):
gnamespace = "live.who.%s" % net
mapspace = "live.map.%s" % net
@ -206,21 +283,36 @@ def _delChannels(net, channels):
namespace = "live.who.%s.%s" % (net, channel)
for i in main.r.smembers(namespace):
nick = i.decode()
#user = getUserByNick(net, nick) -- far too many function calls
# user = getUserByNick(net, nick) -- far too many function calls
user = main.r.hget(mapspace, nick)
if not user:
warn("User lookup failed: %s on %s" % (nick, net))
if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}:
if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}:
if user:
p.srem(gnamespace, user)
p.delete("live.chan."+net+"."+nick)
p.hdel(mapspace, nick) # remove map entry
p.delete("live.chan." + net + "." + nick)
p.hdel(mapspace, nick) # remove map entry
else:
p.srem("live.chan."+net+"."+nick, channel)
p.srem("live.chan." + net + "." + nick, channel)
p.delete(namespace)
p.delete("live.prefix." + net + "." + channel)
p.execute()
def delChannels(net, channels):
debug("Purging channel %s for %s" % (", ".join(channels), net))
d = deferToThread(_delChannels, net, channels)
#d.addCallback(testCallback)
def delChannels(net, channels): # we have left a channel
trace("Purging channel %s for %s" % (", ".join(channels), net))
dupes = chankeep.getDuplicateChannels(net, total=True)
print("dupes: %s" % dupes)
if not dupes:
deferToThread(_delChannels, net, channels)
else:
for channel in channels:
if channel in dupes[net]:
if dupes[net][channel] != 0:
channels.remove(channel)
debug(
f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it"
)
deferToThread(_delChannels, net, channels)
# d.addCallback(testCallback)

9
legacy/requirements.txt Normal file
View File

@ -0,0 +1,9 @@
wheel
pre-commit
twisted
pyOpenSSL
redis
pyYaML
service_identity
siphashc
Klein

3
legacy/runtest.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
#pre-commit run -a
python -m unittest discover -s tests -p 'test_*.py'

17
legacy/stack.env Normal file
View File

@ -0,0 +1,17 @@
THRESHOLD_LISTENER_HOST=0.0.0.0
THRESHOLD_LISTENER_PORT=13867
THRESHOLD_LISTENER_SSL=1
THRESHOLD_RELAY_ENABLED=0
THRESHOLD_RELAY_HOST=0.0.0.0
THRESHOLD_RELAY_PORT=13868
THRESHOLD_RELAY_SSL=1
THRESHOLD_API_ENABLED=1
THRESHOLD_API_HOST=0.0.0.0
THRESHOLD_API_PORT=13869
PORTAINER_GIT_DIR=..
THRESHOLD_CONFIG_DIR=../conf/live/
THRESHOLD_TEMPLATE_DIR=../conf/example/
THRESHOLD_CERT_DIR=../conf/cert/

119
legacy/tests/test_chankeep.py Normal file
View File

@ -0,0 +1,119 @@
from math import ceil
from random import randint
from unittest import TestCase
from unittest.mock import MagicMock, patch
from modules import chankeep
class TestChanKeep(TestCase):
def setUp(self):
self.net = "testnet"
self.num = 1
self.chanlimit = 100
chankeep.main.initConf()
chankeep.main.r = MagicMock()
chankeep.main.g = MagicMock()
chankeep.main.g.pipeline = MagicMock()
chankeep.main.config["ChanKeep"]["Provision"] = False
chankeep.getAverageChanlimit = MagicMock()
chankeep.getAverageChanlimit.return_value = self.chanlimit
self.listinfo = self.generate_listinfo()
self.chan_name_list = [x[0] for x in self.listinfo]
self.chan_member_list = [x[1] for x in self.listinfo]
def generate_listinfo(self, ranges=None):
"""
Create a fake listinfo.
Where #channel has 192 users, and #channel2 has 188 users.
listinfo = [["#channel", 192], ["#channel2", 188]]
"""
if not ranges:
ranges = [[100, 5, 10], [400, 100, 200], [2, 500, 1000]]
listinfo = []
for channum, min, max in ranges:
for i in range(channum):
chan_name = f"#num-{channum}-{i}"
chan_users = randint(min, max)
listinfo.append([chan_name, chan_users])
return listinfo
def percent_diff(self, a, b):
return (abs(b - a) / a) * 100.0
def test_alt_listinfo(self):
# We're looking for a perc of 1000-1100
# And a sigrelay of 2
# We only want those 10 big channels
instances = 1
chanlimit = 5
max_chans = instances * chanlimit
listinfo = self.generate_listinfo(
ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]]
)
# listinfo_num = [x[1] for x in listinfo]
listlength = len(listinfo)
cumul = 0
try:
cumul += sum(int(i[1]) for i in listinfo)
except TypeError:
return
mean = round(cumul / listlength, 2)
siglength = 0
insiglength = 0
sigcumul = 0
insigcumul = 0
for i in listinfo:
if int(i[1]) > mean:
siglength += 1
sigcumul += int(i[1])
elif int(i[1]) < mean:
insiglength += 1
insigcumul += int(i[1])
sigrelay = ceil(siglength / chanlimit)
relay = ceil(listlength / chanlimit)
print(
(
f"len:{listlength} cumul:{cumul} mean:{mean} "
f"siglength:{siglength} insiglength:{insiglength} "
f"sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} "
f"insigcumul:{insigcumul}"
)
)
# We want a return between 1000 and 1100
# list_insig = [x for x in listinfo_num if x < mean]
list_sig = [x for x in listinfo if x[1] > mean]
chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans]
self.assertEqual(len(chosen), 5)
@patch("modules.chankeep.keepChannels")
def test__initialList(self, keepchannels):
chankeep._initialList(self.net, self.num, self.listinfo)
net, passed_list, mean, sigrelay, relay = keepchannels.call_args_list[0][0]
self.assertEqual(net, self.net)
self.assertEqual(passed_list, self.listinfo)
# self.assertEqual(chanlimit, self.chanlimit)
# print(net, mean, sigrelay, relay)
@patch("modules.chankeep.getChanFree")
def test_empty_chan_allocate(self, getchanfree):
getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free
eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list)
self.assertEqual(len(eca), 1)
num = list(eca.keys())[0]
chans = eca[list(eca.keys())[0]]
self.assertEqual(num, self.num)
self.assertCountEqual(chans, self.chan_name_list)
getchanfree.return_value = ({1: 100}, 10)
eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list)
self.assertEqual(len(eca), 1)
num = list(eca.keys())[0]
chans = eca[list(eca.keys())[0]]
self.assertEqual(num, self.num)
self.assertEqual(len(chans), 100)
# self.assertCountEqual(chans, self.chan_name_list)

125
legacy/threshold Executable file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env python
import sys
from codecs import getwriter
from os import getenv
from signal import SIGINT, signal
from sys import stderr, stdout
import main
import modules.counters
from api.views import API
from core.relay import RelayFactory
from core.server import ServerFactory
from twisted.internet import reactor
# Webapp stuff
from twisted.internet.protocol import Factory
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from utils.cleanup import handler
from utils.loaders.command_loader import loadCommands
from utils.logging.log import log
Factory.noisy = False
main.initMain()
if "--debug" in sys.argv: # yes really
main.config["Debug"] = True
if "--trace" in sys.argv:
main.config["Trace"] = True
if "--migrate" in sys.argv:
from modules.network import migrate
migrate()
exit()
loadCommands()
# core.logstash.init_logstash()
signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine
stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know
stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using
# unicode quit messages for no reason
trues = ("true", "1", "t", True)
# Main listener
listener_address = getenv("THRESHOLD_LISTENER_HOST", main.config["Listener"]["Address"])
listener_port = int(getenv("THRESHOLD_LISTENER_PORT", main.config["Listener"]["Port"]))
listener_ssl = (
getenv("THRESHOLD_LISTENER_SSL", main.config["Listener"]["UseSSL"]) in trues
)
# RelayAPI
relay_enabled = (
getenv("THRESHOLD_RELAY_ENABLED", main.config["RelayAPI"]["Enabled"]) in trues
)
relay_address = getenv("THRESHOLD_RELAY_HOST", main.config["RelayAPI"]["Address"])
relay_port = int(getenv("THRESHOLD_RELAY_PORT", main.config["RelayAPI"]["Port"]))
relay_ssl = getenv("THRESHOLD_RELAY_SSL", main.config["RelayAPI"]["UseSSL"]) in trues
# Web API
api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in trues
api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"])
api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"]))
# Debugging
debug_enabled = getenv("THRESHOLD_DEBUG", main.config["Debug"]) in trues
trace_enabled = getenv("THRESHOLD_TRACE", main.config["Trace"]) in trues
if debug_enabled:
main.config["Debug"] = True
if trace_enabled:
main.config["Trace"] = True
if __name__ == "__main__":
listener = ServerFactory()
if listener_ssl is True:
reactor.listenSSL(
listener_port,
listener,
DefaultOpenSSLContextFactory(
main.certPath + main.config["Key"],
main.certPath + main.config["Certificate"],
),
interface=listener_address,
)
log("Threshold running with SSL on %s:%s" % (listener_address, listener_port))
else:
reactor.listenTCP(
listener_port,
listener,
interface=listener_address,
)
log("Threshold running on %s:%s" % (listener_address, listener_port))
if relay_enabled:
relay = RelayFactory()
if relay_ssl is True:
reactor.listenSSL(
relay_port,
relay,
DefaultOpenSSLContextFactory(
main.certPath + main.config["Key"],
main.certPath + main.config["Certificate"],
),
interface=relay_address,
)
log(
"Threshold relay running with SSL on %s:%s"
% (relay_address, relay_port)
)
else:
reactor.listenTCP(
relay_port,
relay,
interface=relay_address,
)
log("Threshold relay running on %s:%s" % (relay_address, relay_port))
for net in main.network.keys():
main.network[net].start_bots()
modules.counters.setupCounterLoop()
if api_enabled:
api = API()
api.app.run(api_address, api_port)
else:
reactor.run()

7
utils/cleanup.py → legacy/utils/cleanup.py
View File

@ -1,15 +1,16 @@
import main
from twisted.internet import reactor
from utils.logging.debug import debug
from utils.logging.log import *
import sys
from utils.logging.log import log
def handler(sig, frame):
log("Received SIGINT, cleaning up")
cleanup()
def cleanup():
debug("Flushing Redis database")
main.r.flushdb()
reactor.stop()
#sys.exit(1)
# sys.exit(1)

34
legacy/utils/dedup.py Normal file
View File

@ -0,0 +1,34 @@
from copy import deepcopy
from datetime import datetime
from json import dumps
import main
from siphashc import siphash
from utils.logging.debug import debug
def dedup(numName, b):
c = deepcopy(b)
if "ts" in c.keys():
del c["ts"]
c["approxtime"] = str(datetime.utcnow().timestamp())[
: main.config["Tweaks"]["DedupPrecision"]
]
castHash = siphash(main.hashKey, dumps(c, sort_keys=True))
del c["approxtime"]
isDuplicate = any(
castHash in main.lastEvents[x]
for x in main.lastEvents.keys()
if not x == numName
)
if isDuplicate:
debug("Duplicate: %s" % (c))
return True
if numName in main.lastEvents.keys():
main.lastEvents[numName].insert(0, castHash)
main.lastEvents[numName] = main.lastEvents[numName][
0 : main.config["Tweaks"]["MaxHash"]
] # noqa
else:
main.lastEvents[numName] = [castHash]
return False

Some files were not shown because too many files have changed in this diff Show More