Implement indexing into Apache Druid #1
|
|
@ -1,13 +1,161 @@
|
||||||
*.pyc
|
# ---> Python
|
||||||
*.pem
|
# Byte-compiled / optimized / DLL files
|
||||||
*.swp
|
|
||||||
__pycache__/
|
__pycache__/
|
||||||
conf/config.json
|
*.py[cod]
|
||||||
conf/wholist.json
|
*$py.class
|
||||||
conf/counters.json
|
|
||||||
conf/monitor.json
|
# C extensions
|
||||||
conf/tokens.json
|
*.so
|
||||||
conf/network.dat
|
|
||||||
conf/alias.json
|
# Distribution / packaging
|
||||||
conf/dist.sh
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
# lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
#poetry.lock
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
env/
|
env/
|
||||||
|
venv/
|
||||||
|
env-glibc/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
.idea/
|
||||||
|
|
||||||
|
.bash_history
|
||||||
|
.vscode/
|
||||||
|
docker/data
|
||||||
|
*.pem
|
||||||
|
legacy/conf/live/
|
||||||
|
legacy/conf/cert/
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/psf/black
|
||||||
|
rev: 22.6.0
|
||||||
|
hooks:
|
||||||
|
- id: black
|
||||||
|
- repo: https://github.com/PyCQA/isort
|
||||||
|
rev: 5.10.1
|
||||||
|
hooks:
|
||||||
|
- id: isort
|
||||||
|
args: ["--profile", "black"]
|
||||||
|
- repo: https://github.com/PyCQA/flake8
|
||||||
|
rev: 4.0.1
|
||||||
|
hooks:
|
||||||
|
- id: flake8
|
||||||
|
args: [--max-line-length=88]
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
import main
|
|
||||||
|
|
||||||
class ListCommand:
|
|
||||||
def __init__(self, *args):
|
|
||||||
self.list(*args)
|
|
||||||
|
|
||||||
def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
|
||||||
if authed:
|
|
||||||
if length == 2:
|
|
||||||
if not spl[1] in main.network.keys():
|
|
||||||
failure("No such network: %s" % spl[1])
|
|
||||||
return
|
|
||||||
if not 1 in main.network[spl[1]].relays.keys():
|
|
||||||
failure("Network has no first instance")
|
|
||||||
return
|
|
||||||
if not spl[1]+"1" in main.IRCPool.keys():
|
|
||||||
failure("No IRC instance: %s - 1" % spl[1])
|
|
||||||
return
|
|
||||||
main.IRCPool[spl[1]+"1"].list()
|
|
||||||
success("Requested list with first instance of %s" % spl[1])
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
incUsage("list")
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
incUsage(None)
|
|
||||||
280
commands/mon.py
280
commands/mon.py
|
|
@ -1,280 +0,0 @@
|
||||||
import main
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
from io import StringIO
|
|
||||||
from yaml import dump
|
|
||||||
|
|
||||||
class MonCommand:
|
|
||||||
def __init__(self, *args):
|
|
||||||
self.mon(*args)
|
|
||||||
|
|
||||||
def setup_arguments(self, ArgumentParser):
|
|
||||||
self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.")
|
|
||||||
group1 = self.parser.add_mutually_exclusive_group(required=True)
|
|
||||||
group1.add_argument("-a", "--add", metavar="entry", dest="addEntry", help="Add an entry")
|
|
||||||
group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry")
|
|
||||||
group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries")
|
|
||||||
group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry")
|
|
||||||
|
|
||||||
group2 = self.parser.add_mutually_exclusive_group()
|
|
||||||
group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing")
|
|
||||||
group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing")
|
|
||||||
|
|
||||||
self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values")
|
|
||||||
|
|
||||||
self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who")
|
|
||||||
self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching")
|
|
||||||
self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching")
|
|
||||||
self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching")
|
|
||||||
self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching")
|
|
||||||
self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who")
|
|
||||||
|
|
||||||
self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)")
|
|
||||||
self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action")
|
|
||||||
self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick")
|
|
||||||
self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode")
|
|
||||||
self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode")
|
|
||||||
self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who")
|
|
||||||
self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who")
|
|
||||||
self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)")
|
|
||||||
|
|
||||||
def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
|
||||||
# We need to override the ArgumentParser class because
|
|
||||||
# it's only meant for CLI applications and exits
|
|
||||||
# after running, so we override the exit function
|
|
||||||
# to do nothing. We also need to do it here in order
|
|
||||||
# to catch the error messages and show them to the user.
|
|
||||||
# It sucks. I know.
|
|
||||||
if authed:
|
|
||||||
class ArgumentParser(argparse.ArgumentParser):
|
|
||||||
def exit(self, status=0, message=None):
|
|
||||||
if message:
|
|
||||||
failure(message)
|
|
||||||
self.setup_arguments(ArgumentParser)
|
|
||||||
if length == 1:
|
|
||||||
info("Incorrect usage, use mon -h for help")
|
|
||||||
return
|
|
||||||
old_stdout = sys.stdout
|
|
||||||
old_stderr = sys.stderr
|
|
||||||
my_stdout = sys.stdout = StringIO()
|
|
||||||
my_stderr = sys.stderr = StringIO()
|
|
||||||
try:
|
|
||||||
parsed = self.parser.parse_args(spl[1:])
|
|
||||||
except:
|
|
||||||
return
|
|
||||||
sys.stdout = old_stdout
|
|
||||||
sys.stderr = old_stderr
|
|
||||||
stdout = my_stdout.getvalue()
|
|
||||||
stderr = my_stdout.getvalue()
|
|
||||||
if not stdout == "":
|
|
||||||
info(stdout)
|
|
||||||
elif not stderr == "":
|
|
||||||
failure(my_stdout.getvalue())
|
|
||||||
return
|
|
||||||
my_stdout.close()
|
|
||||||
my_stderr.close()
|
|
||||||
|
|
||||||
if parsed.addEntry:
|
|
||||||
if parsed.addEntry in main.monitor.keys():
|
|
||||||
failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry)
|
|
||||||
return
|
|
||||||
cast = self.makeCast(parsed, failure, info)
|
|
||||||
if cast == False:
|
|
||||||
return
|
|
||||||
main.monitor[parsed.addEntry] = cast
|
|
||||||
main.saveConf("monitor")
|
|
||||||
success("Successfully created monitor group %s" % parsed.addEntry)
|
|
||||||
return
|
|
||||||
elif parsed.delEntry:
|
|
||||||
if not parsed.delEntry in main.monitor.keys():
|
|
||||||
failure("No such monitor group: %s" % parsed.delEntry)
|
|
||||||
return
|
|
||||||
del main.monitor[parsed.delEntry]
|
|
||||||
main.saveConf("monitor")
|
|
||||||
success("Successfully removed monitor group: %s" % parsed.delEntry)
|
|
||||||
return
|
|
||||||
elif parsed.modEntry:
|
|
||||||
if not parsed.doAppend and not parsed.doRemove:
|
|
||||||
failure("Specify --append or --remove with --mod")
|
|
||||||
return
|
|
||||||
if not parsed.modEntry in main.monitor.keys():
|
|
||||||
failure("No such monitor group: %s" % parsed.modEntry)
|
|
||||||
return
|
|
||||||
cast = self.makeCast(parsed, failure, info)
|
|
||||||
if cast == False:
|
|
||||||
return
|
|
||||||
if parsed.doAppend:
|
|
||||||
merged = self.addCast(main.monitor[parsed.modEntry], cast, info)
|
|
||||||
main.monitor[parsed.modEntry] = merged
|
|
||||||
elif parsed.doRemove:
|
|
||||||
merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info)
|
|
||||||
if merged == {}:
|
|
||||||
del main.monitor[parsed.modEntry]
|
|
||||||
info("Group %s deleted due to having no attributes" % parsed.modEntry)
|
|
||||||
main.saveConf("monitor")
|
|
||||||
return
|
|
||||||
main.monitor[parsed.modEntry] = merged
|
|
||||||
main.saveConf("monitor")
|
|
||||||
success("Successfully updated entry %s" % parsed.modEntry)
|
|
||||||
return
|
|
||||||
elif parsed.listEntry:
|
|
||||||
info(dump(main.monitor))
|
|
||||||
return
|
|
||||||
|
|
||||||
else:
|
|
||||||
incUsage(None)
|
|
||||||
|
|
||||||
def dedup(self, data):
|
|
||||||
if not isinstance(data, bool):
|
|
||||||
return list(set(data))
|
|
||||||
return data
|
|
||||||
|
|
||||||
def parseNetworkFormat(self, lst, failure, info):
|
|
||||||
cast = {}
|
|
||||||
if lst == None:
|
|
||||||
return "nil"
|
|
||||||
if len(lst) == 0:
|
|
||||||
failure("List has no entries")
|
|
||||||
return False
|
|
||||||
elif len(lst) > 0:
|
|
||||||
if len(lst[0]) == 0:
|
|
||||||
failure("Nested list has no entries")
|
|
||||||
return False
|
|
||||||
for i in lst:
|
|
||||||
if not i[0] in main.pool.keys():
|
|
||||||
failure("Name does not exist: %s" % i[0])
|
|
||||||
return False
|
|
||||||
if i[0] in main.IRCPool.keys():
|
|
||||||
for x in i[1:]:
|
|
||||||
if not x in main.IRCPool[i[0]].channels:
|
|
||||||
info("%s: Bot not on channel: %s" % (i[0], x))
|
|
||||||
if len(i) == 1:
|
|
||||||
cast[i[0]] = True
|
|
||||||
else:
|
|
||||||
if i[0] in cast.keys():
|
|
||||||
if not cast[i[0]] == True:
|
|
||||||
for x in self.dedup(i[1:]):
|
|
||||||
cast[i[0]].append(x)
|
|
||||||
else:
|
|
||||||
cast[i[0]] = self.dedup(i[1:])
|
|
||||||
else:
|
|
||||||
cast[i[0]] = self.dedup(i[1:])
|
|
||||||
for i in cast.keys():
|
|
||||||
deduped = self.dedup(cast[i])
|
|
||||||
cast[i] = deduped
|
|
||||||
return cast
|
|
||||||
|
|
||||||
# Create or modify a monitor group magically
|
|
||||||
def makeCast(self, obj, failure, info):
|
|
||||||
validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"]
|
|
||||||
cast = {}
|
|
||||||
if not obj.specType == None:
|
|
||||||
types = self.dedup(obj.specType)
|
|
||||||
for i in types:
|
|
||||||
if not i in validTypes:
|
|
||||||
failure("Invalid type: %s" % i)
|
|
||||||
info("Available types: %s" % ", ".join(validTypes))
|
|
||||||
return False
|
|
||||||
cast["type"] = types
|
|
||||||
if not obj.source == None:
|
|
||||||
sourceParse = self.parseNetworkFormat(obj.source, failure, info)
|
|
||||||
if not sourceParse:
|
|
||||||
return False
|
|
||||||
if not sourceParse == "nil":
|
|
||||||
cast["sources"] = sourceParse
|
|
||||||
|
|
||||||
if not obj.send == None:
|
|
||||||
sendParse = self.parseNetworkFormat(obj.send, failure, info)
|
|
||||||
if not sendParse:
|
|
||||||
return False
|
|
||||||
if not sendParse == "nil":
|
|
||||||
cast["send"] = sendParse
|
|
||||||
|
|
||||||
if not obj.message == None:
|
|
||||||
cast["message"] = []
|
|
||||||
for i in obj.message:
|
|
||||||
cast["message"].append(" ".join(i))
|
|
||||||
|
|
||||||
if not obj.user == None:
|
|
||||||
cast["user"] = obj.user
|
|
||||||
if not obj.modes == None:
|
|
||||||
cast["modes"] = obj.modes
|
|
||||||
if not obj.modeargs == None:
|
|
||||||
cast["modeargs"] = obj.modeargs
|
|
||||||
if not obj.server == None:
|
|
||||||
cast["server"] = obj.server
|
|
||||||
if not obj.status == None:
|
|
||||||
cast["status"] = obj.status
|
|
||||||
if not obj.free == None:
|
|
||||||
cast["free"] = obj.free
|
|
||||||
if not obj.nick == None:
|
|
||||||
cast["nick"] = obj.nick
|
|
||||||
if not obj.ident == None:
|
|
||||||
cast["ident"] = obj.ident
|
|
||||||
if not obj.host == None:
|
|
||||||
cast["host"] = obj.host
|
|
||||||
if not obj.real == None:
|
|
||||||
cast["real"] = []
|
|
||||||
for i in obj.real:
|
|
||||||
cast["real"].append(" ".join(i))
|
|
||||||
if not obj.inside == None:
|
|
||||||
if obj.inside.lower() in ["yes", "true", "on"]:
|
|
||||||
cast["inside"] = True
|
|
||||||
elif obj.inside.lower() in ["no", "false", "off"]:
|
|
||||||
cast["inside"] = False
|
|
||||||
else:
|
|
||||||
failure("inside: Unknown operand: %s" % obj.inside)
|
|
||||||
return
|
|
||||||
return cast
|
|
||||||
|
|
||||||
def subtractCast(self, source, patch, info):
|
|
||||||
for i in patch.keys():
|
|
||||||
if i in source.keys():
|
|
||||||
if isinstance(source[i], dict):
|
|
||||||
result = self.subtractCast(source[i], patch[i], info)
|
|
||||||
if result == {}:
|
|
||||||
info("Removing upper element: %s" % i)
|
|
||||||
del source[i]
|
|
||||||
continue
|
|
||||||
source[i] = result
|
|
||||||
continue
|
|
||||||
if isinstance(patch[i], bool):
|
|
||||||
del source[i]
|
|
||||||
info("Removing entire element: %s" % i)
|
|
||||||
continue
|
|
||||||
for x in patch[i]:
|
|
||||||
if isinstance(source[i], bool):
|
|
||||||
info("Attempt to remove %s from network-wide definition" % x)
|
|
||||||
continue
|
|
||||||
if x in source[i]:
|
|
||||||
source[i].remove(x)
|
|
||||||
if source[i] == []:
|
|
||||||
del source[i]
|
|
||||||
else:
|
|
||||||
info("Element %s not in source %s" % (x, i))
|
|
||||||
else:
|
|
||||||
info("Non-matched key: %s" % i)
|
|
||||||
return source
|
|
||||||
|
|
||||||
def addCast(self, source, patch, info):
|
|
||||||
for i in patch.keys():
|
|
||||||
if i in source.keys():
|
|
||||||
if isinstance(source[i], dict):
|
|
||||||
result = self.addCast(source[i], patch[i], info)
|
|
||||||
source[i] = result
|
|
||||||
continue
|
|
||||||
if isinstance(patch[i], bool):
|
|
||||||
source[i] = patch[i]
|
|
||||||
info("Overriding local element %s with network-wide definition" % i)
|
|
||||||
continue
|
|
||||||
for x in patch[i]:
|
|
||||||
if isinstance(source[i], bool):
|
|
||||||
source[i] = []
|
|
||||||
info("Overriding element %s, previously set network-wide" % i)
|
|
||||||
if x in source[i]:
|
|
||||||
info("Element %s already in source %s" % (x, i))
|
|
||||||
else:
|
|
||||||
source[i].append(x)
|
|
||||||
else:
|
|
||||||
source[i] = patch[i]
|
|
||||||
return source
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
{}
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
{}
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
[]
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
{}
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
{}
|
|
||||||
706
core/bot.py
706
core/bot.py
|
|
@ -1,706 +0,0 @@
|
||||||
from twisted.internet.protocol import ReconnectingClientFactory
|
|
||||||
from twisted.words.protocols.irc import IRCClient
|
|
||||||
from twisted.internet.defer import Deferred
|
|
||||||
from twisted.internet.task import LoopingCall
|
|
||||||
from twisted.internet import reactor, task
|
|
||||||
from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage
|
|
||||||
|
|
||||||
import sys
|
|
||||||
from string import digits
|
|
||||||
from random import randint
|
|
||||||
from copy import deepcopy
|
|
||||||
|
|
||||||
from modules import userinfo
|
|
||||||
from modules import counters
|
|
||||||
from modules import monitor
|
|
||||||
from modules import chankeep
|
|
||||||
|
|
||||||
from core.relay import sendRelayNotification
|
|
||||||
from utils.dedup import dedup
|
|
||||||
from utils.get import getRelay
|
|
||||||
|
|
||||||
import main
|
|
||||||
from utils.logging.log import *
|
|
||||||
from utils.logging.debug import *
|
|
||||||
from utils.logging.send import *
|
|
||||||
from utils.parsing import parsen
|
|
||||||
|
|
||||||
from twisted.internet.ssl import DefaultOpenSSLContextFactory
|
|
||||||
|
|
||||||
def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
|
|
||||||
keyFN = main.certPath+main.config["Key"]
|
|
||||||
certFN = main.certPath+main.config["Certificate"]
|
|
||||||
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"),
|
|
||||||
certFN.encode("utf-8", "replace"))
|
|
||||||
bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
|
|
||||||
host, port = getRelay(num)
|
|
||||||
rct = reactor.connectSSL(host,
|
|
||||||
port,
|
|
||||||
bot, contextFactory)
|
|
||||||
|
|
||||||
def parsemsg(s):
|
|
||||||
"""
|
|
||||||
Breaks a message from an IRC server into its prefix, command, and
|
|
||||||
arguments.
|
|
||||||
@param s: The message to break.
|
|
||||||
@type s: L{bytes}
|
|
||||||
@return: A tuple of (prefix, command, args).
|
|
||||||
@rtype: L{tuple}
|
|
||||||
"""
|
|
||||||
prefix = ''
|
|
||||||
trailing = []
|
|
||||||
if not s:
|
|
||||||
raise IRCBadMessage("Empty line.")
|
|
||||||
if s[0:1] == ':':
|
|
||||||
prefix, s = s[1:].split(' ', 1)
|
|
||||||
if s.find(' :') != -1:
|
|
||||||
s, trailing = s.split(' :', 1)
|
|
||||||
args = s.split(' ')
|
|
||||||
args.append(trailing)
|
|
||||||
else:
|
|
||||||
args = s.split(' ')
|
|
||||||
command = args.pop(0)
|
|
||||||
return prefix, command, args
|
|
||||||
|
|
||||||
class IRCRelay(IRCClient):
|
|
||||||
def __init__(self, num, relayCommands, user, stage2):
|
|
||||||
self.isconnected = False
|
|
||||||
self.buffer = ""
|
|
||||||
if user == None:
|
|
||||||
self.user = main.config["Relay"]["User"]
|
|
||||||
else:
|
|
||||||
self.user = user
|
|
||||||
password = main.config["Relay"]["Password"]
|
|
||||||
self.nickname = "relay"
|
|
||||||
self.realname = "relay"
|
|
||||||
self.username = self.user
|
|
||||||
self.password = self.user+":"+password
|
|
||||||
|
|
||||||
self.relayCommands = relayCommands
|
|
||||||
self.num = num
|
|
||||||
self.stage2 = stage2
|
|
||||||
self.loop = None
|
|
||||||
|
|
||||||
def privmsg(self, user, channel, msg):
|
|
||||||
nick, ident, host = parsen(user)
|
|
||||||
for i in main.ZNCErrors:
|
|
||||||
if i in msg:
|
|
||||||
error("ZNC issue:", msg)
|
|
||||||
if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
|
|
||||||
nick = nick[1:]
|
|
||||||
if nick in self.relayCommands.keys():
|
|
||||||
sendAll("[%s] %s -> %s" % (self.num, nick, msg))
|
|
||||||
|
|
||||||
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
|
|
||||||
log("%s: relay password mismatch" % self.num)
|
|
||||||
sendAll("%s: relay password mismatch" % self.num)
|
|
||||||
|
|
||||||
def sendStage2(self):
|
|
||||||
if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []]
|
|
||||||
if not len(self.stage2) == 0:
|
|
||||||
user = self.stage2[0].pop(0)
|
|
||||||
commands = self.stage2[0].pop(0)
|
|
||||||
del self.stage2[0]
|
|
||||||
deliverRelayCommands(self.num, commands, user, self.stage2)
|
|
||||||
|
|
||||||
def signedOn(self):
|
|
||||||
if not self.isconnected:
|
|
||||||
self.isconnected = True
|
|
||||||
log("signed on as a relay: %s" % self.num)
|
|
||||||
#sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares
|
|
||||||
sleeptime = 0
|
|
||||||
increment = 0.8
|
|
||||||
for i in self.relayCommands.keys():
|
|
||||||
for x in self.relayCommands[i]:
|
|
||||||
reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x)
|
|
||||||
sleeptime += increment
|
|
||||||
increment += 0.8
|
|
||||||
reactor.callLater(sleeptime, self.sendStage2)
|
|
||||||
reactor.callLater(sleeptime+5, self.transport.loseConnection)
|
|
||||||
return
|
|
||||||
|
|
||||||
class IRCBot(IRCClient):
|
|
||||||
def __init__(self, net, num):
|
|
||||||
self.isconnected = False
|
|
||||||
self.channels = []
|
|
||||||
self.net = net
|
|
||||||
self.num = num
|
|
||||||
self.buffer = ""
|
|
||||||
self.name = net + str(num)
|
|
||||||
alias = main.alias[num]
|
|
||||||
relay = main.network[self.net].relays[num]
|
|
||||||
self.nickname = alias["nick"]
|
|
||||||
self.realname = alias["realname"]
|
|
||||||
self.username = alias["nick"]+"/"+relay["net"]
|
|
||||||
self.password = main.config["Relay"]["Password"]
|
|
||||||
self.userinfo = None #
|
|
||||||
self.fingerReply = None #
|
|
||||||
self.versionName = None # don't tell anyone information about us
|
|
||||||
self.versionNum = None #
|
|
||||||
self.versionEnv = None #
|
|
||||||
self.sourceURL = None #
|
|
||||||
|
|
||||||
self._getWho = {} # LoopingCall objects -- needed to be able to stop them
|
|
||||||
|
|
||||||
self._tempWho = {} # temporary storage for gathering WHO info
|
|
||||||
self._tempNames = {} # temporary storage for gathering NAMES info
|
|
||||||
self._tempList = ([], []) # temporary storage for gathering LIST info
|
|
||||||
self.listOngoing = False # we are currently receiving a LIST
|
|
||||||
self.listRetried = False # we asked and got nothing so asked again
|
|
||||||
self.listAttempted = False # we asked for a list
|
|
||||||
self.listSimple = False # after asking again we got the list, so use the simple
|
|
||||||
# syntax from now on
|
|
||||||
self.wantList = False # we want to send a LIST, but not all relays are active yet
|
|
||||||
self.chanlimit = 0
|
|
||||||
self.servername = None
|
|
||||||
|
|
||||||
def lineReceived(self, line):
|
|
||||||
if bytes != str and isinstance(line, bytes):
|
|
||||||
# decode bytes from transport to unicode
|
|
||||||
line = line.decode("utf-8", "replace")
|
|
||||||
|
|
||||||
line = lowDequote(line)
|
|
||||||
try:
|
|
||||||
prefix, command, params = parsemsg(line)
|
|
||||||
if command in numeric_to_symbolic:
|
|
||||||
command = numeric_to_symbolic[command]
|
|
||||||
self.handleCommand(command, prefix, params)
|
|
||||||
except IRCBadMessage:
|
|
||||||
self.badMessage(line, *sys.exc_info())
|
|
||||||
|
|
||||||
def joinChannels(self, channels):
|
|
||||||
sleeptime = 0.0
|
|
||||||
increment = 0.8
|
|
||||||
for i in channels:
|
|
||||||
if not i in self.channels:
|
|
||||||
debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds")
|
|
||||||
reactor.callLater(sleeptime, self.join, i)
|
|
||||||
sleeptime += increment
|
|
||||||
if sleeptime == 10:
|
|
||||||
sleeptime = 0.0
|
|
||||||
increment = 0.7
|
|
||||||
increment += 0.1
|
|
||||||
else:
|
|
||||||
error("%s - %i - Cannot join channel we are already on - %s" % (self.net, self.num, i))
|
|
||||||
|
|
||||||
def checkChannels(self):
|
|
||||||
if self.net in main.TempChan.keys():
|
|
||||||
if self.num in main.TempChan[self.net].keys():
|
|
||||||
self.joinChannels(main.TempChan[self.net][self.num])
|
|
||||||
del main.TempChan[self.net][self.num]
|
|
||||||
if not main.TempChan[self.net]:
|
|
||||||
del main.TempChan[self.net]
|
|
||||||
|
|
||||||
def event(self, **cast):
|
|
||||||
for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over
|
|
||||||
if cast[i] == "": # a dictionary that changes length with each iteration
|
|
||||||
del cast[i]
|
|
||||||
if "muser" in cast.keys():
|
|
||||||
if cast["muser"] == self.servername:
|
|
||||||
return
|
|
||||||
if "channel" in cast.keys():
|
|
||||||
if cast["channel"] == "*":
|
|
||||||
return
|
|
||||||
if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
|
|
||||||
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
|
|
||||||
if set(["nick", "ident", "host", "msg"]).issubset(set(cast)):
|
|
||||||
if "msg" in cast.keys():
|
|
||||||
if cast["ident"] == "znc" and cast["host"] == "znc.in":
|
|
||||||
cast["type"] = "znc"
|
|
||||||
cast["num"] = self.num
|
|
||||||
del cast["nick"]
|
|
||||||
del cast["ident"]
|
|
||||||
del cast["host"]
|
|
||||||
del cast["channel"]
|
|
||||||
if "Disconnected from IRC" in cast["msg"]:
|
|
||||||
log("ZNC disconnected on %s - %i" % (self.net, self.num))
|
|
||||||
self.isconnected = False
|
|
||||||
if "Connected!" in cast["msg"]:
|
|
||||||
log("ZNC connected on %s - %i" % (self.net, self.num))
|
|
||||||
self.isconnected = True
|
|
||||||
if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
|
|
||||||
if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here
|
|
||||||
if cast["channel"].lower() == self.nickname.lower(): # as they are channel == nickname
|
|
||||||
#castDup = deepcopy(cast) # however modes are not queries!
|
|
||||||
cast["mtype"] = cast["type"]
|
|
||||||
cast["type"] = "query"
|
|
||||||
cast["num"] = self.num
|
|
||||||
#self.event(**castDup)
|
|
||||||
# Don't call self.event for this one because queries are not events on a
|
|
||||||
# channel, but we still want to see them
|
|
||||||
if "user" in cast.keys():
|
|
||||||
if cast["user"].lower() == self.nickname.lower():
|
|
||||||
castDup = deepcopy(cast)
|
|
||||||
castDup["mtype"] = cast["type"]
|
|
||||||
castDup["type"] = "self"
|
|
||||||
cast["num"] = self.num
|
|
||||||
self.event(**castDup)
|
|
||||||
if "nick" in cast.keys():
|
|
||||||
if cast["nick"].lower() == self.nickname.lower():
|
|
||||||
castDup = deepcopy(cast)
|
|
||||||
castDup["mtype"] = cast["type"]
|
|
||||||
castDup["type"] = "self"
|
|
||||||
cast["num"] = self.num
|
|
||||||
if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly
|
|
||||||
self.event(**castDup) # don't tell anyone else
|
|
||||||
if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
|
|
||||||
if not cast["msg"] == None:
|
|
||||||
if self.nickname.lower() in cast["msg"].lower():
|
|
||||||
castDup = deepcopy(cast)
|
|
||||||
castDup["mtype"] = cast["type"]
|
|
||||||
castDup["type"] = "highlight"
|
|
||||||
cast["num"] = self.num
|
|
||||||
self.event(**castDup)
|
|
||||||
|
|
||||||
if not "net" in cast.keys():
|
|
||||||
cast["net"] = self.net
|
|
||||||
counters.event(self.net, cast["type"])
|
|
||||||
monitor.event(self.net, cast)
|
|
||||||
|
|
||||||
def privmsg(self, user, channel, msg):
|
|
||||||
self.event(type="msg", muser=user, channel=channel, msg=msg)
|
|
||||||
|
|
||||||
def noticed(self, user, channel, msg):
|
|
||||||
self.event(type="notice", muser=user, channel=channel, msg=msg)
|
|
||||||
|
|
||||||
def action(self, user, channel, msg):
|
|
||||||
self.event(type="action", muser=user, channel=channel, msg=msg)
|
|
||||||
|
|
||||||
def get(self, var):
|
|
||||||
try:
|
|
||||||
result = getattr(self, var)
|
|
||||||
except AttributeError:
|
|
||||||
result = None
|
|
||||||
return result
|
|
||||||
|
|
||||||
def setNick(self, nickname):
|
|
||||||
self._attemptedNick = nickname
|
|
||||||
self.sendLine("NICK %s" % nickname)
|
|
||||||
self.nickname = nickname
|
|
||||||
|
|
||||||
def alterCollidedNick(self, nickname):
|
|
||||||
newnick = nickname + "_"
|
|
||||||
return newnick
|
|
||||||
|
|
||||||
def nickChanged(self, olduser, newnick):
|
|
||||||
self.nickname = newnick
|
|
||||||
self.event(type="self", mtype="nick", muser=olduser, user=newnick)
|
|
||||||
|
|
||||||
def irc_ERR_NICKNAMEINUSE(self, prefix, params):
|
|
||||||
self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
|
|
||||||
self.setNick(self._attemptedNick)
|
|
||||||
|
|
||||||
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
|
|
||||||
log("%s - %i: password mismatch" % (self.net, self.num))
|
|
||||||
sendAll("%s - %i: password mismatch" % (self.net, self.num))
|
|
||||||
|
|
||||||
def _who(self, channel):
|
|
||||||
d = Deferred()
|
|
||||||
if channel not in self._tempWho:
|
|
||||||
self._tempWho[channel] = ([], [])
|
|
||||||
self._tempWho[channel][0].append(d)
|
|
||||||
self.sendLine("WHO %s" % channel)
|
|
||||||
return d
|
|
||||||
|
|
||||||
def who(self, channel):
|
|
||||||
self._who(channel).addCallback(self.got_who)
|
|
||||||
|
|
||||||
def irc_RPL_WHOREPLY(self, prefix, params):
|
|
||||||
channel = params[1]
|
|
||||||
ident = params[2]
|
|
||||||
host = params[3]
|
|
||||||
server = params[4]
|
|
||||||
nick = params[5]
|
|
||||||
status = params[6]
|
|
||||||
realname = params[7]
|
|
||||||
if channel not in self._tempWho:
|
|
||||||
return
|
|
||||||
n = self._tempWho[channel][1]
|
|
||||||
n.append([nick, nick, host, server, status, realname])
|
|
||||||
self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status)
|
|
||||||
|
|
||||||
def irc_RPL_ENDOFWHO(self, prefix, params):
|
|
||||||
channel = params[1]
|
|
||||||
if channel not in self._tempWho:
|
|
||||||
return
|
|
||||||
callbacks, info = self._tempWho[channel]
|
|
||||||
for cb in callbacks:
|
|
||||||
cb.callback((channel, info))
|
|
||||||
del self._tempWho[channel]
|
|
||||||
|
|
||||||
def got_who(self, whoinfo):
|
|
||||||
userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
|
|
||||||
|
|
||||||
def sanit(self, data):
|
|
||||||
if len(data) >= 1:
|
|
||||||
if data[0] in ["!", "~", "&", "@", "%", "+"]:
|
|
||||||
data = data[1:]
|
|
||||||
return data
|
|
||||||
return data
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def names(self, channel):
|
|
||||||
d = Deferred()
|
|
||||||
if channel not in self._tempNames:
|
|
||||||
self._tempNames[channel] = ([], [])
|
|
||||||
self._tempNames[channel][0].append(d)
|
|
||||||
self.sendLine("NAMES %s" % channel)
|
|
||||||
return d
|
|
||||||
|
|
||||||
def irc_RPL_NAMREPLY(self, prefix, params):
|
|
||||||
channel = params[2]
|
|
||||||
nicklist = params[3].split(' ')
|
|
||||||
if channel not in self._tempNames:
|
|
||||||
return
|
|
||||||
n = self._tempNames[channel][1]
|
|
||||||
n.append(nicklist)
|
|
||||||
|
|
||||||
def irc_RPL_ENDOFNAMES(self, prefix, params):
|
|
||||||
channel = params[1]
|
|
||||||
if channel not in self._tempNames:
|
|
||||||
return
|
|
||||||
callbacks, namelist = self._tempNames[channel]
|
|
||||||
for cb in callbacks:
|
|
||||||
cb.callback((channel, namelist))
|
|
||||||
del self._tempNames[channel]
|
|
||||||
|
|
||||||
def got_names(self, nicklist):
|
|
||||||
newNicklist = []
|
|
||||||
for i in nicklist[1]:
|
|
||||||
for x in i:
|
|
||||||
f = self.sanit(x) # need to store this as well, or potentially just do not remove it...
|
|
||||||
if f:
|
|
||||||
newNicklist.append(f)
|
|
||||||
userinfo.initialNames(self.net, nicklist[0], newNicklist)
|
|
||||||
|
|
||||||
def myInfo(self, servername, version, umodes, cmodes):
|
|
||||||
self.servername = servername
|
|
||||||
|
|
||||||
def _list(self, noargs):
|
|
||||||
d = Deferred()
|
|
||||||
self._tempList = ([], [])
|
|
||||||
self._tempList[0].append(d)
|
|
||||||
if self.listSimple:
|
|
||||||
self.sendLine("LIST")
|
|
||||||
return d # return early if we know what to do
|
|
||||||
|
|
||||||
if noargs:
|
|
||||||
self.sendLine("LIST")
|
|
||||||
else:
|
|
||||||
self.sendLine("LIST >0")
|
|
||||||
return d
|
|
||||||
|
|
||||||
def list(self, noargs=False, nocheck=False):
|
|
||||||
if self.listAttempted:
|
|
||||||
debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num))
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
self.listAttempted = True
|
|
||||||
if self.listOngoing:
|
|
||||||
debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num))
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
if nocheck:
|
|
||||||
allRelays = True # override the system - if this is
|
|
||||||
else: # specified, we already did this
|
|
||||||
allRelays = chankeep.allRelaysActive(self.net)
|
|
||||||
if not allRelays:
|
|
||||||
self.wantList = True
|
|
||||||
debug("Not all relays were active for LIST request")
|
|
||||||
return
|
|
||||||
self._list(noargs).addCallback(self.got_list)
|
|
||||||
|
|
||||||
def irc_RPL_LISTSTART(self, prefix, params):
|
|
||||||
self.listAttempted = False
|
|
||||||
self.listOngoing = True
|
|
||||||
self.wantList = False
|
|
||||||
|
|
||||||
def irc_RPL_LIST(self, prefix, params):
|
|
||||||
channel = params[1]
|
|
||||||
users = params[2]
|
|
||||||
topic = params[3]
|
|
||||||
self._tempList[1].append([channel, users, topic])
|
|
||||||
|
|
||||||
def irc_RPL_LISTEND(self, prefix, params):
|
|
||||||
if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there
|
|
||||||
debug("We didn't ask for this LIST, discarding")
|
|
||||||
self._tempList[0].clear()
|
|
||||||
self._tempList[1].clear()
|
|
||||||
return
|
|
||||||
callbacks, info = self._tempList
|
|
||||||
self.listOngoing = False
|
|
||||||
for cb in callbacks:
|
|
||||||
cb.callback((info))
|
|
||||||
noResults = False
|
|
||||||
if len(self._tempList[1]) == 0:
|
|
||||||
noResults = True
|
|
||||||
self._tempList[0].clear()
|
|
||||||
self._tempList[1].clear()
|
|
||||||
if noResults:
|
|
||||||
if self.listRetried:
|
|
||||||
warn("LIST still empty after retry: %s - %i" % (net, num))
|
|
||||||
self.listRetried = False
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
self.list(True)
|
|
||||||
self.listRetried = True
|
|
||||||
else:
|
|
||||||
if self.listRetried:
|
|
||||||
self.listRetried = False
|
|
||||||
debug("List received after retry - defaulting to simple list syntax")
|
|
||||||
self.listSimple = True
|
|
||||||
|
|
||||||
def got_list(self, listinfo):
|
|
||||||
if len(listinfo) == 0: # probably ngircd not supporting LIST >0
|
|
||||||
return
|
|
||||||
chankeep.initialList(self.net, self.num, listinfo, self.chanlimit)
|
|
||||||
|
|
||||||
def isupport(self, options):
|
|
||||||
interested = ("CHANLIMIT", "MAXCHANNELS")
|
|
||||||
if not any((x for x in options if any(y in x for y in interested))):
|
|
||||||
return # check if any of interested is in any of options, some networks
|
|
||||||
chanlimit = None # call isupport() more than once, so discard swiftly anything
|
|
||||||
if not self.isconnected: # we don't care about
|
|
||||||
log("endpoint connected: %s - %i" % (self.net, self.num))
|
|
||||||
self.isconnected = True
|
|
||||||
for i in options:
|
|
||||||
if i.startswith("CHANLIMIT"):
|
|
||||||
if ":" in i:
|
|
||||||
split = i.split(":")
|
|
||||||
if len(split) >= 2:
|
|
||||||
chanlimit = split[1]
|
|
||||||
break
|
|
||||||
elif i.startswith("MAXCHANNELS"):
|
|
||||||
if "=" in i:
|
|
||||||
split = i.split("=")
|
|
||||||
if len(split) == 2:
|
|
||||||
chanlimit = split[1]
|
|
||||||
break
|
|
||||||
print("chanlimit", chanlimit)
|
|
||||||
try:
|
|
||||||
self.chanlimit = int(chanlimit)
|
|
||||||
except TypeError:
|
|
||||||
warn("Invalid chanlimit: %s" % i)
|
|
||||||
if self.chanlimit == 0:
|
|
||||||
self.chanlimit = 200 # don't take the piss if it's unlimited
|
|
||||||
allRelays = chankeep.allRelaysActive(self.net)
|
|
||||||
print(self.net, self.num, allRelays)
|
|
||||||
if allRelays:
|
|
||||||
for i in main.network.keys():
|
|
||||||
for x in main.network[i].relays.keys():
|
|
||||||
name = i+str(x)
|
|
||||||
if main.IRCPool[name].wantList == True:
|
|
||||||
main.IRCPool[name].list(nocheck=True)
|
|
||||||
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
|
|
||||||
if self.num == 1: # Only one instance should do a list
|
|
||||||
if self.chanlimit:
|
|
||||||
if allRelays:
|
|
||||||
self.list()
|
|
||||||
else:
|
|
||||||
self.wantList = True
|
|
||||||
else:
|
|
||||||
debug("Aborting LIST due to bad chanlimit")
|
|
||||||
self.checkChannels()
|
|
||||||
|
|
||||||
#twisted sucks so i have to do this to actually get the user info
|
|
||||||
def irc_JOIN(self, prefix, params):
|
|
||||||
"""
|
|
||||||
Called when a user joins a channel.
|
|
||||||
"""
|
|
||||||
nick = prefix.split('!')[0]
|
|
||||||
channel = params[-1]
|
|
||||||
if nick == self.nickname:
|
|
||||||
self.joined(channel)
|
|
||||||
else:
|
|
||||||
self.userJoined(prefix, channel)
|
|
||||||
|
|
||||||
def irc_PART(self, prefix, params):
|
|
||||||
"""
|
|
||||||
Called when a user leaves a channel.
|
|
||||||
"""
|
|
||||||
nick = prefix.split('!')[0]
|
|
||||||
channel = params[0]
|
|
||||||
if len(params) >= 2:
|
|
||||||
message = params[1]
|
|
||||||
else:
|
|
||||||
message = None
|
|
||||||
if nick == self.nickname:
|
|
||||||
self.left(prefix, channel, message)
|
|
||||||
else:
|
|
||||||
self.userLeft(prefix, channel, message)
|
|
||||||
|
|
||||||
def irc_QUIT(self, prefix, params):
|
|
||||||
"""
|
|
||||||
Called when a user has quit.
|
|
||||||
"""
|
|
||||||
nick = prefix.split('!')[0]
|
|
||||||
#if nick == self.nickname:
|
|
||||||
#self.botQuit(prefix, params[0])
|
|
||||||
#else:
|
|
||||||
self.userQuit(prefix, params[0])
|
|
||||||
|
|
||||||
def irc_NICK(self, prefix, params):
|
|
||||||
"""
|
|
||||||
Called when a user changes their nickname.
|
|
||||||
"""
|
|
||||||
nick = prefix.split('!', 1)[0]
|
|
||||||
if nick == self.nickname:
|
|
||||||
self.nickChanged(prefix, params[0])
|
|
||||||
else:
|
|
||||||
self.userRenamed(prefix, params[0])
|
|
||||||
|
|
||||||
def irc_KICK(self, prefix, params):
|
|
||||||
"""
|
|
||||||
Called when a user is kicked from a channel.
|
|
||||||
"""
|
|
||||||
#kicker = prefix.split('!')[0]
|
|
||||||
channel = params[0]
|
|
||||||
kicked = params[1]
|
|
||||||
message = params[-1]
|
|
||||||
# Checks on whether it was us that was kicked are done in userKicked
|
|
||||||
self.userKicked(kicked, channel, prefix, message)
|
|
||||||
|
|
||||||
def irc_TOPIC(self, prefix, params):
|
|
||||||
"""
|
|
||||||
Someone in the channel set the topic.
|
|
||||||
"""
|
|
||||||
#user = prefix.split('!')[0]
|
|
||||||
channel = params[0]
|
|
||||||
newtopic = params[1]
|
|
||||||
self.topicUpdated(prefix, channel, newtopic)
|
|
||||||
#END hacks
|
|
||||||
|
|
||||||
def signedOn(self):
|
|
||||||
log("signed on: %s - %i" % (self.net, self.num))
|
|
||||||
#self.event(type="conn", status="connected")
|
|
||||||
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"})
|
|
||||||
|
|
||||||
def joined(self, channel):
|
|
||||||
if not channel in self.channels:
|
|
||||||
self.channels.append(channel)
|
|
||||||
self.names(channel).addCallback(self.got_names)
|
|
||||||
if main.config["Toggles"]["Who"]:
|
|
||||||
#self.who(channel).addCallback(self.got_who)
|
|
||||||
#self.who(channel)
|
|
||||||
lc = LoopingCall(self.who, channel)
|
|
||||||
self._getWho[channel] = lc
|
|
||||||
intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
|
|
||||||
minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
|
|
||||||
interval = randint(minint, minint+intrange)
|
|
||||||
lc.start(interval)
|
|
||||||
|
|
||||||
def botLeft(self, channel):
|
|
||||||
if channel in self.channels:
|
|
||||||
self.channels.remove(channel)
|
|
||||||
if channel in self._getWho.keys():
|
|
||||||
lc = self._getWho[channel]
|
|
||||||
lc.stop()
|
|
||||||
del self._getWho[channel]
|
|
||||||
userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this
|
|
||||||
#log("Can no longer cover %s, removing records" % channel)# as it will only be matched once --
|
|
||||||
# other bots have different nicknames so
|
|
||||||
def left(self, user, channel, message): # even if they saw it, they wouldn't react
|
|
||||||
self.event(type="part", muser=user, channel=channel, message=message)
|
|
||||||
self.botLeft(channel)
|
|
||||||
|
|
||||||
def userJoined(self, user, channel):
|
|
||||||
self.event(type="join", muser=user, channel=channel)
|
|
||||||
|
|
||||||
def userLeft(self, user, channel, message):
|
|
||||||
self.event(type="part", muser=user, channel=channel, message=message)
|
|
||||||
|
|
||||||
def userQuit(self, user, quitMessage):
|
|
||||||
self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage})
|
|
||||||
|
|
||||||
def userKicked(self, kickee, channel, kicker, message):
|
|
||||||
if kickee.lower() == self.nickname.lower():
|
|
||||||
self.botLeft(channel)
|
|
||||||
self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee)
|
|
||||||
|
|
||||||
def chanlessEvent(self, cast):
|
|
||||||
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
|
|
||||||
if dedup(self.name, cast): # Needs to be kept self.name until the dedup
|
|
||||||
# function is converted to the new net, num
|
|
||||||
# format
|
|
||||||
return # stop right there sir!
|
|
||||||
chans = userinfo.getChanList(self.net, cast["nick"])
|
|
||||||
if chans == None:
|
|
||||||
error("No channels returned for chanless event: %s" % cast)
|
|
||||||
# self.event(**cast) -- no, should NEVER happen
|
|
||||||
return
|
|
||||||
# getChansSingle returns all channels of the user, we only want to use
|
|
||||||
# ones we have common with them
|
|
||||||
realChans = set(chans).intersection(set(self.channels))
|
|
||||||
for i in realChans:
|
|
||||||
cast["channel"] = i
|
|
||||||
self.event(**cast)
|
|
||||||
|
|
||||||
def userRenamed(self, oldname, newname):
|
|
||||||
self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
|
|
||||||
|
|
||||||
def topicUpdated(self, user, channel, newTopic):
|
|
||||||
self.event(type="topic", muser=user, channel=channel, message= newTopic)
|
|
||||||
|
|
||||||
def modeChanged(self, user, channel, toset, modes, args):
|
|
||||||
argList = list(args)
|
|
||||||
modeList = [i for i in modes]
|
|
||||||
for a, m in zip(argList, modeList):
|
|
||||||
self.event(type="mode", muser=user, channel=channel, modes=m, status=toset, modeargs=a)
|
|
||||||
|
|
||||||
class IRCBotFactory(ReconnectingClientFactory):
|
|
||||||
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
|
|
||||||
if net == None:
|
|
||||||
self.num = num
|
|
||||||
self.net = None
|
|
||||||
self.name = "relay - %i" % num
|
|
||||||
self.relay = True
|
|
||||||
else:
|
|
||||||
self.name = net+str(num)
|
|
||||||
self.num = num
|
|
||||||
self.net = net
|
|
||||||
self.relay = False
|
|
||||||
self.client = None
|
|
||||||
self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
|
|
||||||
self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
|
|
||||||
self.factor = main.config["Tweaks"]["Delays"]["Factor"]
|
|
||||||
self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
|
|
||||||
|
|
||||||
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
|
|
||||||
|
|
||||||
def buildProtocol(self, addr):
|
|
||||||
if self.relay == False:
|
|
||||||
entry = IRCBot(self.net, self.num)
|
|
||||||
main.IRCPool[self.name] = entry
|
|
||||||
else:
|
|
||||||
entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
|
|
||||||
|
|
||||||
self.client = entry
|
|
||||||
return entry
|
|
||||||
|
|
||||||
def clientConnectionLost(self, connector, reason):
|
|
||||||
if not self.relay:
|
|
||||||
userinfo.delChannels(self.net, self.client.channels)
|
|
||||||
if not self.client == None:
|
|
||||||
self.client.isconnected = False
|
|
||||||
self.client.channels = []
|
|
||||||
error = reason.getErrorMessage()
|
|
||||||
log("%s - %i: connection lost: %s" % (self.net, self.num, error))
|
|
||||||
if not self.relay:
|
|
||||||
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
|
|
||||||
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error})
|
|
||||||
self.retry(connector)
|
|
||||||
#ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
|
|
||||||
|
|
||||||
def clientConnectionFailed(self, connector, reason):
|
|
||||||
if not self.client == None:
|
|
||||||
self.client.isconnected = False
|
|
||||||
self.client.channels = []
|
|
||||||
error = reason.getErrorMessage()
|
|
||||||
log("%s - %i: connection failed: %s" % (self.net, self.num, error))
|
|
||||||
if not self.relay:
|
|
||||||
sendAll("%s -%s: connection failed: %s" % (self.net, self.num, error))
|
|
||||||
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error})
|
|
||||||
self.retry(connector)
|
|
||||||
#ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
|
|
||||||
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
import main
|
|
||||||
from utils.logging.log import *
|
|
||||||
from utils.logging.send import *
|
|
||||||
|
|
||||||
def parseCommand(addr, authed, data):
|
|
||||||
#call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
|
|
||||||
spl = data.split()
|
|
||||||
if addr in main.connections.keys():
|
|
||||||
obj = main.connections[addr]
|
|
||||||
else:
|
|
||||||
warn("Got connection object with no instance in the address pool")
|
|
||||||
return
|
|
||||||
|
|
||||||
success = lambda data: sendSuccess(addr, data)
|
|
||||||
failure = lambda data: sendFailure(addr, data)
|
|
||||||
info = lambda data: sendInfo(addr, data)
|
|
||||||
|
|
||||||
incUsage = lambda mode: incorrectUsage(addr, mode)
|
|
||||||
length = len(spl)
|
|
||||||
if len(spl) > 0:
|
|
||||||
cmd = spl[0]
|
|
||||||
else:
|
|
||||||
failure("No text was sent")
|
|
||||||
return
|
|
||||||
if spl[0] in main.CommandMap.keys():
|
|
||||||
main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length)
|
|
||||||
return
|
|
||||||
incUsage(None)
|
|
||||||
return
|
|
||||||
|
|
@ -0,0 +1,110 @@
|
||||||
|
import random
|
||||||
|
|
||||||
|
import aioredis
|
||||||
|
import orjson
|
||||||
|
|
||||||
|
# Kafka
|
||||||
|
from aiokafka import AIOKafkaProducer
|
||||||
|
from redis import StrictRedis
|
||||||
|
|
||||||
|
import util
|
||||||
|
|
||||||
|
# KAFKA_TOPIC = "msg"
|
||||||
|
|
||||||
|
log = util.get_logger("db")
|
||||||
|
|
||||||
|
# Redis (legacy)
|
||||||
|
r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
|
||||||
|
|
||||||
|
# AIORedis
|
||||||
|
ar = aioredis.from_url("unix:///var/run/redis/redis.sock", db=0)
|
||||||
|
|
||||||
|
TYPES_MAIN = [
|
||||||
|
"msg",
|
||||||
|
"notice",
|
||||||
|
"action",
|
||||||
|
"part",
|
||||||
|
"join",
|
||||||
|
"kick",
|
||||||
|
"quit",
|
||||||
|
"nick",
|
||||||
|
"mode",
|
||||||
|
"topic",
|
||||||
|
"update",
|
||||||
|
]
|
||||||
|
TYPES_META = ["who"]
|
||||||
|
TYPES_INT = ["conn", "highlight", "znc", "query", "self"]
|
||||||
|
KEYPREFIX = "queue."
|
||||||
|
|
||||||
|
|
||||||
|
async def store_kafka_batch(data):
|
||||||
|
log.debug(f"Storing Kafka batch of {len(data)} messages")
|
||||||
|
producer = AIOKafkaProducer(bootstrap_servers="kafka:9092")
|
||||||
|
await producer.start()
|
||||||
|
batch = producer.create_batch()
|
||||||
|
for msg in data:
|
||||||
|
if msg["type"] in TYPES_MAIN:
|
||||||
|
index = "main"
|
||||||
|
# schema = mc_s.schema_main
|
||||||
|
elif msg["type"] in TYPES_META:
|
||||||
|
index = "meta"
|
||||||
|
# schema = mc_s.schema_meta
|
||||||
|
elif msg["type"] in TYPES_INT:
|
||||||
|
index = "internal"
|
||||||
|
# schema = mc_s.schema_int
|
||||||
|
|
||||||
|
KAFKA_TOPIC = index
|
||||||
|
# normalise fields
|
||||||
|
for key, value in list(msg.items()):
|
||||||
|
if value is None:
|
||||||
|
del msg[key]
|
||||||
|
# if key in schema:
|
||||||
|
# if isinstance(value, int):
|
||||||
|
# if schema[key].startswith("string") or schema[key].startswith(
|
||||||
|
# "text"
|
||||||
|
# ):
|
||||||
|
# msg[key] = str(value)
|
||||||
|
body = orjson.dumps(msg)
|
||||||
|
# orjson returns bytes
|
||||||
|
# body = str.encode(message)
|
||||||
|
if "ts" not in msg:
|
||||||
|
raise Exception("No TS in msg")
|
||||||
|
metadata = batch.append(key=None, value=body, timestamp=msg["ts"])
|
||||||
|
if metadata is None:
|
||||||
|
partitions = await producer.partitions_for(KAFKA_TOPIC)
|
||||||
|
partition = random.choice(tuple(partitions))
|
||||||
|
await producer.send_batch(batch, KAFKA_TOPIC, partition=partition)
|
||||||
|
log.debug(f"{batch.record_count()} messages sent to partition {partition}")
|
||||||
|
batch = producer.create_batch()
|
||||||
|
continue
|
||||||
|
|
||||||
|
partitions = await producer.partitions_for(KAFKA_TOPIC)
|
||||||
|
partition = random.choice(tuple(partitions))
|
||||||
|
await producer.send_batch(batch, KAFKA_TOPIC, partition=partition)
|
||||||
|
log.debug(f"{batch.record_count()} messages sent to partition {partition}")
|
||||||
|
await producer.stop()
|
||||||
|
|
||||||
|
|
||||||
|
async def queue_message(msg):
|
||||||
|
"""
|
||||||
|
Queue a message on the Redis buffer.
|
||||||
|
"""
|
||||||
|
src = msg["src"]
|
||||||
|
message = orjson.dumps(msg)
|
||||||
|
|
||||||
|
key = f"{KEYPREFIX}{src}"
|
||||||
|
# log.debug(f"Queueing single message of string length {len(message)}")
|
||||||
|
await ar.sadd(key, message)
|
||||||
|
|
||||||
|
|
||||||
|
async def queue_message_bulk(data):
|
||||||
|
"""
|
||||||
|
Queue multiple messages on the Redis buffer.
|
||||||
|
"""
|
||||||
|
# log.debug(f"Queueing message batch of length {len(data)}")
|
||||||
|
for msg in data:
|
||||||
|
src = msg["src"]
|
||||||
|
message = orjson.dumps(msg)
|
||||||
|
|
||||||
|
key = f"{KEYPREFIX}{src}"
|
||||||
|
await ar.sadd(key, message)
|
||||||
|
|
@ -0,0 +1,259 @@
|
||||||
|
version: "2.2"
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
metadata_data: {}
|
||||||
|
middle_var: {}
|
||||||
|
historical_var: {}
|
||||||
|
broker_var: {}
|
||||||
|
coordinator_var: {}
|
||||||
|
router_var: {}
|
||||||
|
druid_shared: {}
|
||||||
|
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: pathogen/monolith:latest
|
||||||
|
build: ./docker
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}:/code
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
depends_on:
|
||||||
|
broker:
|
||||||
|
condition: service_started
|
||||||
|
kafka:
|
||||||
|
condition: service_healthy
|
||||||
|
tmp:
|
||||||
|
condition: service_started
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
# - db
|
||||||
|
|
||||||
|
threshold:
|
||||||
|
image: pathogen/threshold:latest
|
||||||
|
build: ./legacy/docker
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}:/code
|
||||||
|
- ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
|
||||||
|
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
|
||||||
|
- ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
|
||||||
|
ports:
|
||||||
|
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
|
||||||
|
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
|
||||||
|
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
# for development
|
||||||
|
extra_hosts:
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
depends_on:
|
||||||
|
tmp:
|
||||||
|
condition: service_started
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
turnilo:
|
||||||
|
container_name: turnilo
|
||||||
|
image: uchhatre/turnilo:latest
|
||||||
|
ports:
|
||||||
|
- 9093:9090
|
||||||
|
environment:
|
||||||
|
- DRUID_BROKER_URL=http://broker:8082
|
||||||
|
depends_on:
|
||||||
|
- broker
|
||||||
|
|
||||||
|
metabase:
|
||||||
|
container_name: metabase
|
||||||
|
image: metabase/metabase:latest
|
||||||
|
ports:
|
||||||
|
- 3001:3000
|
||||||
|
depends_on:
|
||||||
|
- broker
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
container_name: postgres
|
||||||
|
image: postgres:latest
|
||||||
|
volumes:
|
||||||
|
- metadata_data:/var/lib/postgresql/data
|
||||||
|
environment:
|
||||||
|
- POSTGRES_PASSWORD=FoolishPassword
|
||||||
|
- POSTGRES_USER=druid
|
||||||
|
- POSTGRES_DB=druid
|
||||||
|
|
||||||
|
# Need 3.5 or later for container nodes
|
||||||
|
zookeeper:
|
||||||
|
container_name: zookeeper
|
||||||
|
image: zookeeper:3.5
|
||||||
|
ports:
|
||||||
|
- "2181:2181"
|
||||||
|
environment:
|
||||||
|
- ZOO_MY_ID=1
|
||||||
|
|
||||||
|
kafka:
|
||||||
|
image: bitnami/kafka
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- broker
|
||||||
|
ports:
|
||||||
|
- 29092:29092
|
||||||
|
- 9092:9092
|
||||||
|
environment:
|
||||||
|
KAFKA_BROKER_ID: 1
|
||||||
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||||
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
|
||||||
|
#KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||||
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
||||||
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||||
|
ALLOW_PLAINTEXT_LISTENER: yes
|
||||||
|
# healthcheck:
|
||||||
|
# test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic main --describe"]
|
||||||
|
# interval: 2s
|
||||||
|
# timeout: 2s
|
||||||
|
# retries: 15
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "kafka:9092"]
|
||||||
|
start_period: 15s
|
||||||
|
interval: 2s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
coordinator:
|
||||||
|
image: apache/druid:0.23.0
|
||||||
|
container_name: coordinator
|
||||||
|
volumes:
|
||||||
|
- druid_shared:/opt/shared
|
||||||
|
- coordinator_var:/opt/druid/var
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- postgres
|
||||||
|
ports:
|
||||||
|
- "8081:8081"
|
||||||
|
command:
|
||||||
|
- coordinator
|
||||||
|
env_file:
|
||||||
|
- environment
|
||||||
|
|
||||||
|
broker:
|
||||||
|
image: apache/druid:0.23.0
|
||||||
|
container_name: broker
|
||||||
|
volumes:
|
||||||
|
- broker_var:/opt/druid/var
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- postgres
|
||||||
|
- coordinator
|
||||||
|
ports:
|
||||||
|
- "8082:8082"
|
||||||
|
command:
|
||||||
|
- broker
|
||||||
|
env_file:
|
||||||
|
- environment
|
||||||
|
|
||||||
|
historical:
|
||||||
|
image: apache/druid:0.23.0
|
||||||
|
container_name: historical
|
||||||
|
volumes:
|
||||||
|
- druid_shared:/opt/shared
|
||||||
|
- historical_var:/opt/druid/var
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- postgres
|
||||||
|
- coordinator
|
||||||
|
ports:
|
||||||
|
- "8083:8083"
|
||||||
|
command:
|
||||||
|
- historical
|
||||||
|
env_file:
|
||||||
|
- environment
|
||||||
|
|
||||||
|
middlemanager:
|
||||||
|
image: apache/druid:0.23.0
|
||||||
|
container_name: middlemanager
|
||||||
|
volumes:
|
||||||
|
- druid_shared:/opt/shared
|
||||||
|
- middle_var:/opt/druid/var
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- postgres
|
||||||
|
- coordinator
|
||||||
|
ports:
|
||||||
|
- "8091:8091"
|
||||||
|
- "8100-8105:8100-8105"
|
||||||
|
command:
|
||||||
|
- middleManager
|
||||||
|
env_file:
|
||||||
|
- environment
|
||||||
|
|
||||||
|
router:
|
||||||
|
image: apache/druid:0.23.0
|
||||||
|
container_name: router
|
||||||
|
volumes:
|
||||||
|
- router_var:/opt/druid/var
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- postgres
|
||||||
|
- coordinator
|
||||||
|
ports:
|
||||||
|
- "8888:8888"
|
||||||
|
command:
|
||||||
|
- router
|
||||||
|
env_file:
|
||||||
|
- environment
|
||||||
|
|
||||||
|
# db:
|
||||||
|
# #image: pathogen/manticore:kibana
|
||||||
|
# image: manticoresearch/manticore:dev
|
||||||
|
# #build:
|
||||||
|
# # context: ./docker/manticore
|
||||||
|
# # args:
|
||||||
|
# # DEV: 1
|
||||||
|
# restart: always
|
||||||
|
# ports:
|
||||||
|
# - 9308
|
||||||
|
# - 9312
|
||||||
|
# - 9306
|
||||||
|
# ulimits:
|
||||||
|
# nproc: 65535
|
||||||
|
# nofile:
|
||||||
|
# soft: 65535
|
||||||
|
# hard: 65535
|
||||||
|
# memlock:
|
||||||
|
# soft: -1
|
||||||
|
# hard: -1
|
||||||
|
# environment:
|
||||||
|
# - MCL=1
|
||||||
|
# volumes:
|
||||||
|
# - ./docker/data:/var/lib/manticore
|
||||||
|
# - ./docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
||||||
|
|
||||||
|
tmp:
|
||||||
|
image: busybox
|
||||||
|
command: chmod -R 777 /var/run/redis
|
||||||
|
volumes:
|
||||||
|
- /var/run/redis
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis
|
||||||
|
command: redis-server /etc/redis.conf
|
||||||
|
ulimits:
|
||||||
|
nproc: 65535
|
||||||
|
nofile:
|
||||||
|
soft: 65535
|
||||||
|
hard: 65535
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
healthcheck:
|
||||||
|
test: "redis-cli -s /var/run/redis/redis.sock ping"
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 15
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
external:
|
||||||
|
name: pathogen
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM python:3
|
||||||
|
|
||||||
|
RUN useradd -d /code pathogen
|
||||||
|
RUN mkdir /code
|
||||||
|
RUN chown pathogen:pathogen /code
|
||||||
|
|
||||||
|
RUN mkdir /venv
|
||||||
|
RUN chown pathogen:pathogen /venv
|
||||||
|
|
||||||
|
USER pathogen
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
WORKDIR /code
|
||||||
|
COPY requirements.txt /code/
|
||||||
|
COPY discord-patched.tgz /code/
|
||||||
|
|
||||||
|
RUN python -m venv /venv
|
||||||
|
RUN . /venv/bin/activate && pip install -r requirements.txt && python -m spacy download en_core_web_sm
|
||||||
|
|
||||||
|
RUN tar xf /code/discord-patched.tgz -C /venv/lib/python3.10/site-packages
|
||||||
|
|
||||||
|
CMD . /venv/bin/activate && exec python monolith.py
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,77 @@
|
||||||
|
version: "2"
|
||||||
|
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: pathogen/monolith:latest
|
||||||
|
build: ./docker
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}:/code
|
||||||
|
env_file:
|
||||||
|
- ../stack.env
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
depends_on:
|
||||||
|
- db
|
||||||
|
|
||||||
|
threshold:
|
||||||
|
image: pathogen/threshold:latest
|
||||||
|
build: ./legacy/docker
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}:/code
|
||||||
|
- ${THRESHOLD_CONFIG_DIR}:/code/legacy/conf/live
|
||||||
|
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
|
||||||
|
- ${THRESHOLD_CERT_DIR}:/code/legacy/conf/cert
|
||||||
|
ports:
|
||||||
|
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
|
||||||
|
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
|
||||||
|
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
|
||||||
|
env_file:
|
||||||
|
- ../stack.env
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
|
||||||
|
db:
|
||||||
|
image: manticoresearch/manticore:latest
|
||||||
|
restart: always
|
||||||
|
ports:
|
||||||
|
- 9308
|
||||||
|
- 9312
|
||||||
|
- 9306
|
||||||
|
ulimits:
|
||||||
|
nproc: 65535
|
||||||
|
nofile:
|
||||||
|
soft: 65535
|
||||||
|
hard: 65535
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
environment:
|
||||||
|
- MCL=1
|
||||||
|
volumes:
|
||||||
|
- /opt/docker/pathogen/monolith/prod:/var/lib/manticore
|
||||||
|
- ${PORTAINER_GIT_DIR}/docker/manticore.conf:/etc/manticoresearch/manticore.conf
|
||||||
|
|
||||||
|
|
||||||
|
tmp:
|
||||||
|
image: busybox
|
||||||
|
command: chmod -R 777 /var/run/redis
|
||||||
|
ulimits:
|
||||||
|
nproc: 65535
|
||||||
|
nofile:
|
||||||
|
soft: 65535
|
||||||
|
hard: 65535
|
||||||
|
volumes:
|
||||||
|
- /var/run/redis
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis
|
||||||
|
command: redis-server /etc/redis.conf
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
external:
|
||||||
|
name: pathogen
|
||||||
|
|
@ -0,0 +1,265 @@
|
||||||
|
#!/bin/sh
|
||||||
|
ip=`hostname -i|rev|cut -d\ -f 1|rev`
|
||||||
|
cat << EOF
|
||||||
|
searchd {
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
|
||||||
|
# access_plain_attrs = mmap_preread
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
|
||||||
|
# access_blob_attrs = mmap_preread
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
|
||||||
|
# access_doclists = file
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
|
||||||
|
# access_hitlists = file
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
|
||||||
|
# agent_connect_timeout =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
|
||||||
|
# agent_query_timeout =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
|
||||||
|
# agent_retry_count = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
|
||||||
|
# agent_retry_delay = 500
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
|
||||||
|
# attr_flush_period = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
|
||||||
|
# binlog_flush = 2
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
|
||||||
|
# binlog_max_log_size = 268435456
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
|
||||||
|
# binlog_path =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
|
||||||
|
# client_timeout = 300
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
|
||||||
|
# collation_libc_locale = C
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
|
||||||
|
# collation_server = libc_ci
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
|
||||||
|
data_dir = /var/lib/manticore
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
|
||||||
|
# docstore_cache_size = 16m
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
|
||||||
|
# expansion_limit = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
|
||||||
|
# grouping_in_utc = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
|
||||||
|
# ha_period_karma = 60
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
|
||||||
|
# ha_ping_interval = 1000
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
|
||||||
|
# hostname_lookup =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
|
||||||
|
# jobs_queue_size =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
|
||||||
|
# listen_backlog = 5
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#listen
|
||||||
|
# listen_env = this directive allows to append listeners from environment variables
|
||||||
|
|
||||||
|
listen = 9306:mysql41
|
||||||
|
listen = /var/run/mysqld/mysqld.sock:mysql41
|
||||||
|
listen = $ip:9312
|
||||||
|
listen = 9308:http
|
||||||
|
listen = $ip:9315-9325:replication
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
|
||||||
|
# listen_tfo = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#log
|
||||||
|
log = /var/log/manticore/searchd.log
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
|
||||||
|
# max_batch_queries = 32
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#threads
|
||||||
|
# threads =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
|
||||||
|
# max_filters = 256
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
|
||||||
|
# max_filter_values = 4096
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
|
||||||
|
# max_open_files = max
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
|
||||||
|
max_packet_size = 128M
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
|
||||||
|
# mysql_version_string =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
|
||||||
|
# net_workers = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
|
||||||
|
# net_wait_tm = -1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
|
||||||
|
# net_throttle_accept = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
|
||||||
|
# net_throttle_action = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#node_address
|
||||||
|
# node_address =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
|
||||||
|
# ondisk_attrs_default = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
|
||||||
|
# persistent_connections_limit =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
|
||||||
|
pid_file = /var/run/manticore/searchd.pid
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
|
||||||
|
# predicted_time_costs = doc=64, hit=48, skip=2048, match=64
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
|
||||||
|
# preopen_indexes = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
|
||||||
|
qcache_max_bytes = 128Mb
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
|
||||||
|
qcache_thresh_msec = 150
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
|
||||||
|
qcache_ttl_sec = 120
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
|
||||||
|
query_log_format = sphinxql
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
|
||||||
|
# query_log_min_msec = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log
|
||||||
|
# query_log = /var/log/manticore/query.log
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
|
||||||
|
# query_log_mode = 600
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
|
||||||
|
# max_connections =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
|
||||||
|
# network_timeout = 5
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
|
||||||
|
# read_buffer = 256K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
|
||||||
|
# read_buffer_docs = 256K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
|
||||||
|
# read_buffer_hits = 256K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
|
||||||
|
# read_unhinted 32K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
|
||||||
|
# rt_flush_period =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
|
||||||
|
# rt_merge_iops = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
|
||||||
|
# rt_merge_maxiosize = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
|
||||||
|
# seamless_rotate = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#server_id
|
||||||
|
# server_id =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
|
||||||
|
# shutdown_timeout = 3
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
|
||||||
|
# shutdown_token =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
|
||||||
|
# snippets_file_prefix =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
|
||||||
|
# sphinxql_state =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
|
||||||
|
# sphinxql_timeout = 900
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
|
||||||
|
# ssl_ca =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
|
||||||
|
# ssl_cert =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
|
||||||
|
# ssl_key =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
|
||||||
|
# subtree_docs_cache = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
|
||||||
|
# subtree_hits_cache = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
|
||||||
|
# thread_stack =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
|
||||||
|
# unlink_old = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
|
||||||
|
# watchdog = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
common {
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
|
||||||
|
# lemmatizer_base = /usr/local/share
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
|
||||||
|
# progressive_merge =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
|
||||||
|
# json_autoconv_keynames =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
|
||||||
|
# json_autoconv_numbers = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
|
||||||
|
# on_json_attr_error = ignore_attr
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
|
||||||
|
# plugin_dir =
|
||||||
|
}
|
||||||
|
|
||||||
|
# indexer {
|
||||||
|
# lemmatizer_cache = 1024M
|
||||||
|
# max_iops = 0
|
||||||
|
# max_iosize = 0
|
||||||
|
# mem_limit = 1024M
|
||||||
|
# }
|
||||||
|
|
||||||
|
EOF
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
_HiStOrY_V2_
|
||||||
|
SELECT * FROM films WHERE MATCH('"shark monkey boy robot"/2') AND release_year IN(2006,2007) AND rental_rate BETWEEN 2.0 and 3.0;
|
||||||
|
SELECT title, HIGHLIGHT({},'description') FROM films WHERE MATCH('"shark monkey boy robot"/2');
|
||||||
|
SELECT * FROM films WHERE MATCH('" shark monkey boy robot "/2');
|
||||||
|
SELECT * FROM films WHERE MATCH('Emotional drama') FACET release_year FACET category_id;
|
||||||
|
SELECT * FROM films WHERE MATCH('Emotional drama') GROUP BY release_year;
|
||||||
|
SELECT * FROM films WHERE MATCH('Emotional drama -dog -shark');
|
||||||
|
SELECT * FROM films WHERE MATCH('Emotional drama');
|
||||||
|
SELECT * FROM films;
|
||||||
|
DESCRIBE films;
|
||||||
|
SHOW TABLES;
|
||||||
|
SOURCE /sandbox.sql
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
FROM ubuntu:focal
|
||||||
|
|
||||||
|
ARG DEV
|
||||||
|
ARG DAEMON_URL
|
||||||
|
ARG MCL_URL
|
||||||
|
|
||||||
|
RUN groupadd -r manticore && useradd -r -g manticore manticore
|
||||||
|
|
||||||
|
ENV GOSU_VERSION 1.11
|
||||||
|
ENV MCL_URL=${MCL_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/focal/main/binary-amd64/manticore-columnar-lib_1.15.4-220522-2fef34e_amd64.deb"}
|
||||||
|
ENV DAEMON_URL=${DAEMON_URL:-"https://repo.manticoresearch.com/repository/manticoresearch_focal/dists/manticore_5.0.2-220530-348514c86_amd64.tgz"}
|
||||||
|
ENV BETA_URL=${BETA_URL:-"https://repo.manticoresearch.com/repository/kibana_beta/ubuntu/focal.zip"}
|
||||||
|
|
||||||
|
|
||||||
|
RUN set -x \
|
||||||
|
&& apt-get update && apt-get -y install --no-install-recommends ca-certificates binutils wget gnupg dirmngr unzip && rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
|
||||||
|
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
|
||||||
|
&& export GNUPGHOME="$(mktemp -d)" \
|
||||||
|
&& gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
|
||||||
|
&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
|
||||||
|
&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
|
||||||
|
&& rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc \
|
||||||
|
&& chmod +x /usr/local/bin/gosu \
|
||||||
|
&& gosu nobody true && \
|
||||||
|
if [ "${DEV}" = "1" ]; then \
|
||||||
|
echo "DEV IS ONE" && \
|
||||||
|
exit && \
|
||||||
|
wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb \
|
||||||
|
&& dpkg -i manticore-dev-repo.noarch.deb \
|
||||||
|
&& apt-key adv --fetch-keys 'https://repo.manticoresearch.com/GPG-KEY-manticore' && apt-get -y update && apt-get -y install manticore \
|
||||||
|
&& apt-get update \
|
||||||
|
&& echo $(apt-get -y download --print-uris manticore-columnar-lib | cut -d" " -f1 | cut -d "'" -f 2) > /mcl.url ;\
|
||||||
|
elif [ "${DEV}" = "2" ]; then \
|
||||||
|
echo "DEV IS TWO" && \
|
||||||
|
wget $BETA_URL && unzip focal.zip && rm focal.zip && \
|
||||||
|
dpkg -i build/* && echo $MCL_URL > /mcl.url; rm build/* ;\
|
||||||
|
else \
|
||||||
|
echo "DEV NOT EITHER" && \
|
||||||
|
exit && \
|
||||||
|
wget $DAEMON_URL && ARCHIVE_NAME=$(ls | grep '.tgz' | head -n1 ) && tar -xf $ARCHIVE_NAME && rm $ARCHIVE_NAME && \
|
||||||
|
dpkg -i manticore* && echo $MCL_URL > /mcl.url && rm *.deb ; \
|
||||||
|
fi \
|
||||||
|
&& mkdir -p /var/run/manticore && mkdir -p /var/lib/manticore/replication \
|
||||||
|
&& apt-get update && apt-get -y install libexpat1 libodbc1 libpq5 openssl libcrypto++6 libmysqlclient21 mysql-client \
|
||||||
|
&& apt-get -y purge --auto-remove \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& rm -f /usr/bin/mariabackup /usr/bin/mysqldump /usr/bin/mysqlslap /usr/bin/mysqladmin /usr/bin/mysqlimport \
|
||||||
|
/usr/bin/mysqlshow /usr/bin/mbstream /usr/bin/mysql_waitpid /usr/bin/innotop /usr/bin/mysqlaccess /usr/bin/mytop \
|
||||||
|
/usr/bin/mysqlreport /usr/bin/mysqldumpslow /usr/bin/mysql_find_rows /usr/bin/mysql_fix_extensions \
|
||||||
|
/usr/bin/mysql_embedded /usr/bin/mysqlcheck \
|
||||||
|
&& rm -f /usr/bin/spelldump /usr/bin/wordbreaker \
|
||||||
|
&& mkdir -p /var/run/mysqld/ && chown manticore:manticore /var/run/mysqld/ \
|
||||||
|
&& echo "\n[mysql]\nsilent\nwait\ntable\n" >> /etc/mysql/my.cnf && \
|
||||||
|
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/en.pak.tgz && \
|
||||||
|
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/de.pak.tgz && \
|
||||||
|
wget -P /tmp https://repo.manticoresearch.com/repository/morphology/ru.pak.tgz && \
|
||||||
|
tar -xf /tmp/en.pak.tgz -C /usr/share/manticore/ && \
|
||||||
|
tar -xf /tmp/de.pak.tgz -C /usr/share/manticore/ && \
|
||||||
|
tar -xf /tmp/ru.pak.tgz -C /usr/share/manticore/
|
||||||
|
|
||||||
|
|
||||||
|
COPY manticore.conf /etc/manticoresearch/
|
||||||
|
COPY sandbox.sql /sandbox.sql
|
||||||
|
COPY .mysql_history /root/.mysql_history
|
||||||
|
|
||||||
|
COPY docker-entrypoint.sh /usr/local/bin/
|
||||||
|
RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh
|
||||||
|
WORKDIR /var/lib/manticore
|
||||||
|
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||||
|
EXPOSE 9306
|
||||||
|
EXPOSE 9308
|
||||||
|
EXPOSE 9312
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
ENV LC_ALL C.UTF-8
|
||||||
|
CMD ["sh", "-c", "(echo 'START WAIT' && sleep 5 && echo 'END WAIT' && mysql -P9306 -h0 -e 'set global log_management = 0; set global log_management = 1;') & searchd --nodetach"]
|
||||||
|
|
@ -0,0 +1,278 @@
|
||||||
|
# Manticore Search Docker image
|
||||||
|
|
||||||
|
This is the git repo of official [Docker image](https://hub.docker.com/r/manticoresearch/manticore/) for [Manticore Search](https://github.com/manticoresoftware/manticoresearch).
|
||||||
|
|
||||||
|
Manticore Search is an easy to use open source fast database for search. It helps thousands of companies from small to large, such as Craigslist, to search and filter petabytes of text data on a single or hundreds of nodes, do stream full-text filtering, add auto-complete, spell correction, more-like-this, faceting and other search-related technologies to their websites and applications.
|
||||||
|
|
||||||
|
The default configuration includes a sample Real-Time index and listens on the default ports:
|
||||||
|
* `9306` for connections from a MySQL client
|
||||||
|
* `9308` for connections via HTTP
|
||||||
|
* `9312` for connections via a binary protocol (e.g. in case you run a cluster)
|
||||||
|
|
||||||
|
The image comes with libraries for easy indexing data from MySQL, PostgreSQL XML and CSV files.
|
||||||
|
|
||||||
|
# How to run Manticore Search Docker image
|
||||||
|
|
||||||
|
## Quick usage
|
||||||
|
|
||||||
|
The below is the simplest way to start Manticore in a container and log in to it via mysql client:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --name manticore --rm -d manticoresearch/manticore && sleep 3 && docker exec -it manticore mysql && docker stop manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
When you exit from the mysql client it stops and removes the container, so **use it only for testing / sandboxing purposes**. See below how to use it in production.
|
||||||
|
|
||||||
|
The image comes with a sample index which can be loaded like this:
|
||||||
|
|
||||||
|
```mysql
|
||||||
|
mysql> source /sandbox.sql
|
||||||
|
```
|
||||||
|
|
||||||
|
Also the mysql client has in history several sample queries that you can run on the above index, just use Up/Down keys in the client to see and run them.
|
||||||
|
|
||||||
|
## Production use
|
||||||
|
|
||||||
|
|
||||||
|
### Ports and mounting points
|
||||||
|
|
||||||
|
For data persistence `/var/lib/manticore/` should be mounted to local storage or other desired storage engine.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --name manticore -v $(pwd)/data:/var/lib/manticore -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
Configuration file inside the instance is located at `/etc/manticoresearch/manticore.conf`. For custom settings, this file should be mounted to your own configuration file.
|
||||||
|
|
||||||
|
The ports are 9306/9308/9312 for SQL/HTTP/Binary, expose them depending on how you are going to use Manticore. For example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data:/var/lib/manticore/ -p 127.0.0.1:9306:9306 -p 127.0.0.1:9308:9308 -d manticoresearch/manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
Make sure to remove `127.0.0.1:` if you want the ports to be available for external hosts.
|
||||||
|
|
||||||
|
### Manticore Columnar Library
|
||||||
|
|
||||||
|
The docker image doesn't include [Manticore Columnar Library](https://github.com/manticoresoftware/columnar) which has to be used if you need:
|
||||||
|
* columnar storage
|
||||||
|
* secondary indexes
|
||||||
|
|
||||||
|
but you can easily enable it in runtime by using environment variable `MCL=1`, i.e. `docker run -e MCL=1 ... manticoresearch/manticore`. It will then download and install the library and put it to the data dir (which is normally mapped as a volume in production). Next time you run the container the library will be already there, hence it won't be downloaded again unless you change the Manticore Search version.
|
||||||
|
|
||||||
|
### Docker-compose
|
||||||
|
|
||||||
|
In many cases you might want to use Manticore together with other images specified in a docker-compose YAML file. Here is the minimal recommended specification for Manticore Search in docker-compose.yml:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
version: '2.2'
|
||||||
|
|
||||||
|
services:
|
||||||
|
manticore:
|
||||||
|
container_name: manticore
|
||||||
|
image: manticoresearch/manticore
|
||||||
|
restart: always
|
||||||
|
ports:
|
||||||
|
- 127.0.0.1:9306:9306
|
||||||
|
- 127.0.0.1:9308:9308
|
||||||
|
ulimits:
|
||||||
|
nproc: 65535
|
||||||
|
nofile:
|
||||||
|
soft: 65535
|
||||||
|
hard: 65535
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
environment:
|
||||||
|
- MCL=1
|
||||||
|
volumes:
|
||||||
|
- ./data:/var/lib/manticore
|
||||||
|
# - ./manticore.conf:/etc/manticoresearch/manticore.conf # uncommment if you use a custom config
|
||||||
|
```
|
||||||
|
|
||||||
|
Besides using the exposed ports 9306 and 9308 you can log into the instance by running `docker-compose exec manticore mysql`.
|
||||||
|
|
||||||
|
### HTTP protocol
|
||||||
|
|
||||||
|
Manticore is accessible via HTTP on ports 9308 and 9312. You can map either of them locally and connect with curl:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --name manticore -p 9308:9308 -d manticoresearch/manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a table:
|
||||||
|
```bash
|
||||||
|
curl -X POST 'http://127.0.0.1:9308/sql' -d 'mode=raw&query=CREATE TABLE testrt ( title text, content text, gid integer)'
|
||||||
|
```
|
||||||
|
Insert a document:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST 'http://127.0.0.1:9308/json/insert' -d'{"index":"testrt","id":1,"doc":{"title":"Hello","content":"world","gid":1}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Perform a simple search:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST 'http://127.0.0.1:9308/json/search' -d '{"index":"testrt","query":{"match":{"*":"hello world"}}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Logging
|
||||||
|
|
||||||
|
By default, Manticore logs to `/dev/stdout`, so you can watch the log on the host with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker logs manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
If you want to get log of your queries the same way you can do it by passing environment variable `QUERY_LOG_TO_STDOUT=true`.
|
||||||
|
|
||||||
|
### Multi-node cluster with replication
|
||||||
|
|
||||||
|
Here is a simple `docker-compose.yml` for defining a two node cluster:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
version: '2.2'
|
||||||
|
|
||||||
|
services:
|
||||||
|
|
||||||
|
manticore-1:
|
||||||
|
image: manticoresearch/manticore
|
||||||
|
restart: always
|
||||||
|
ulimits:
|
||||||
|
nproc: 65535
|
||||||
|
nofile:
|
||||||
|
soft: 65535
|
||||||
|
hard: 65535
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
environment:
|
||||||
|
- MCL=1
|
||||||
|
networks:
|
||||||
|
- manticore
|
||||||
|
manticore-2:
|
||||||
|
image: manticoresearch/manticore
|
||||||
|
restart: always
|
||||||
|
ulimits:
|
||||||
|
nproc: 65535
|
||||||
|
nofile:
|
||||||
|
soft: 65535
|
||||||
|
hard: 65535
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
environment:
|
||||||
|
- MCL=1
|
||||||
|
networks:
|
||||||
|
- manticore
|
||||||
|
networks:
|
||||||
|
manticore:
|
||||||
|
driver: bridge
|
||||||
|
```
|
||||||
|
* Start it: `docker-compose up`
|
||||||
|
* Create a cluster with a table:
|
||||||
|
```mysql
|
||||||
|
$ docker-compose exec manticore-1 mysql
|
||||||
|
|
||||||
|
mysql> CREATE TABLE testrt ( title text, content text, gid integer);
|
||||||
|
|
||||||
|
mysql> CREATE CLUSTER posts;
|
||||||
|
Query OK, 0 rows affected (0.24 sec)
|
||||||
|
|
||||||
|
mysql> ALTER CLUSTER posts ADD testrt;
|
||||||
|
Query OK, 0 rows affected (0.07 sec)
|
||||||
|
|
||||||
|
MySQL [(none)]> exit
|
||||||
|
Bye
|
||||||
|
```
|
||||||
|
* Join to the the cluster on the 2nd instance and insert smth to the table:
|
||||||
|
```mysql
|
||||||
|
$ docker-compose exec manticore-2 mysql
|
||||||
|
|
||||||
|
mysql> JOIN CLUSTER posts AT 'manticore-1:9312';
|
||||||
|
mysql> INSERT INTO posts:testrt(title,content,gid) VALUES('hello','world',1);
|
||||||
|
Query OK, 1 row affected (0.00 sec)
|
||||||
|
|
||||||
|
MySQL [(none)]> exit
|
||||||
|
Bye
|
||||||
|
```
|
||||||
|
|
||||||
|
* If you now go back to the first instance you'll see the new record:
|
||||||
|
```mysql
|
||||||
|
$ docker-compose exec manticore-1 mysql
|
||||||
|
|
||||||
|
MySQL [(none)]> select * from testrt;
|
||||||
|
+---------------------+------+-------+---------+
|
||||||
|
| id | gid | title | content |
|
||||||
|
+---------------------+------+-------+---------+
|
||||||
|
| 3891565839006040065 | 1 | hello | world |
|
||||||
|
+---------------------+------+-------+---------+
|
||||||
|
1 row in set (0.00 sec)
|
||||||
|
|
||||||
|
MySQL [(none)]> exit
|
||||||
|
Bye
|
||||||
|
```
|
||||||
|
|
||||||
|
## Memory locking and limits
|
||||||
|
|
||||||
|
It's recommended to overwrite the default ulimits of docker for the Manticore instance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
--ulimit nofile=65536:65536
|
||||||
|
```
|
||||||
|
|
||||||
|
For the best performance, Manticore tables' components can be locked into memory. When Manticore is run under Docker, the instance requires additional privileges to allow memory locking. The following options must be added when running the instance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
--cap-add=IPC_LOCK --ulimit memlock=-1:-1
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuring Manticore Search with Docker
|
||||||
|
|
||||||
|
If you want to run Manticore with your custom config containing indexes definition you will need to mount the configuration to the instance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --name manticore -v $(pwd)/manticore.conf:/etc/manticoresearch/manticore.conf -v $(pwd)/data/:/var/lib/manticore -p 127.0.0.1:9306:9306 -d manticoresearch/manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
Take into account that Manticore search inside the container is run under user `manticore`. Performing operations with tables (like creating or rotating plain indexes) should be also done under `manticore`. Otherwise the files will be created under `root` and the search daemon won't have rights to open them. For example here is how you can rotate all plain indexes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it manticore gosu manticore indexer --all --rotate
|
||||||
|
```
|
||||||
|
|
||||||
|
### Environment variables
|
||||||
|
|
||||||
|
You can also set individual `searchd` and `common` configuration settings using Docker environment variables.
|
||||||
|
|
||||||
|
The settings must be prefixed with their section name, for example to change value of setting `mysql_version_string` in section `searchd` the variable must be named `searchd_mysql_version_string`:
|
||||||
|
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --name manticore -p 127.0.0.1:9306:9306 -e searchd_mysql_version_string='5.5.0' -d manticoresearch/manticore
|
||||||
|
```
|
||||||
|
|
||||||
|
In case of `listen` directive, you can pass using Docker variable `searchd_listen` new listening interfaces in addition to the default ones. Multiple interfaces can be declared separated by semi-colon ("|").
|
||||||
|
For listening only on network address, the `$ip` (retrieved internally from `hostname -i`) can be used as address alias.
|
||||||
|
|
||||||
|
For example `-e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip'` will add an additional SQL interface on port 9307, a SQL VIP on 5443 running only on the instance IP and HTTP on port 9316, beside the defaults on 9306 and 9308, respectively.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ docker run --rm -p 1188:9307 -e searchd_mysql_version_string='5.5.0' -e searchd_listen='9316:http|9307:mysql|$ip:5443:mysql_vip' manticore
|
||||||
|
[Mon Aug 17 07:31:58.719 2020] [1] using config file '/etc/manticoresearch/manticore.conf' (9130 chars)...
|
||||||
|
listening on all interfaces for http, port=9316
|
||||||
|
listening on all interfaces for mysql, port=9307
|
||||||
|
listening on 172.17.0.17:5443 for VIP mysql
|
||||||
|
listening on all interfaces for mysql, port=9306
|
||||||
|
listening on UNIX socket /var/run/mysqld/mysqld.sock
|
||||||
|
listening on 172.17.0.17:9312 for sphinx
|
||||||
|
listening on all interfaces for http, port=9308
|
||||||
|
prereading 0 indexes
|
||||||
|
prereaded 0 indexes in 0.000 sec
|
||||||
|
accepting connections
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
# Issues
|
||||||
|
|
||||||
|
For reporting issues, please use the [issue tracker](https://github.com/manticoresoftware/docker/issues).
|
||||||
|
|
@ -0,0 +1,118 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -eo pipefail
|
||||||
|
echo "RUNNING ENTRYPOINT"
|
||||||
|
|
||||||
|
# check to see if this file is being run or sourced from another script
|
||||||
|
_is_sourced() {
|
||||||
|
# https://unix.stackexchange.com/a/215279
|
||||||
|
[ "${#FUNCNAME[@]}" -ge 2 ] &&
|
||||||
|
[ "${FUNCNAME[0]}" = '_is_sourced' ] &&
|
||||||
|
[ "${FUNCNAME[1]}" = 'source' ]
|
||||||
|
}
|
||||||
|
_searchd_want_help() {
|
||||||
|
local arg
|
||||||
|
for arg; do
|
||||||
|
case "$arg" in
|
||||||
|
-'?' | --help | -h | -v)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
docker_setup_env() {
|
||||||
|
if [ -n "$QUERY_LOG_TO_STDOUT" ]; then
|
||||||
|
export searchd_query_log=/var/log/manticore/query.log
|
||||||
|
[ ! -f /var/log/manticore/query.log ] && ln -sf /dev/stdout /var/log/manticore/query.log
|
||||||
|
fi
|
||||||
|
if [[ "${MCL}" == "1" ]]; then
|
||||||
|
LIB_MANTICORE_COLUMNAR="/var/lib/manticore/.mcl/lib_manticore_columnar.so"
|
||||||
|
LIB_MANTICORE_SECONDARY="/var/lib/manticore/.mcl/lib_manticore_secondary.so"
|
||||||
|
|
||||||
|
[ -L /usr/share/manticore/modules/lib_manticore_columnar.so ] || ln -s $LIB_MANTICORE_COLUMNAR /usr/share/manticore/modules/lib_manticore_columnar.so
|
||||||
|
[ -L /usr/share/manticore/modules/lib_manticore_secondary.so ] || ln -s $LIB_MANTICORE_SECONDARY /usr/share/manticore/modules/lib_manticore_secondary.so
|
||||||
|
|
||||||
|
searchd -v|grep -i error|egrep "trying to load" \
|
||||||
|
&& rm $LIB_MANTICORE_COLUMNAR $LIB_MANTICORE_SECONDARY \
|
||||||
|
&& echo "WARNING: wrong MCL version was removed, installing the correct one"
|
||||||
|
|
||||||
|
if [[ ! -f "$LIB_MANTICORE_COLUMNAR" || ! -f "$LIB_MANTICORE_SECONDARY" ]]; then
|
||||||
|
if ! mkdir -p /var/lib/manticore/.mcl/ ; then
|
||||||
|
echo "ERROR: Manticore Columnar Library is inaccessible: couldn't create /var/lib/manticore/.mcl/."
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
|
MCL_URL=$(cat /mcl.url)
|
||||||
|
wget -P /tmp $MCL_URL
|
||||||
|
|
||||||
|
LAST_PATH=$(pwd)
|
||||||
|
cd /tmp
|
||||||
|
PACKAGE_NAME=$(ls | grep manticore-columnar | head -n 1)
|
||||||
|
ar -x $PACKAGE_NAME
|
||||||
|
tar -xf data.tar.gz
|
||||||
|
find . -name '*.so' -exec cp {} /var/lib/manticore/.mcl/ \;
|
||||||
|
cd $LAST_PATH
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
_main() {
|
||||||
|
# first arg is `h` or some `--option`
|
||||||
|
if [ "${1#-}" != "$1" ]; then
|
||||||
|
set -- searchd "$@"
|
||||||
|
fi
|
||||||
|
# Amended from searchd to sh since we're using sh to wait until searchd starts, then set the Kibana-specific options
|
||||||
|
if [ "$1" = 'sh' ] && ! _searchd_want_help "@"; then
|
||||||
|
docker_setup_env "$@"
|
||||||
|
# allow the container to be started with `--user`
|
||||||
|
if [ "$(id -u)" = '0' ]; then
|
||||||
|
find /var/lib/manticore /var/log/manticore /var/run/manticore /etc/manticoresearch \! -user manticore -exec chown manticore '{}' +
|
||||||
|
exec gosu manticore "$0" "$@"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
_replace_conf_from_env
|
||||||
|
exec "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
_replace_conf_from_env() {
|
||||||
|
|
||||||
|
sed_query=""
|
||||||
|
|
||||||
|
while IFS='=' read -r oldname value; do
|
||||||
|
if [[ $oldname == 'searchd_'* || $oldname == 'common_'* ]]; then
|
||||||
|
value=$(echo ${!oldname} | sed 's/\//\\\//g')
|
||||||
|
oldname=$(echo $oldname | sed "s/searchd_//g;s/common_//g;")
|
||||||
|
newname=$oldname
|
||||||
|
|
||||||
|
if [[ $newname == 'listen' ]]; then
|
||||||
|
oldname="listen_env"
|
||||||
|
IFS='|' read -ra ADDR <<<"$value"
|
||||||
|
count=0
|
||||||
|
|
||||||
|
for i in "${ADDR[@]}"; do
|
||||||
|
if [[ $count == 0 ]]; then
|
||||||
|
value=$i
|
||||||
|
else
|
||||||
|
value="$value\n listen = $i"
|
||||||
|
fi
|
||||||
|
count=$((count + 1))
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z $sed_query ]]; then
|
||||||
|
sed_query="s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
|
||||||
|
else
|
||||||
|
sed_query="$sed_query;s/(#\s)*?$oldname\s?=\s?.*?$/$newname = $value/g"
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
done < <(env)
|
||||||
|
|
||||||
|
if [[ ! -z $sed_query ]]; then
|
||||||
|
sed -i -E "$sed_query" /etc/manticoresearch/manticore.conf
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
# If we are sourced from elsewhere, don't perform any further actions
|
||||||
|
if ! _is_sourced; then
|
||||||
|
_main "$@"
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,259 @@
|
||||||
|
#!/bin/sh
|
||||||
|
ip=`hostname -i|rev|cut -d\ -f 1|rev`
|
||||||
|
cat << EOF
|
||||||
|
searchd {
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_plain_attrs
|
||||||
|
# access_plain_attrs = mmap_preread
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_blob_attrs
|
||||||
|
# access_blob_attrs = mmap_preread
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_doclists
|
||||||
|
# access_doclists = file
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#access_hitlists
|
||||||
|
# access_hitlists = file
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_connect_timeout
|
||||||
|
# agent_connect_timeout =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_query_timeout
|
||||||
|
# agent_query_timeout =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_count
|
||||||
|
# agent_retry_count = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#agent_retry_delay
|
||||||
|
# agent_retry_delay = 500
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#attr_flush_period
|
||||||
|
# attr_flush_period = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_flush
|
||||||
|
# binlog_flush = 2
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_max_log_size
|
||||||
|
# binlog_max_log_size = 268435456
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#binlog_path
|
||||||
|
# binlog_path =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#client_timeout
|
||||||
|
# client_timeout = 300
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_libc_locale
|
||||||
|
# collation_libc_locale = C
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#collation_server
|
||||||
|
# collation_server = libc_ci
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#data_dir
|
||||||
|
data_dir = /var/lib/manticore
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#docstore_cache_size
|
||||||
|
# docstore_cache_size = 16m
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#expansion_limit
|
||||||
|
# expansion_limit = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#grouping_in_utc
|
||||||
|
# grouping_in_utc = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_period_karma
|
||||||
|
# ha_period_karma = 60
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ha_ping_interval
|
||||||
|
# ha_ping_interval = 1000
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#hostname_lookup
|
||||||
|
# hostname_lookup =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#jobs_queue_size
|
||||||
|
# jobs_queue_size =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_backlog
|
||||||
|
# listen_backlog = 5
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#listen
|
||||||
|
# listen_env = this directive allows to append listeners from environment variables
|
||||||
|
|
||||||
|
listen = 9306:mysql41
|
||||||
|
listen = /var/run/mysqld/mysqld.sock:mysql41
|
||||||
|
listen = $ip:9312
|
||||||
|
listen = 9308:http
|
||||||
|
listen = $ip:9315-9325:replication
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#listen_tfo
|
||||||
|
# listen_tfo = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#log
|
||||||
|
log = /var/log/manticore/searchd.log
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_batch_queries
|
||||||
|
# max_batch_queries = 32
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#threads
|
||||||
|
# threads =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filters
|
||||||
|
# max_filters = 256
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_filter_values
|
||||||
|
# max_filter_values = 4096
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_open_files
|
||||||
|
# max_open_files =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_packet_size
|
||||||
|
max_packet_size = 128M
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#mysql_version_string
|
||||||
|
# mysql_version_string =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_workers
|
||||||
|
# net_workers = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_wait_tm
|
||||||
|
# net_wait_tm = -1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_accept
|
||||||
|
# net_throttle_accept = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#net_throttle_action
|
||||||
|
# net_throttle_action = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#node_address
|
||||||
|
# node_address =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ondisk_attrs_default
|
||||||
|
# ondisk_attrs_default = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#persistent_connections_limit
|
||||||
|
# persistent_connections_limit =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#pid_file
|
||||||
|
pid_file = /var/run/manticore/searchd.pid
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#predicted_time_costs
|
||||||
|
# predicted_time_costs = doc=64, hit=48, skip=2048, match=64
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#preopen_indexes
|
||||||
|
# preopen_indexes = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_max_bytes
|
||||||
|
# qcache_max_bytes = 16Mb
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_thresh_msec
|
||||||
|
# qcache_thresh_msec = 3000
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#qcache_ttl_sec
|
||||||
|
# qcache_ttl_sec = 60
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_format
|
||||||
|
query_log_format = sphinxql
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_min_msec
|
||||||
|
query_log_min_msec = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log
|
||||||
|
query_log = /var/log/manticore/query.log
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#query_log_mode
|
||||||
|
# query_log_mode = 600
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#max_connections
|
||||||
|
# max_connections =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#network_timeout
|
||||||
|
# network_timeout = 5
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer
|
||||||
|
# read_buffer = 256K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_docs
|
||||||
|
# read_buffer_docs = 256K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_buffer_hits
|
||||||
|
# read_buffer_hits = 256K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#read_unhinted
|
||||||
|
# read_unhinted 32K
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_flush_period
|
||||||
|
# rt_flush_period =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_iops
|
||||||
|
# rt_merge_iops = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#rt_merge_maxiosize
|
||||||
|
# rt_merge_maxiosize = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#seamless_rotate
|
||||||
|
# seamless_rotate = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#server_id
|
||||||
|
# server_id =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_timeout
|
||||||
|
# shutdown_timeout = 3
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#shutdown_token
|
||||||
|
# shutdown_token =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#snippets_file_prefix
|
||||||
|
# snippets_file_prefix =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_state
|
||||||
|
# sphinxql_state =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#sphinxql_timeout
|
||||||
|
# sphinxql_timeout = 900
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_ca
|
||||||
|
# ssl_ca =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_cert
|
||||||
|
# ssl_cert =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#ssl_key
|
||||||
|
# ssl_key =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_docs_cache
|
||||||
|
# subtree_docs_cache = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#subtree_hits_cache
|
||||||
|
# subtree_hits_cache = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#thread_stack
|
||||||
|
# thread_stack =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#unlink_old
|
||||||
|
# unlink_old = 1
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Searchd#watchdog
|
||||||
|
# watchdog = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
common {
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#lemmatizer_base
|
||||||
|
# lemmatizer_base = /usr/local/share
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#progressive_merge
|
||||||
|
# progressive_merge =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_keynames
|
||||||
|
# json_autoconv_keynames =
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#json_autoconv_numbers
|
||||||
|
# json_autoconv_numbers = 0
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#on_json_attr_error
|
||||||
|
# on_json_attr_error = ignore_attr
|
||||||
|
|
||||||
|
# https://manual.manticoresearch.com/Server_settings/Common#plugin_dir
|
||||||
|
# plugin_dir =
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
EOF
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,2 @@
|
||||||
|
unixsocket /var/run/redis/redis.sock
|
||||||
|
unixsocketperm 777
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
wheel
|
||||||
|
beautifulsoup4
|
||||||
|
redis
|
||||||
|
siphashc
|
||||||
|
aiohttp[speedups]
|
||||||
|
python-dotenv
|
||||||
|
#manticoresearch
|
||||||
|
numpy
|
||||||
|
ujson
|
||||||
|
aioredis[hiredis]
|
||||||
|
aiokafka
|
||||||
|
vaderSentiment
|
||||||
|
polyglot
|
||||||
|
pyicu
|
||||||
|
pycld2
|
||||||
|
morfessor
|
||||||
|
six
|
||||||
|
nltk
|
||||||
|
spacy
|
||||||
|
python-Levenshtein
|
||||||
|
orjson
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
# Java tuning
|
||||||
|
DRUID_XMX=1g
|
||||||
|
DRUID_XMS=1g
|
||||||
|
DRUID_MAXNEWSIZE=250m
|
||||||
|
DRUID_NEWSIZE=250m
|
||||||
|
DRUID_MAXDIRECTMEMORYSIZE=6172m
|
||||||
|
|
||||||
|
druid_emitter_logging_logLevel=debug
|
||||||
|
|
||||||
|
druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
|
||||||
|
|
||||||
|
druid_zk_service_host=zookeeper
|
||||||
|
|
||||||
|
druid_metadata_storage_host=
|
||||||
|
druid_metadata_storage_type=postgresql
|
||||||
|
druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
|
||||||
|
druid_metadata_storage_connector_user=druid
|
||||||
|
druid_metadata_storage_connector_password=FoolishPassword
|
||||||
|
|
||||||
|
druid_coordinator_balancer_strategy=cachingCost
|
||||||
|
|
||||||
|
druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
|
||||||
|
druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
|
||||||
|
|
||||||
|
druid_storage_type=local
|
||||||
|
druid_storage_storageDirectory=/opt/shared/segments
|
||||||
|
druid_indexer_logs_type=file
|
||||||
|
druid_indexer_logs_directory=/opt/shared/indexing-logs
|
||||||
|
|
||||||
|
druid_processing_numThreads=2
|
||||||
|
druid_processing_numMergeBuffers=2
|
||||||
|
|
||||||
|
DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
THRESHOLD_LISTENER_HOST=0.0.0.0
|
||||||
|
THRESHOLD_LISTENER_PORT=13867
|
||||||
|
THRESHOLD_LISTENER_SSL=1
|
||||||
|
|
||||||
|
THRESHOLD_RELAY_ENABLED=1
|
||||||
|
THRESHOLD_RELAY_HOST=0.0.0.0
|
||||||
|
THRESHOLD_RELAY_PORT=13868
|
||||||
|
THRESHOLD_RELAY_SSL=1
|
||||||
|
|
||||||
|
THRESHOLD_API_ENABLED=1
|
||||||
|
THRESHOLD_API_HOST=0.0.0.0
|
||||||
|
THRESHOLD_API_PORT=13869
|
||||||
|
PORTAINER_GIT_DIR=.
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
*.pyc
|
||||||
|
*.pem
|
||||||
|
*.swp
|
||||||
|
__pycache__/
|
||||||
|
conf/live/
|
||||||
|
conf/cert/
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
.idea/
|
||||||
|
.env
|
||||||
|
.bash_history
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/psf/black
|
||||||
|
rev: 22.6.0
|
||||||
|
hooks:
|
||||||
|
- id: black
|
||||||
|
args:
|
||||||
|
- --line-length=120
|
||||||
|
- repo: https://github.com/PyCQA/isort
|
||||||
|
rev: 5.10.1
|
||||||
|
hooks:
|
||||||
|
- id: isort
|
||||||
|
args: ["--profile", "black"]
|
||||||
|
- repo: https://github.com/PyCQA/flake8
|
||||||
|
rev: 4.0.1
|
||||||
|
hooks:
|
||||||
|
- id: flake8
|
||||||
|
args: [--max-line-length=120]
|
||||||
|
|
@ -0,0 +1,746 @@
|
||||||
|
import functools
|
||||||
|
from json import JSONDecodeError, dumps, loads
|
||||||
|
from string import digits
|
||||||
|
|
||||||
|
import main
|
||||||
|
from klein import Klein
|
||||||
|
from modules import chankeep, helpers, provision, regproc, userinfo
|
||||||
|
from modules.network import Network
|
||||||
|
from twisted.web.server import Request
|
||||||
|
from utils.logging.log import warn
|
||||||
|
|
||||||
|
|
||||||
|
def login_required(func):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapper(self, *args, **kwargs):
|
||||||
|
if isinstance(args[0], Request):
|
||||||
|
request = args[0]
|
||||||
|
apikey = request.getHeader("ApiKey")
|
||||||
|
token = request.getHeader("Token")
|
||||||
|
if not apikey:
|
||||||
|
return "No API key provided"
|
||||||
|
if not token:
|
||||||
|
return "No token provided"
|
||||||
|
if apikey not in main.tokens:
|
||||||
|
return "No such API key"
|
||||||
|
config_token = main.tokens[apikey]
|
||||||
|
if not token == config_token["hello"]:
|
||||||
|
return "Invalid token"
|
||||||
|
counter = config_token["counter"]
|
||||||
|
request.setHeader("Counter", counter)
|
||||||
|
return func(self, *args, **kwargs)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
class API(object):
|
||||||
|
"""
|
||||||
|
Our API webapp.
|
||||||
|
"""
|
||||||
|
|
||||||
|
app = Klein()
|
||||||
|
|
||||||
|
@app.route("/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def hello(self, request):
|
||||||
|
return "Hello"
|
||||||
|
|
||||||
|
@app.route("/who/<query>/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def who(self, request, query):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "query" not in data:
|
||||||
|
return "No query provided"
|
||||||
|
result = userinfo.getWho(data["query"])
|
||||||
|
# Expand the generator
|
||||||
|
return dumps({k: [x for x in v] for k, v in result.items()})
|
||||||
|
|
||||||
|
@app.route("/chans/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def chans(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "net" not in data:
|
||||||
|
return "No net provided"
|
||||||
|
if "query" not in data:
|
||||||
|
return "No query provided"
|
||||||
|
if not data["query"]:
|
||||||
|
warn(f"No query provided: for chans {data}")
|
||||||
|
return dumps({})
|
||||||
|
result = userinfo.getChansSingle(data["net"], data["query"])
|
||||||
|
if not result:
|
||||||
|
return dumps({})
|
||||||
|
return dumps({"chans": [x for x in result]})
|
||||||
|
|
||||||
|
@app.route("/users/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def users(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "net" not in data:
|
||||||
|
return "No net provided"
|
||||||
|
if "query" not in data:
|
||||||
|
return "No query provided"
|
||||||
|
if not data["query"]:
|
||||||
|
warn(f"No query provided for users: {data}")
|
||||||
|
return dumps({})
|
||||||
|
result = userinfo.getUsersSingle(data["net"], data["query"])
|
||||||
|
if not result:
|
||||||
|
return dumps({})
|
||||||
|
return dumps({"users": [x for x in result]})
|
||||||
|
|
||||||
|
@app.route("/online/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def online(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "net" not in data:
|
||||||
|
return "No net provided"
|
||||||
|
if "query" not in data:
|
||||||
|
return "No users provided"
|
||||||
|
if not data["query"]:
|
||||||
|
warn(f"No users provided: for online {data}")
|
||||||
|
return dumps({})
|
||||||
|
net = data["net"]
|
||||||
|
usermap = {}
|
||||||
|
for user in data["query"]:
|
||||||
|
channels = userinfo.getChansSingle(net, [user])
|
||||||
|
if channels:
|
||||||
|
usermap[user] = True
|
||||||
|
else:
|
||||||
|
usermap[user] = False
|
||||||
|
|
||||||
|
return dumps(usermap)
|
||||||
|
|
||||||
|
@app.route("/num_users/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def num_users(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "net" not in data:
|
||||||
|
return "No net provided"
|
||||||
|
if "query" not in data:
|
||||||
|
return "No users provided"
|
||||||
|
if not data["query"]:
|
||||||
|
warn(f"No chans provided: for num_users {data}")
|
||||||
|
return dumps({})
|
||||||
|
net = data["net"]
|
||||||
|
results = userinfo.getUserNum(net, data["query"])
|
||||||
|
|
||||||
|
return dumps(results)
|
||||||
|
|
||||||
|
@app.route("/num_chans/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def num_chans(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "net" not in data:
|
||||||
|
return "No net provided"
|
||||||
|
if "query" not in data:
|
||||||
|
return "No users provided"
|
||||||
|
if not data["query"]:
|
||||||
|
warn(f"No users provided: for num_chans {data}")
|
||||||
|
return dumps({})
|
||||||
|
net = data["net"]
|
||||||
|
results = userinfo.getChanNum(net, data["query"])
|
||||||
|
|
||||||
|
return dumps(results)
|
||||||
|
|
||||||
|
@app.route("/irc/stats/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_stats(self, request):
|
||||||
|
stats = {}
|
||||||
|
numChannels = 0
|
||||||
|
numWhoEntries = 0
|
||||||
|
for i in main.IRCPool.keys():
|
||||||
|
numChannels += len(main.IRCPool[i].channels)
|
||||||
|
numWhoEntries += userinfo.getNumTotalWhoEntries()
|
||||||
|
numRelays = 0
|
||||||
|
for net in main.network.keys():
|
||||||
|
numRelays += len(main.network[net].relays)
|
||||||
|
stats["servers_total_total"] = numRelays
|
||||||
|
stats["servers_total_unique"] = len(main.network.keys())
|
||||||
|
stats["servers_online_total"] = len(main.IRCPool.keys())
|
||||||
|
stats["servers_online_unique"] = len(main.liveNets())
|
||||||
|
stats["channels"] = numChannels
|
||||||
|
stats["records"] = numWhoEntries
|
||||||
|
stats["eventrate"] = main.lastMinuteSample
|
||||||
|
return dumps(stats)
|
||||||
|
|
||||||
|
@app.route("/irc/networks/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_networks(self, request):
|
||||||
|
networks = {}
|
||||||
|
for net in main.network.keys():
|
||||||
|
networks[net] = {
|
||||||
|
"active": chankeep.allRelaysActive(net),
|
||||||
|
"relays": len(main.network[net].relays),
|
||||||
|
"channels": userinfo.getTotalChanNum(net),
|
||||||
|
"records": userinfo.getNumWhoEntries(net),
|
||||||
|
}
|
||||||
|
return dumps(networks)
|
||||||
|
|
||||||
|
@app.route("/irc/auth/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_recheckauth(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "func" not in data:
|
||||||
|
return dumps({"success": False, "reason": "no function specified."})
|
||||||
|
func = data["func"]
|
||||||
|
|
||||||
|
if "net" not in data:
|
||||||
|
return dumps({"success": False, "reason": "no net specified."})
|
||||||
|
net = data["net"]
|
||||||
|
if not net or net == "None":
|
||||||
|
nets = main.network.keys()
|
||||||
|
else:
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
nets = [net]
|
||||||
|
for net_name in nets:
|
||||||
|
conns = helpers.get_connected_relays(net_name)
|
||||||
|
if not conns:
|
||||||
|
return dumps(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"reason": f"failed to get instances for {net_name}.",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if func == "recheckauth":
|
||||||
|
for conn in conns:
|
||||||
|
conn.regPing(reset=True)
|
||||||
|
elif func == "resetauth":
|
||||||
|
for conn in conns:
|
||||||
|
conn.authenticated = False
|
||||||
|
conn.regPing(reset=True)
|
||||||
|
elif func == "register":
|
||||||
|
for conn in conns:
|
||||||
|
regproc.registerAccount(conn.net, conn.num)
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
inst = main.network[net]
|
||||||
|
network = {}
|
||||||
|
network["net"] = inst.net
|
||||||
|
network["auth"] = inst.auth
|
||||||
|
network["host"] = inst.host
|
||||||
|
network["last"] = inst.last
|
||||||
|
network["port"] = inst.port
|
||||||
|
network["security"] = inst.security
|
||||||
|
network["relays"] = len(inst.relays)
|
||||||
|
network["chanlimit"] = inst.chanlimit
|
||||||
|
network["channels"] = userinfo.getTotalChanNum(net)
|
||||||
|
network["records"] = userinfo.getNumWhoEntries(net)
|
||||||
|
return dumps(network)
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/", methods=["DELETE"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_delete(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
main.network[net].seppuku()
|
||||||
|
del main.network[net]
|
||||||
|
main.saveConf("network")
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/edit/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_edit(self, request, net):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
inst = main.network[net]
|
||||||
|
for item in data:
|
||||||
|
if item == "auth":
|
||||||
|
auth = data[item][0]
|
||||||
|
if auth not in ["sasl", "ns", "none"]:
|
||||||
|
return dumps({"success": False, "reason": "invalid auth."})
|
||||||
|
elif item == "host":
|
||||||
|
host = data[item][0]
|
||||||
|
elif item == "last":
|
||||||
|
last = data[item][0]
|
||||||
|
if not last.isdigit():
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "invalid last: not a number."}
|
||||||
|
)
|
||||||
|
elif item == "port":
|
||||||
|
port = data[item][0]
|
||||||
|
if not port.isdigit():
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "invalid port: not a number."}
|
||||||
|
)
|
||||||
|
port = int(port)
|
||||||
|
elif item == "chanlimit":
|
||||||
|
chanlimit = data[item][0]
|
||||||
|
if chanlimit == "None":
|
||||||
|
chanlimit = None
|
||||||
|
elif not chanlimit.isdigit():
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "invalid chanlimit: not a number."}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
chanlimit = int(chanlimit)
|
||||||
|
online_relays = helpers.get_connected_relays(net)
|
||||||
|
for r in online_relays:
|
||||||
|
r.chanlimit = chanlimit
|
||||||
|
elif item == "security":
|
||||||
|
security = data[item][0]
|
||||||
|
if security not in ["ssl", "plain"]:
|
||||||
|
return dumps({"success": False, "reason": "invalid security."})
|
||||||
|
inst.auth = auth
|
||||||
|
inst.host = host
|
||||||
|
inst.last = last
|
||||||
|
inst.port = port
|
||||||
|
inst.security = security
|
||||||
|
inst.chanlimit = chanlimit
|
||||||
|
main.saveConf("network")
|
||||||
|
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/create/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_create(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
fields = ["net", "auth", "host", "port", "security"]
|
||||||
|
if not set(fields).issubset(set(data)):
|
||||||
|
return dumps({"success": False, "reason": "not enough fields."})
|
||||||
|
for item in data:
|
||||||
|
if item == "net":
|
||||||
|
net = data[item]
|
||||||
|
if net in main.network.keys():
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "network already exists."}
|
||||||
|
)
|
||||||
|
if set(net).intersection(set(digits)):
|
||||||
|
return dumps(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"reason": "network name cannot contain numbers.",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
elif item == "auth":
|
||||||
|
auth = data[item]
|
||||||
|
if auth not in ["sasl", "ns", "none"]:
|
||||||
|
return dumps({"success": False, "reason": "invalid auth."})
|
||||||
|
elif item == "host":
|
||||||
|
host = data[item]
|
||||||
|
elif item == "port":
|
||||||
|
port = data[item]
|
||||||
|
if not port.isdigit():
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "invalid port: not a number."}
|
||||||
|
)
|
||||||
|
port = int(port)
|
||||||
|
elif item == "security":
|
||||||
|
security = data[item]
|
||||||
|
if security not in ["ssl", "plain"]:
|
||||||
|
return dumps({"success": False, "reason": "invalid security."})
|
||||||
|
main.network[net] = Network(net, host, int(port), security, auth)
|
||||||
|
main.saveConf("network")
|
||||||
|
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/relays/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_relays(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
relays_inst = main.network[net].relays
|
||||||
|
relays = []
|
||||||
|
for num in relays_inst.keys():
|
||||||
|
to_append = relays_inst[num]
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
to_append["chans"] = len(main.IRCPool[name].channels)
|
||||||
|
to_append["nick"] = main.IRCPool[name].nickname
|
||||||
|
to_append["conn"] = main.IRCPool[name].isconnected
|
||||||
|
to_append["limit"] = main.IRCPool[name].chanlimit
|
||||||
|
to_append["authed"] = main.IRCPool[name].authenticated
|
||||||
|
else:
|
||||||
|
to_append["chans"] = 0
|
||||||
|
to_append["nick"] = None
|
||||||
|
to_append["conn"] = False
|
||||||
|
to_append["limit"] = None
|
||||||
|
to_append["authed"] = None
|
||||||
|
relays.append(to_append)
|
||||||
|
return dumps({"relays": relays})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/<num>/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_relay(self, request, net, num):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
net_inst = main.network[net]
|
||||||
|
if num not in net_inst.relays:
|
||||||
|
return dumps({"success": False, "reason": "network has no such relay."})
|
||||||
|
if "status" in data:
|
||||||
|
if not type(data["status"]) == int:
|
||||||
|
return dumps({"success": False, "reason": "invalid type for enabled."})
|
||||||
|
enabled = data["status"]
|
||||||
|
if enabled:
|
||||||
|
net_inst.enable_relay(num)
|
||||||
|
else:
|
||||||
|
net_inst.disable_relay(num)
|
||||||
|
main.saveConf("network")
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/<num>/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_relay_add(self, request, net, num):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
net_inst = main.network[net]
|
||||||
|
if num in net_inst.relays:
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "network already has this relay."}
|
||||||
|
)
|
||||||
|
id, alias = net_inst.add_relay(num)
|
||||||
|
main.saveConf("network")
|
||||||
|
return dumps({"success": True, "id": id, "alias": alias})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_relay_add_next(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
net_inst = main.network[net]
|
||||||
|
id, alias = net_inst.add_relay()
|
||||||
|
main.saveConf("network")
|
||||||
|
return dumps({"success": True, "id": id, "alias": alias})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/<num>/", methods=["DELETE"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_relay_del(self, request, net, num):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
net_inst = main.network[net]
|
||||||
|
if num not in net_inst.relays:
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "network does not have this relay."}
|
||||||
|
)
|
||||||
|
net_inst.delete_relay(num)
|
||||||
|
main.saveConf("network")
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/<num>/provision/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_relay_provision(self, request, net, num):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
net_inst = main.network[net]
|
||||||
|
if num not in net_inst.relays:
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "network does not have this relay."}
|
||||||
|
)
|
||||||
|
provision.provisionRelay(num, net)
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/channels/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_channels(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
relays_inst = main.network[net].relays
|
||||||
|
channels = []
|
||||||
|
for num in relays_inst.keys():
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
net_chans = main.IRCPool[name].channels
|
||||||
|
channels_annotated = userinfo.getUserNum(net, net_chans)
|
||||||
|
for channel in net_chans:
|
||||||
|
channel_inst = {
|
||||||
|
"name": channel,
|
||||||
|
"users": channels_annotated[channel],
|
||||||
|
"num": num,
|
||||||
|
}
|
||||||
|
channels.append(channel_inst)
|
||||||
|
# channels[channel] = channels_annotated[channel]
|
||||||
|
|
||||||
|
return dumps({"channels": channels})
|
||||||
|
|
||||||
|
# API version
|
||||||
|
@app.route("/irc/network/<net>/channel/", methods=["DELETE"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_channel_part_api(self, request, net):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "channel" not in data:
|
||||||
|
return dumps({"success": False, "reason": "no channel specified."})
|
||||||
|
channel = data["channel"]
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
parted = chankeep.partSingle(net, channel)
|
||||||
|
if not parted:
|
||||||
|
dumps({"success": False, "reason": "no channels matched."})
|
||||||
|
return dumps({"success": True, "relays": parted})
|
||||||
|
|
||||||
|
# API version
|
||||||
|
@app.route("/irc/network/<net>/channel/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_channel_join_api(self, request, net):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "channel" not in data:
|
||||||
|
return dumps({"success": False, "reason": "no channel specified."})
|
||||||
|
channel = data["channel"]
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
joined = chankeep.joinSingle(net, channel)
|
||||||
|
if not joined:
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "could not allocate channel to relay."}
|
||||||
|
)
|
||||||
|
return dumps({"success": True, "relays": joined})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/channel/<channel>/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_network_channel_join(self, request, net, channel):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
joined = chankeep.joinSingle(net, channel)
|
||||||
|
if not joined:
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": "could not allocate channel to relay."}
|
||||||
|
)
|
||||||
|
return dumps({"success": True, "relays": joined})
|
||||||
|
|
||||||
|
@app.route("/aliases/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def aliases(self, request):
|
||||||
|
alias_list = []
|
||||||
|
for num, alias in main.alias.items():
|
||||||
|
alias_dup = dict(alias)
|
||||||
|
alias_dup["num"] = num
|
||||||
|
alias_list.append(alias_dup)
|
||||||
|
|
||||||
|
return dumps({"aliases": alias_list})
|
||||||
|
|
||||||
|
@app.route("/aliases/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def aliases_update(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
for alias, fields in data.items():
|
||||||
|
if not alias.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "alias not a number."})
|
||||||
|
alias = int(alias)
|
||||||
|
if alias not in main.alias.keys():
|
||||||
|
return dumps({"success": False, "reason": "alias does not exist."})
|
||||||
|
if fields:
|
||||||
|
for field in fields:
|
||||||
|
if field in main.alias[alias]:
|
||||||
|
main.alias[alias][field] = fields[field]
|
||||||
|
if "emails" in fields:
|
||||||
|
if not fields["emails"]:
|
||||||
|
main.alias[alias]["emails"] = []
|
||||||
|
main.saveConf("alias")
|
||||||
|
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/auto/<net>/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_auto_network(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if 1 in main.network[net].relays.keys():
|
||||||
|
return dumps({"success": False, "reason": f"first relay exists on {net}"})
|
||||||
|
num, alias = main.network[net].add_relay(1)
|
||||||
|
provision.provisionRelay(num, net)
|
||||||
|
main.saveConf("network")
|
||||||
|
|
||||||
|
return dumps(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"message": f"created relay {num} with alias {alias} on {net}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.route("/irc/list/<net>/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_list_network(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
first_relay = helpers.get_first_relay(net)
|
||||||
|
if not first_relay:
|
||||||
|
return dumps(
|
||||||
|
{"success": False, "reason": f"could not get first relay for {net}"}
|
||||||
|
)
|
||||||
|
first_relay.list()
|
||||||
|
return dumps(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"message": f"requested list with instance {first_relay.num} of {net}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.route("/irc/list/<net>/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def get_irc_list_info(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
listinfo = chankeep.getListInfo(net)
|
||||||
|
return dumps({"success": True, "listinfo": listinfo})
|
||||||
|
|
||||||
|
@app.route("/irc/msg/<net>/<num>/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_send_message(self, request, net, num):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
if "msg" not in data:
|
||||||
|
return dumps({"success": False, "reason": "no msg."})
|
||||||
|
if "channel" not in data:
|
||||||
|
return dumps({"success": False, "reason": "no msg."})
|
||||||
|
msg = data["msg"]
|
||||||
|
channel = data["channel"]
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
if num not in main.network[net].relays.keys():
|
||||||
|
return dumps({"success": False, "reason": f"no relay {num} on {net}"})
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name not in main.IRCPool.keys():
|
||||||
|
return dumps({"success": False, "reason": f"relay {num} not on {net}"})
|
||||||
|
# We are in a query
|
||||||
|
in_query = False
|
||||||
|
if "nick" in data:
|
||||||
|
nick = data["nick"]
|
||||||
|
if nick == channel:
|
||||||
|
in_query = True
|
||||||
|
else:
|
||||||
|
nick = None
|
||||||
|
if channel == main.IRCPool[name].nickname or nick == channel:
|
||||||
|
in_query = True
|
||||||
|
if not nick:
|
||||||
|
return dumps({"success": False, "reason": "no nick specified to query"})
|
||||||
|
else:
|
||||||
|
main.IRCPool[name].sendmsg(nick, msg, in_query=in_query)
|
||||||
|
else:
|
||||||
|
main.IRCPool[name].sendmsg(channel, msg, in_query=in_query)
|
||||||
|
return dumps(
|
||||||
|
{"success": True, "message": f"sent message to {channel} on {name}"}
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.route("/irc/nick/<net>/<num>/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def irc_get_nick(self, request, net, num):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
if num not in main.network[net].relays.keys():
|
||||||
|
return dumps({"success": False, "reason": f"no relay {num} on {net}"})
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name not in main.IRCPool.keys():
|
||||||
|
return dumps({"success": False, "reason": f"relay {num} not on {net}"})
|
||||||
|
return dumps({"nickname": main.IRCPool[name].nickname})
|
||||||
|
|
||||||
|
@app.route("/irc/reg/<net>/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def irc_get_unreg_net(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
unreg = regproc.get_unregistered_relays(net)
|
||||||
|
return dumps({"success": True, "unreg": unreg})
|
||||||
|
|
||||||
|
@app.route("/irc/reg/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def irc_get_unreg(self, request):
|
||||||
|
unreg = regproc.get_unregistered_relays()
|
||||||
|
return dumps({"success": True, "unreg": unreg})
|
||||||
|
|
||||||
|
@app.route("/irc/reg/", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
def irc_confirm_accounts(self, request):
|
||||||
|
try:
|
||||||
|
data = loads(request.content.read())
|
||||||
|
except JSONDecodeError:
|
||||||
|
return "Invalid JSON"
|
||||||
|
for item, token in data.items():
|
||||||
|
if "|" not in item:
|
||||||
|
return dumps({"success": False, "reason": f"malformed item: {item}"})
|
||||||
|
spl = item.split("|")
|
||||||
|
if not len(spl) == 2:
|
||||||
|
return dumps({"success": False, "reason": f"malformed item: {item}"})
|
||||||
|
net, num = spl
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
if token:
|
||||||
|
regproc.confirmAccount(net, num, token)
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/network/<net>/<num>/auth/", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
def irc_enable_auth(self, request, net, num):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
if not num.isdigit():
|
||||||
|
return dumps({"success": False, "reason": "invalid num: not a number."})
|
||||||
|
num = int(num)
|
||||||
|
if num not in main.network[net].relays.keys():
|
||||||
|
return dumps({"success": False, "reason": f"no relay {num} on {net}"})
|
||||||
|
regproc.enableAuthentication(net, num, jump=False, run_now=True)
|
||||||
|
return dumps({"success": True})
|
||||||
|
|
||||||
|
@app.route("/irc/sinst/<net>/", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
def irc_get_authentity(self, request, net):
|
||||||
|
if net not in main.network.keys():
|
||||||
|
return dumps({"success": False, "reason": "no such net."})
|
||||||
|
auth = regproc.selectInst(net)
|
||||||
|
if not auth:
|
||||||
|
return dumps({"success": False, "reason": "error getting results."})
|
||||||
|
return dumps({"success": True, "sinst": auth})
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from core.bot import deliverRelayCommands
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
|
||||||
|
|
||||||
class AdmallCommand:
|
class AdmallCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.admall(*args)
|
self.admall(*args)
|
||||||
|
|
||||||
def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def admall(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length > 2:
|
if length > 2:
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
|
|
@ -1,12 +1,15 @@
|
||||||
import main
|
import main
|
||||||
from yaml import dump
|
|
||||||
from modules import alias
|
from modules import alias
|
||||||
|
from yaml import dump
|
||||||
|
|
||||||
|
|
||||||
class AliasCommand:
|
class AliasCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.alias(*args)
|
self.alias(*args)
|
||||||
|
|
||||||
def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def alias(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 1:
|
if length == 1:
|
||||||
info(dump(main.alias))
|
info(dump(main.alias))
|
||||||
|
|
@ -31,7 +34,7 @@ class AliasCommand:
|
||||||
failure("Must be a number, not %s" % spl[2])
|
failure("Must be a number, not %s" % spl[2])
|
||||||
return
|
return
|
||||||
num = int(spl[2])
|
num = int(spl[2])
|
||||||
if not num in main.alias.keys():
|
if num not in main.alias.keys():
|
||||||
failure("No such alias: %i" % num)
|
failure("No such alias: %i" % num)
|
||||||
return
|
return
|
||||||
failed = False
|
failed = False
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from core.bot import deliverRelayCommands
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
|
||||||
|
|
||||||
class AllCommand:
|
class AllCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.all(*args)
|
self.all(*args)
|
||||||
|
|
||||||
def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def all(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length > 2:
|
if length > 2:
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
|
|
@ -14,7 +17,10 @@ class AllCommand:
|
||||||
net = main.network[i].relays[x]["net"]
|
net = main.network[i].relays[x]["net"]
|
||||||
alias = main.alias[x]["nick"]
|
alias = main.alias[x]["nick"]
|
||||||
commands = {spl[1]: [" ".join(spl[2:])]}
|
commands = {spl[1]: [" ".join(spl[2:])]}
|
||||||
success("Sending commands to relay %s as user %s" % (num, alias+"/"+net))
|
success(
|
||||||
|
"Sending commands to relay %s as user %s"
|
||||||
|
% (num, alias + "/" + net)
|
||||||
|
)
|
||||||
deliverRelayCommands(num, commands, user=alias + "/" + net)
|
deliverRelayCommands(num, commands, user=alias + "/" + net)
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from core.bot import deliverRelayCommands
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
|
||||||
|
|
||||||
class AllcCommand:
|
class AllcCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.allc(*args)
|
self.allc(*args)
|
||||||
|
|
||||||
def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def allc(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length > 4:
|
if length > 4:
|
||||||
targets = []
|
targets = []
|
||||||
|
|
@ -16,9 +19,12 @@ class AllcCommand:
|
||||||
targets.append((i, x))
|
targets.append((i, x))
|
||||||
elif spl[1] == "alias":
|
elif spl[1] == "alias":
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
[targets.append((i, x)) for x in main.alias.keys() if
|
[
|
||||||
main.alias[x]["nick"] == spl[2] and
|
targets.append((i, x))
|
||||||
x in main.network[i].aliases.keys()]
|
for x in main.alias.keys()
|
||||||
|
if main.alias[x]["nick"] == spl[2]
|
||||||
|
and x in main.network[i].aliases.keys()
|
||||||
|
]
|
||||||
else:
|
else:
|
||||||
incUsage("allc")
|
incUsage("allc")
|
||||||
return
|
return
|
||||||
|
|
@ -30,7 +36,10 @@ class AllcCommand:
|
||||||
num = i[1]
|
num = i[1]
|
||||||
alias = main.alias[num]["nick"]
|
alias = main.alias[num]["nick"]
|
||||||
commands = {spl[3]: [" ".join(spl[4:])]}
|
commands = {spl[3]: [" ".join(spl[4:])]}
|
||||||
success("Sending commands to relay %i as user %s" % (num, alias+"/"+net))
|
success(
|
||||||
|
"Sending commands to relay %i as user %s"
|
||||||
|
% (num, alias + "/" + net)
|
||||||
|
)
|
||||||
deliverRelayCommands(num, commands, user=alias + "/" + net)
|
deliverRelayCommands(num, commands, user=alias + "/" + net)
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
import main
|
||||||
|
|
||||||
|
|
||||||
|
class AuthcheckCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.authcheck(*args)
|
||||||
|
|
||||||
|
def authcheck(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
results = []
|
||||||
|
for i in main.IRCPool.keys():
|
||||||
|
num = main.IRCPool[i].num
|
||||||
|
net = main.IRCPool[i].net
|
||||||
|
if not main.IRCPool[i].authenticated:
|
||||||
|
results.append(
|
||||||
|
"%s - %s: %s" % (net, num, main.alias[num]["nick"])
|
||||||
|
)
|
||||||
|
info("\n".join(results))
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
results = []
|
||||||
|
for i in main.IRCPool.keys():
|
||||||
|
num = main.IRCPool[i].num
|
||||||
|
net = main.IRCPool[i].net
|
||||||
|
if not net == spl[1]:
|
||||||
|
continue
|
||||||
|
if not main.IRCPool[i].authenticated:
|
||||||
|
results.append(
|
||||||
|
"%s - %s: %s" % (net, num, main.alias[num]["nick"])
|
||||||
|
)
|
||||||
|
info("\n".join(results))
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("authcheck")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from modules import provision
|
from modules import provision
|
||||||
|
|
||||||
|
|
||||||
class AutoCommand:
|
class AutoCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.auto(*args)
|
self.auto(*args)
|
||||||
|
|
||||||
def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def auto(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 1:
|
if length == 1:
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
|
|
@ -13,9 +16,15 @@ class AutoCommand:
|
||||||
info("Skipping %s - first relay exists" % i)
|
info("Skipping %s - first relay exists" % i)
|
||||||
else:
|
else:
|
||||||
num, alias = main.network[i].add_relay(1)
|
num, alias = main.network[i].add_relay(1)
|
||||||
success("Successfully created first relay on network %s with alias %s" % (i, alias))
|
success(
|
||||||
|
"Successfully created first relay on network %s with alias %s"
|
||||||
|
% (i, alias)
|
||||||
|
)
|
||||||
provision.provisionRelay(num, i)
|
provision.provisionRelay(num, i)
|
||||||
success("Started provisioning network %s on first relay for alias %s" % (i, alias))
|
success(
|
||||||
|
"Started provisioning network %s on first relay for alias %s"
|
||||||
|
% (i, alias)
|
||||||
|
)
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
return
|
return
|
||||||
elif length == 2:
|
elif length == 2:
|
||||||
|
|
@ -26,9 +35,15 @@ class AutoCommand:
|
||||||
failure("First relay exists on %s" % spl[1])
|
failure("First relay exists on %s" % spl[1])
|
||||||
return
|
return
|
||||||
num, alias = main.network[spl[1]].add_relay(1)
|
num, alias = main.network[spl[1]].add_relay(1)
|
||||||
success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias))
|
success(
|
||||||
|
"Successfully created relay %i on network %s with alias %s"
|
||||||
|
% (num, spl[1], alias)
|
||||||
|
)
|
||||||
provision.provisionRelay(num, spl[1])
|
provision.provisionRelay(num, spl[1])
|
||||||
success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias))
|
success(
|
||||||
|
"Started provisioning network %s on relay %s for alias %s"
|
||||||
|
% (spl[1], num, alias)
|
||||||
|
)
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
import main
|
||||||
|
from yaml import dump
|
||||||
|
|
||||||
|
|
||||||
|
class BlacklistCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.blacklist(*args)
|
||||||
|
|
||||||
|
def blacklist(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
info(dump(main.blacklist))
|
||||||
|
return
|
||||||
|
elif length == 4:
|
||||||
|
if spl[1] == "add":
|
||||||
|
if spl[2] in main.blacklist.keys():
|
||||||
|
main.blacklist[spl[2]].append(spl[3])
|
||||||
|
else:
|
||||||
|
main.blacklist[spl[2]] = [spl[3]]
|
||||||
|
success("Blacklisted %s on %s" % (spl[3], spl[2]))
|
||||||
|
main.saveConf("blacklist")
|
||||||
|
return
|
||||||
|
elif spl[1] == "del":
|
||||||
|
if spl[2] in main.blacklist.keys():
|
||||||
|
if spl[3] in main.blacklist[spl[2]]:
|
||||||
|
main.blacklist[spl[2]].remove(spl[3])
|
||||||
|
if len(main.blacklist[spl[2]]) == 0:
|
||||||
|
del main.blacklist[spl[2]]
|
||||||
|
else:
|
||||||
|
failure("Not in list: %s" % spl[3])
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
failure("No such entry: %s" % spl[2])
|
||||||
|
return
|
||||||
|
success("Removed blacklist %s on %s" % (spl[3], spl[2]))
|
||||||
|
main.saveConf("blacklist")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("blacklist")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -1,11 +1,13 @@
|
||||||
import main
|
|
||||||
import modules.userinfo as userinfo
|
import modules.userinfo as userinfo
|
||||||
|
|
||||||
|
|
||||||
class ChansCommand:
|
class ChansCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.chans(*args)
|
self.chans(*args)
|
||||||
|
|
||||||
def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def chans(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if len(spl) < 2:
|
if len(spl) < 2:
|
||||||
incUsage("chans")
|
incUsage("chans")
|
||||||
|
|
@ -16,7 +18,7 @@ class ChansCommand:
|
||||||
rtrn += "Matches from: %s" % i
|
rtrn += "Matches from: %s" % i
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
for x in result[i]:
|
for x in result[i]:
|
||||||
rtrn += (x)
|
rtrn += x
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
info(rtrn)
|
info(rtrn)
|
||||||
|
|
@ -1,11 +1,13 @@
|
||||||
import main
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
from core.bot import deliverRelayCommands
|
|
||||||
|
|
||||||
class CmdCommand:
|
class CmdCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.cmd(*args)
|
self.cmd(*args)
|
||||||
|
|
||||||
def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def cmd(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length > 4:
|
if length > 4:
|
||||||
if not spl[1].isdigit():
|
if not spl[1].isdigit():
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
import main
|
||||||
|
from modules import regproc
|
||||||
|
|
||||||
|
|
||||||
|
class ConfirmCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.confirm(*args)
|
||||||
|
|
||||||
|
def confirm(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 4:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
if not spl[2].isdigit():
|
||||||
|
failure("Must be a number, not %s" % spl[2])
|
||||||
|
return
|
||||||
|
if not int(spl[2]) in main.network[spl[1]].relays.keys():
|
||||||
|
failure("No such relay on %s: %s" % (spl[1], spl[2]))
|
||||||
|
return
|
||||||
|
regproc.confirmAccount(spl[1], int(spl[2]), spl[3])
|
||||||
|
success(
|
||||||
|
"Requested confirmation on %s - %s with token %s"
|
||||||
|
% (spl[1], spl[2], spl[3])
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("confirm")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
from json import dumps
|
||||||
|
|
||||||
|
import main
|
||||||
|
from modules import chankeep
|
||||||
|
|
||||||
|
|
||||||
|
class DedupCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.dedup(*args)
|
||||||
|
|
||||||
|
def dedup(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
dupes = chankeep.getDuplicateChannels()
|
||||||
|
chankeep.partChannels(dupes)
|
||||||
|
info(dumps(dupes))
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if spl[1] not in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
dupes = chankeep.getDuplicateChannels(spl[1])
|
||||||
|
chankeep.partChannels(dupes)
|
||||||
|
info(dumps(dupes))
|
||||||
|
return
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from core.bot import deliverRelayCommands
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
|
||||||
|
|
||||||
class DisableCommand:
|
class DisableCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.disable(*args)
|
self.disable(*args)
|
||||||
|
|
||||||
def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def disable(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 3:
|
if length == 3:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
|
|
@ -18,13 +21,13 @@ class DisableCommand:
|
||||||
name = spl[1] + spl[2]
|
name = spl[1] + spl[2]
|
||||||
if not spl[1] in main.IRCPool.keys():
|
if not spl[1] in main.IRCPool.keys():
|
||||||
info("Note - instance not running, proceeding anyway")
|
info("Note - instance not running, proceeding anyway")
|
||||||
if not relayNum in main.network[spl[1]].relays.keys():
|
if relayNum not in main.network[spl[1]].relays.keys():
|
||||||
failure("No such relay: %s in network %s" % (spl[2], spl[1]))
|
failure("No such relay: %s in network %s" % (spl[2], spl[1]))
|
||||||
return
|
return
|
||||||
main.network[spl[1]].relays[relayNum]["enabled"] = False
|
main.network[spl[1]].relays[relayNum]["enabled"] = False
|
||||||
user = main.alias[relayNum]["nick"]
|
user = main.alias[relayNum]["nick"]
|
||||||
network = spl[1]
|
network = spl[1]
|
||||||
relay = main.network[spl[1]].relays[relayNum]
|
# relay = main.network[spl[1]].relays[relayNum]
|
||||||
commands = {"status": ["Disconnect"]}
|
commands = {"status": ["Disconnect"]}
|
||||||
deliverRelayCommands(relayNum, commands, user=user + "/" + network)
|
deliverRelayCommands(relayNum, commands, user=user + "/" + network)
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
|
|
@ -1,11 +1,15 @@
|
||||||
|
from subprocess import PIPE, run
|
||||||
|
|
||||||
import main
|
import main
|
||||||
from subprocess import run, PIPE
|
|
||||||
|
|
||||||
class DistCommand:
|
class DistCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.dist(*args)
|
self.dist(*args)
|
||||||
|
|
||||||
def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def dist(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if main.config["Dist"]["Enabled"]:
|
if main.config["Dist"]["Enabled"]:
|
||||||
rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE)
|
rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE)
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
import main
|
||||||
|
from yaml import dump
|
||||||
|
|
||||||
|
|
||||||
|
class EmailCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.email(*args)
|
||||||
|
|
||||||
|
def email(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 4:
|
||||||
|
if spl[1] == "add":
|
||||||
|
if spl[2].isdigit():
|
||||||
|
num = int(spl[2])
|
||||||
|
if num not in main.alias.keys():
|
||||||
|
failure("No such alias: %i" % num)
|
||||||
|
return
|
||||||
|
if not spl[3] in main.alias[num]["emails"]:
|
||||||
|
main.alias[num]["emails"].append(spl[3])
|
||||||
|
main.saveConf("alias")
|
||||||
|
success(
|
||||||
|
"Successfully added email %s to alias %i"
|
||||||
|
% (spl[3], num)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
failure(
|
||||||
|
"Email already exists in alias %i: %s" % (num, spl[3])
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
failure("Must be a number, not %s" % spl[2])
|
||||||
|
if spl[2] == "domain":
|
||||||
|
domain = spl[3]
|
||||||
|
if "@" in domain:
|
||||||
|
failure("Not a domain: %s" % domain)
|
||||||
|
return
|
||||||
|
if domain not in main.irc["_"]["domains"]:
|
||||||
|
main.irc["_"]["domains"].append(domain)
|
||||||
|
success(
|
||||||
|
"Successfully added domain %s to default config"
|
||||||
|
% domain
|
||||||
|
)
|
||||||
|
main.saveConf("irc")
|
||||||
|
else:
|
||||||
|
failure(
|
||||||
|
"Domain already exists in default config: %s"
|
||||||
|
% domain
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
elif spl[1] == "del":
|
||||||
|
if not spl[2].isdigit():
|
||||||
|
# failure("Must be a number, not %s" % spl[2])
|
||||||
|
if spl[2] == "domain":
|
||||||
|
domain = spl[3]
|
||||||
|
|
||||||
|
if domain in main.irc["_"]["domains"]:
|
||||||
|
main.irc["_"]["domains"].remove(domain)
|
||||||
|
success(
|
||||||
|
"Successfully removed domain %s to default config"
|
||||||
|
% domain
|
||||||
|
)
|
||||||
|
main.saveConf("irc")
|
||||||
|
else:
|
||||||
|
failure(
|
||||||
|
"Domain does not exist in default config: %s"
|
||||||
|
% domain
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
elif spl[1] == "del":
|
||||||
|
if num not in main.alias.keys():
|
||||||
|
failure("No such alias: %i" % num)
|
||||||
|
return
|
||||||
|
if spl[3] in main.alias[num]["emails"]:
|
||||||
|
main.alias[num]["emails"].remove(spl[3])
|
||||||
|
main.saveConf("alias")
|
||||||
|
success(
|
||||||
|
"Successfully removed email %s from alias %i"
|
||||||
|
% (spl[3], num)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
failure(
|
||||||
|
"Email does not exist in alias %i: %s" % (spl[3], num)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if spl[1] == "list":
|
||||||
|
info(dump(main.alias))
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("email")
|
||||||
|
return
|
||||||
|
elif length == 3:
|
||||||
|
if spl[1] == "list":
|
||||||
|
if spl[2] == "domain":
|
||||||
|
filtered = {
|
||||||
|
f"{k}:{k2}": v2
|
||||||
|
for k, v in main.irc.items()
|
||||||
|
for k2, v2 in v.items()
|
||||||
|
if k2 == "domains"
|
||||||
|
}
|
||||||
|
info(dump(filtered))
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("email")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("email")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("email")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -1,11 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from core.bot import deliverRelayCommands
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
|
||||||
|
|
||||||
class EnableCommand:
|
class EnableCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.enable(*args)
|
self.enable(*args)
|
||||||
|
|
||||||
def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def enable(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 3:
|
if length == 3:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
import main
|
|
||||||
|
|
||||||
class ExecCommand:
|
class ExecCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.exec(*args)
|
self.exec(*args)
|
||||||
|
|
||||||
def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def exec(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length > 1:
|
if length > 1:
|
||||||
try:
|
try:
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
import main
|
||||||
|
from utils.get import getRelay
|
||||||
|
|
||||||
|
|
||||||
|
class GetstrCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.getstr(*args)
|
||||||
|
|
||||||
|
def getstr(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 3:
|
||||||
|
net = spl[1]
|
||||||
|
num = spl[2]
|
||||||
|
if net not in main.network.keys():
|
||||||
|
failure("Network does not exist: %s" % net)
|
||||||
|
return
|
||||||
|
if not num.isdigit():
|
||||||
|
failure("Must be a number, not %s" % num)
|
||||||
|
return
|
||||||
|
num = int(num)
|
||||||
|
alias = main.alias[num]["nick"].lower()
|
||||||
|
host, port = getRelay(num)
|
||||||
|
password = main.config["Relay"]["Password"]
|
||||||
|
connstr = f"/connect -ssl {host} {port}"
|
||||||
|
authstr = f"/quote PASS {alias}/{net}:{password}"
|
||||||
|
obj.send(connstr)
|
||||||
|
obj.send(authstr)
|
||||||
|
|
||||||
|
else:
|
||||||
|
incUsage("getstr")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class HelpCommand:
|
class HelpCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.help(*args)
|
self.help(*args)
|
||||||
|
|
||||||
def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def help(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
helpMap = []
|
helpMap = []
|
||||||
for i in main.help.keys():
|
for i in main.help.keys():
|
||||||
|
|
@ -1,12 +1,23 @@
|
||||||
import main
|
import main
|
||||||
|
import modules.chankeep
|
||||||
|
|
||||||
|
|
||||||
class JoinCommand:
|
class JoinCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.join(*args)
|
self.join(*args)
|
||||||
|
|
||||||
def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def join(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 4:
|
if length == 3:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("Network does not exist: %s" % spl[1])
|
||||||
|
return
|
||||||
|
modules.chankeep.joinSingle(spl[1], spl[2])
|
||||||
|
success("Joined %s" % spl[2])
|
||||||
|
|
||||||
|
elif length == 4:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
failure("Network does not exist: %s" % spl[1])
|
failure("Network does not exist: %s" % spl[1])
|
||||||
return
|
return
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
import main
|
||||||
|
from modules import helpers
|
||||||
|
|
||||||
|
|
||||||
|
class ListCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.list(*args)
|
||||||
|
|
||||||
|
def list(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
for i in main.network.keys():
|
||||||
|
first_relay = helpers.get_first_relay(i)
|
||||||
|
####
|
||||||
|
if not first_relay:
|
||||||
|
info("Network has no first instance: %s" % i)
|
||||||
|
continue
|
||||||
|
first_relay.list()
|
||||||
|
success(f"Requested list with instance {first_relay.num} of {i}")
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
first_relay = helpers.get_first_relay(spl[1])
|
||||||
|
if not first_relay:
|
||||||
|
failure("Could not get first instance")
|
||||||
|
return
|
||||||
|
first_relay.list()
|
||||||
|
success(f"Requested list with instance {first_relay.num} of {spl[1]}")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("list")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class LoadCommand:
|
class LoadCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.load(*args)
|
self.load(*args)
|
||||||
|
|
||||||
def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def load(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 2:
|
if length == 2:
|
||||||
if spl[1] in main.filemap.keys():
|
if spl[1] in main.filemap.keys():
|
||||||
|
|
@ -1,15 +1,17 @@
|
||||||
import main
|
|
||||||
from utils.loaders.single_loader import loadSingle
|
from utils.loaders.single_loader import loadSingle
|
||||||
|
|
||||||
|
|
||||||
class LoadmodCommand:
|
class LoadmodCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.loadmod(*args)
|
self.loadmod(*args)
|
||||||
|
|
||||||
def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def loadmod(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 2:
|
if length == 2:
|
||||||
rtrn = loadSingle(spl[1])
|
rtrn = loadSingle(spl[1])
|
||||||
if rtrn == True:
|
if rtrn is True:
|
||||||
success("Loaded module: %s" % spl[1])
|
success("Loaded module: %s" % spl[1])
|
||||||
return
|
return
|
||||||
elif rtrn == "RELOAD":
|
elif rtrn == "RELOAD":
|
||||||
|
|
@ -1,14 +1,12 @@
|
||||||
import main
|
|
||||||
|
|
||||||
class LogoutCommand:
|
class LogoutCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.logout(*args)
|
self.logout(*args)
|
||||||
|
|
||||||
def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def logout(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
obj.authed = False
|
obj.authed = False
|
||||||
if obj.addr in main.MonitorPool:
|
|
||||||
main.MonitorPool.remove(obj.addr)
|
|
||||||
success("Logged out")
|
success("Logged out")
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -1,12 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from yaml import dump
|
|
||||||
|
|
||||||
class ModCommand:
|
class ModCommand:
|
||||||
# This could be greatly improved, but not really important right now
|
# This could be greatly improved, but not really important right now
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.mod(*args)
|
self.mod(*args)
|
||||||
|
|
||||||
def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def mod(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 4:
|
if length == 4:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
|
|
@ -15,12 +17,14 @@ class ModCommand:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
setattr(main.network[spl[1]], spl[2], spl[3])
|
setattr(main.network[spl[1]], spl[2], spl[3])
|
||||||
except e:
|
except Exception as e:
|
||||||
failure("Something went wrong.")
|
failure(f"Something went wrong: {e}")
|
||||||
return
|
return
|
||||||
|
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]))
|
success(
|
||||||
|
"Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])
|
||||||
|
)
|
||||||
return
|
return
|
||||||
# Find a better way to do this
|
# Find a better way to do this
|
||||||
# elif length == 6:
|
# elif length == 6:
|
||||||
|
|
@ -1,17 +1,20 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class MsgCommand:
|
class MsgCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.msg(*args)
|
self.msg(*args)
|
||||||
|
|
||||||
def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def msg(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length >= 5:
|
if length >= 5:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
failure("Network does not exist: %s" % spl[1])
|
failure("Network does not exist: %s" % spl[1])
|
||||||
return
|
return
|
||||||
if not int(spl[2]) in main.network[spl[1]].relays.keys():
|
if not int(spl[2]) in main.network[spl[1]].relays.keys():
|
||||||
failure("Relay % does not exist on network %" % (spl[2], spl[1]))
|
failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
|
||||||
return
|
return
|
||||||
if not spl[1] + spl[2] in main.IRCPool.keys():
|
if not spl[1] + spl[2] in main.IRCPool.keys():
|
||||||
failure("Name has no instance: %s" % spl[1])
|
failure("Name has no instance: %s" % spl[1])
|
||||||
|
|
@ -19,7 +22,10 @@ class MsgCommand:
|
||||||
if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels:
|
if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels:
|
||||||
info("Bot not on channel: %s" % spl[3])
|
info("Bot not on channel: %s" % spl[3])
|
||||||
main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:]))
|
main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:]))
|
||||||
success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1]))
|
success(
|
||||||
|
"Sent %s to %s on relay %s on network %s"
|
||||||
|
% (" ".join(spl[4:]), spl[3], spl[2], spl[1])
|
||||||
|
)
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
incUsage("msg")
|
incUsage("msg")
|
||||||
|
|
@ -1,13 +1,17 @@
|
||||||
import main
|
|
||||||
from yaml import dump
|
|
||||||
from modules.network import Network
|
|
||||||
from string import digits
|
from string import digits
|
||||||
|
|
||||||
|
import main
|
||||||
|
from modules.network import Network
|
||||||
|
from yaml import dump
|
||||||
|
|
||||||
|
|
||||||
class NetworkCommand:
|
class NetworkCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.network(*args)
|
self.network(*args)
|
||||||
|
|
||||||
def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def network(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 7:
|
if length == 7:
|
||||||
if spl[1] == "add":
|
if spl[1] == "add":
|
||||||
|
|
@ -27,7 +31,9 @@ class NetworkCommand:
|
||||||
failure("Auth must be sasl, ns or none, not %s" % spl[5])
|
failure("Auth must be sasl, ns or none, not %s" % spl[5])
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower())
|
main.network[spl[2]] = Network(
|
||||||
|
spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()
|
||||||
|
)
|
||||||
success("Successfully created network: %s" % spl[2])
|
success("Successfully created network: %s" % spl[2])
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
return
|
return
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class PartCommand:
|
class PartCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.part(*args)
|
self.part(*args)
|
||||||
|
|
||||||
def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def part(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 4:
|
if length == 4:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class PassCommand:
|
class PassCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.password(*args)
|
self.password(*args)
|
||||||
|
|
||||||
def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def password(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
info("You are already authenticated")
|
info("You are already authenticated")
|
||||||
return
|
return
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
import main
|
||||||
|
|
||||||
|
|
||||||
|
class PendingCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.pending(*args)
|
||||||
|
|
||||||
|
def pending(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
results = []
|
||||||
|
for i in main.network.keys():
|
||||||
|
for x in main.network[i].relays.keys():
|
||||||
|
if not main.network[i].relays[x]["registered"]:
|
||||||
|
results.append(
|
||||||
|
"%s: confirm %s %s [code]"
|
||||||
|
% (main.alias[x]["nick"], i, x)
|
||||||
|
)
|
||||||
|
info("\n".join(results))
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
results = []
|
||||||
|
for x in main.network[spl[1]].relays.keys():
|
||||||
|
if not main.network[spl[1]].relays[x]["registered"]:
|
||||||
|
results.append(
|
||||||
|
"%s: confirm %s %s [code]"
|
||||||
|
% (main.alias[x]["nick"], spl[1], x)
|
||||||
|
)
|
||||||
|
info("\n".join(results))
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("pending")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
import main
|
||||||
|
|
||||||
|
|
||||||
|
class RecheckauthCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.recheckauth(*args)
|
||||||
|
|
||||||
|
def recheckauth(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
for i in main.IRCPool.keys():
|
||||||
|
net = main.IRCPool[i].net
|
||||||
|
main.IRCPool[i].authenticated = False
|
||||||
|
main.IRCPool[i].regPing()
|
||||||
|
success("Successfully reset authentication status")
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
for i in main.IRCPool.keys():
|
||||||
|
# num = main.IRCPool[i].num
|
||||||
|
net = main.IRCPool[i].net
|
||||||
|
if not net == spl[1]:
|
||||||
|
continue
|
||||||
|
main.IRCPool[i].authenticated = False
|
||||||
|
main.IRCPool[i].regPing()
|
||||||
|
success("Successfully reset authentication status on %s" % spl[1])
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("recheckauth")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
import main
|
||||||
|
from modules import regproc
|
||||||
|
|
||||||
|
|
||||||
|
class RegCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.reg(*args)
|
||||||
|
|
||||||
|
def reg(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
|
if authed:
|
||||||
|
if length == 2:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
for i in main.network[spl[1]].relays.keys():
|
||||||
|
regproc.registerAccount(spl[1], i)
|
||||||
|
success("Requested registration for all relays on %s" % spl[1])
|
||||||
|
return
|
||||||
|
elif length == 3:
|
||||||
|
if not spl[1] in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
if not spl[2].isdigit():
|
||||||
|
failure("Must be a number, not %s" % spl[2])
|
||||||
|
return
|
||||||
|
if not int(spl[2]) in main.network[spl[1]].relays.keys():
|
||||||
|
failure("No such relay on %s: %s" % (spl[2], spl[1]))
|
||||||
|
return
|
||||||
|
regproc.registerAccount(spl[1], int(spl[2]))
|
||||||
|
success("Requested registration on %s - %s" % (spl[1], spl[2]))
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage("reg")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
incUsage(None)
|
||||||
|
|
@ -1,17 +1,23 @@
|
||||||
import main
|
import main
|
||||||
from yaml import dump
|
from yaml import dump
|
||||||
|
|
||||||
|
|
||||||
class RelayCommand:
|
class RelayCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.relay(*args)
|
self.relay(*args)
|
||||||
|
|
||||||
def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def relay(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 3:
|
if length == 3:
|
||||||
if spl[1] == "add":
|
if spl[1] == "add":
|
||||||
if spl[2] in main.network.keys():
|
if spl[2] in main.network.keys():
|
||||||
id, alias = main.network[spl[2]].add_relay()
|
id, alias = main.network[spl[2]].add_relay()
|
||||||
success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
|
success(
|
||||||
|
"Successfully created relay %s on network %s with alias %s"
|
||||||
|
% (str(id), spl[2], alias)
|
||||||
|
)
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -34,7 +40,10 @@ class RelayCommand:
|
||||||
failure("Must be a number, not %s" % spl[3])
|
failure("Must be a number, not %s" % spl[3])
|
||||||
return
|
return
|
||||||
id, alias = main.network[spl[2]].add_relay(int(spl[3]))
|
id, alias = main.network[spl[2]].add_relay(int(spl[3]))
|
||||||
success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias))
|
success(
|
||||||
|
"Successfully created relay %s on network %s with alias %s"
|
||||||
|
% (str(id), spl[2], alias)
|
||||||
|
)
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -51,7 +60,9 @@ class RelayCommand:
|
||||||
failure("No such relay: %s on network %s" % (spl[3], spl[2]))
|
failure("No such relay: %s on network %s" % (spl[3], spl[2]))
|
||||||
return
|
return
|
||||||
main.network[spl[2]].delete_relay(int(spl[3]))
|
main.network[spl[2]].delete_relay(int(spl[3]))
|
||||||
success("Successfully deleted relay %s on network %s" % (spl[3], spl[2]))
|
success(
|
||||||
|
"Successfully deleted relay %s on network %s" % (spl[3], spl[2])
|
||||||
|
)
|
||||||
main.saveConf("network")
|
main.saveConf("network")
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class SaveCommand:
|
class SaveCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.save(*args)
|
self.save(*args)
|
||||||
|
|
||||||
def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def save(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 2:
|
if length == 2:
|
||||||
if spl[1] in main.filemap.keys():
|
if spl[1] in main.filemap.keys():
|
||||||
|
|
@ -1,13 +1,17 @@
|
||||||
|
from string import digits
|
||||||
|
|
||||||
import main
|
import main
|
||||||
import modules.counters as count
|
import modules.counters as count
|
||||||
import modules.userinfo as userinfo
|
import modules.userinfo as userinfo
|
||||||
from string import digits
|
|
||||||
|
|
||||||
class StatsCommand:
|
class StatsCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.stats(*args)
|
self.stats(*args)
|
||||||
|
|
||||||
def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def stats(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 1:
|
if length == 1:
|
||||||
stats = []
|
stats = []
|
||||||
|
|
@ -25,7 +29,7 @@ class StatsCommand:
|
||||||
stats.append("User records: %s" % numWhoEntries)
|
stats.append("User records: %s" % numWhoEntries)
|
||||||
stats.append("Events/min: %s" % main.lastMinuteSample)
|
stats.append("Events/min: %s" % main.lastMinuteSample)
|
||||||
counterEvents = count.getEvents()
|
counterEvents = count.getEvents()
|
||||||
if counterEvents == None:
|
if counterEvents is None:
|
||||||
stats.append("No counters records")
|
stats.append("No counters records")
|
||||||
else:
|
else:
|
||||||
stats.append("Counters:")
|
stats.append("Counters:")
|
||||||
|
|
@ -42,7 +46,7 @@ class StatsCommand:
|
||||||
numNodes = 0
|
numNodes = 0
|
||||||
|
|
||||||
for i in main.IRCPool.keys():
|
for i in main.IRCPool.keys():
|
||||||
if "".join([x for x in i if not x in digits]) == spl[1]:
|
if "".join([x for x in i if x not in digits]) == spl[1]:
|
||||||
numChannels += len(main.IRCPool[i].channels)
|
numChannels += len(main.IRCPool[i].channels)
|
||||||
found = True
|
found = True
|
||||||
numNodes += 1
|
numNodes += 1
|
||||||
|
|
@ -53,7 +57,7 @@ class StatsCommand:
|
||||||
stats.append("User records: %s" % numWhoEntries)
|
stats.append("User records: %s" % numWhoEntries)
|
||||||
stats.append("Endpoints: %s" % numNodes)
|
stats.append("Endpoints: %s" % numNodes)
|
||||||
counterEvents = count.getEvents(spl[1])
|
counterEvents = count.getEvents(spl[1])
|
||||||
if counterEvents == None:
|
if counterEvents is None:
|
||||||
stats.append("No counters records")
|
stats.append("No counters records")
|
||||||
else:
|
else:
|
||||||
stats.append("Counters:")
|
stats.append("Counters:")
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
import main
|
||||||
|
|
||||||
|
|
||||||
class SwhoCommand:
|
class SwhoCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.swho(*args)
|
self.swho(*args)
|
||||||
|
|
||||||
def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def swho(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 2:
|
if length == 2:
|
||||||
if not spl[1] in main.network.keys():
|
if not spl[1] in main.network.keys():
|
||||||
|
|
@ -1,12 +1,16 @@
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
import main
|
import main
|
||||||
from yaml import dump
|
from yaml import dump
|
||||||
from uuid import uuid4
|
|
||||||
|
|
||||||
class TokenCommand:
|
class TokenCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.token(*args)
|
self.token(*args)
|
||||||
|
|
||||||
def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def token(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 2:
|
if length == 2:
|
||||||
if spl[1] == "list":
|
if spl[1] == "list":
|
||||||
|
|
@ -31,8 +35,9 @@ class TokenCommand:
|
||||||
elif length == 4:
|
elif length == 4:
|
||||||
if spl[1] == "add":
|
if spl[1] == "add":
|
||||||
if not spl[2] in main.tokens.keys():
|
if not spl[2] in main.tokens.keys():
|
||||||
if spl[3] in ["relay"]: # more to come!
|
if spl[3] in ["relay", "api"]: # more to come!
|
||||||
main.tokens[spl[2]] = {"hello": str(uuid4()),
|
main.tokens[spl[2]] = {
|
||||||
|
"hello": str(uuid4()),
|
||||||
"usage": spl[3],
|
"usage": spl[3],
|
||||||
"counter": str(uuid4()),
|
"counter": str(uuid4()),
|
||||||
}
|
}
|
||||||
|
|
@ -1,11 +1,13 @@
|
||||||
import main
|
|
||||||
import modules.userinfo as userinfo
|
import modules.userinfo as userinfo
|
||||||
|
|
||||||
|
|
||||||
class UsersCommand:
|
class UsersCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.users(*args)
|
self.users(*args)
|
||||||
|
|
||||||
def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def users(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if len(spl) < 2:
|
if len(spl) < 2:
|
||||||
incUsage("users")
|
incUsage("users")
|
||||||
|
|
@ -16,7 +18,7 @@ class UsersCommand:
|
||||||
rtrn += "Matches from: %s" % i
|
rtrn += "Matches from: %s" % i
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
for x in result[i]:
|
for x in result[i]:
|
||||||
rtrn += (x)
|
rtrn += x
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
info(rtrn)
|
info(rtrn)
|
||||||
|
|
@ -1,11 +1,13 @@
|
||||||
import main
|
|
||||||
import modules.userinfo as userinfo
|
import modules.userinfo as userinfo
|
||||||
|
|
||||||
|
|
||||||
class WhoCommand:
|
class WhoCommand:
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
self.who(*args)
|
self.who(*args)
|
||||||
|
|
||||||
def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
def who(
|
||||||
|
self, addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
):
|
||||||
if authed:
|
if authed:
|
||||||
if length == 2:
|
if length == 2:
|
||||||
result = userinfo.getWho(spl[1])
|
result = userinfo.getWho(spl[1])
|
||||||
|
|
@ -14,7 +16,7 @@ class WhoCommand:
|
||||||
rtrn += "Matches from: %s" % i
|
rtrn += "Matches from: %s" % i
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
for x in result[i]:
|
for x in result[i]:
|
||||||
rtrn += (x)
|
rtrn += x
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
rtrn += "\n"
|
rtrn += "\n"
|
||||||
info(rtrn)
|
info(rtrn)
|
||||||
|
|
@ -10,19 +10,34 @@
|
||||||
"Address": "127.0.0.1",
|
"Address": "127.0.0.1",
|
||||||
"UseSSL": true
|
"UseSSL": true
|
||||||
},
|
},
|
||||||
|
"API": {
|
||||||
|
"Enabled": true,
|
||||||
|
"Port": 13869,
|
||||||
|
"Address": "127.0.0.1"
|
||||||
|
},
|
||||||
"Key": "key.pem",
|
"Key": "key.pem",
|
||||||
"Certificate": "cert.pem",
|
"Certificate": "cert.pem",
|
||||||
"RedisSocket": "/tmp/redis.sock",
|
"RedisSocket": "/var/run/redis/redis.sock",
|
||||||
"UsePassword": true,
|
"RedisDBEphemeral": 1,
|
||||||
|
"RedisDBPersistent": 0,
|
||||||
|
"UsePassword": false,
|
||||||
"ConnectOnCreate": false,
|
"ConnectOnCreate": false,
|
||||||
|
"AutoReg": false,
|
||||||
"Debug": false,
|
"Debug": false,
|
||||||
|
"Trace": false,
|
||||||
"Relay": {
|
"Relay": {
|
||||||
"Host": "127.0.0.1",
|
"Host": "127.0.0.1",
|
||||||
"Port": "201x",
|
"Port": "2001",
|
||||||
"User": "sir",
|
"User": "x",
|
||||||
"Password": "sir"
|
"Password": "x"
|
||||||
|
},
|
||||||
|
"Ingest": {
|
||||||
|
"Key": "queue.irc",
|
||||||
|
"Enabled": true
|
||||||
},
|
},
|
||||||
"ChanKeep": {
|
"ChanKeep": {
|
||||||
|
"Enabled": false,
|
||||||
|
"Provision": false,
|
||||||
"MaxRelay": 30,
|
"MaxRelay": 30,
|
||||||
"SigSwitch": 20
|
"SigSwitch": 20
|
||||||
},
|
},
|
||||||
|
|
@ -32,10 +47,10 @@
|
||||||
"File": "conf/dist.sh"
|
"File": "conf/dist.sh"
|
||||||
},
|
},
|
||||||
"Toggles": {
|
"Toggles": {
|
||||||
"Who": false,
|
"Who": true,
|
||||||
"CycleChans": true
|
"CycleChans": true
|
||||||
},
|
},
|
||||||
"Password": "s",
|
"Password": "x",
|
||||||
"Tweaks": {
|
"Tweaks": {
|
||||||
"MaxHash": 10,
|
"MaxHash": 10,
|
||||||
"DedupPrecision": 9,
|
"DedupPrecision": 9,
|
||||||
|
|
@ -43,6 +58,7 @@
|
||||||
"Prefix": "*"
|
"Prefix": "*"
|
||||||
},
|
},
|
||||||
"Delays": {
|
"Delays": {
|
||||||
|
"WhoDelay": 3600,
|
||||||
"WhoLoop": 600,
|
"WhoLoop": 600,
|
||||||
"WhoRange": 1800,
|
"WhoRange": 1800,
|
||||||
"Timeout": 30,
|
"Timeout": 30,
|
||||||
|
|
@ -7,14 +7,12 @@
|
||||||
"part": "part <name> <num> <channel>",
|
"part": "part <name> <num> <channel>",
|
||||||
"enable": "enable <name> <num>",
|
"enable": "enable <name> <num>",
|
||||||
"disable": "disable <name> <num>",
|
"disable": "disable <name> <num>",
|
||||||
"list": "list",
|
|
||||||
"stats": "stats [<name>]",
|
"stats": "stats [<name>]",
|
||||||
"save": "save <(file)|list|all>",
|
"save": "save <(file)|list|all>",
|
||||||
"load": "load <(file)|list|all>",
|
"load": "load <(file)|list|all>",
|
||||||
"dist": "dist",
|
"dist": "dist",
|
||||||
"loadmod": "loadmod <module>",
|
"loadmod": "loadmod <module>",
|
||||||
"msg": "msg <name> <target> <message...>",
|
"msg": "msg <network> <num> <target> <message...>",
|
||||||
"mon": "mon -h",
|
|
||||||
"chans": "chans <nick> [<nick> ...]",
|
"chans": "chans <nick> [<nick> ...]",
|
||||||
"users": "users <channel> [<channel> ...]",
|
"users": "users <channel> [<channel> ...]",
|
||||||
"relay": "relay <add|del|list> [<network>] [<num>]",
|
"relay": "relay <add|del|list> [<network>] [<num>]",
|
||||||
|
|
@ -22,11 +20,19 @@
|
||||||
"alias": "alias [<add|del>] [<num>]",
|
"alias": "alias [<add|del>] [<num>]",
|
||||||
"auto": "auto [<network>]",
|
"auto": "auto [<network>]",
|
||||||
"cmd": "cmd <relay> <user> <entity> <text ...>",
|
"cmd": "cmd <relay> <user> <entity> <text ...>",
|
||||||
"token": "token <add|del|list> [<key>] [<relay>]",
|
"token": "token <add|del|list> [<key>] [relay|api]",
|
||||||
"all": "all <entity> <text ...>",
|
"all": "all <entity> <text ...>",
|
||||||
"allc": "allc <network|alias> <(network)|(alias)> <entity> <text ...>",
|
"allc": "allc <network|alias> <(network)|(alias)> <entity> <text ...>",
|
||||||
"admall": "admall <entity> <text ...>",
|
"admall": "admall <entity> <text ...>",
|
||||||
"swho": "swho <network> [<channel>]",
|
"swho": "swho <network> [<channel>]",
|
||||||
"list": "list <network>",
|
"list": "list [<network>]",
|
||||||
"exec": "exec <expr ...>"
|
"exec": "exec <expr ...>",
|
||||||
|
"reg": "reg <network> [<num>]",
|
||||||
|
"confirm": "confirm <network> <num> <token>",
|
||||||
|
"pending": "pending [<network>]",
|
||||||
|
"authcheck": "authcheck [<network>]",
|
||||||
|
"recheckauth": "recheckauth [<network>]",
|
||||||
|
"blacklist": "blacklist <network> <channel>",
|
||||||
|
"email": "email <add|del|list> [(domain)|<num>] [<email>]",
|
||||||
|
"getstr": "getstr <net> <num>"
|
||||||
}
|
}
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
{
|
||||||
|
"_": {
|
||||||
|
"register": true,
|
||||||
|
"entity": "NickServ",
|
||||||
|
"domains": [],
|
||||||
|
"registermsg": "REGISTER {password} {email}",
|
||||||
|
"identifymsg": "IDENTIFY {password}",
|
||||||
|
"confirm": "CONFIRM {token}",
|
||||||
|
"check": true,
|
||||||
|
"ping": true,
|
||||||
|
"negative": true,
|
||||||
|
"pingmsg": "STATUS",
|
||||||
|
"negativemsg": "INFO {nickname}",
|
||||||
|
"checktype": "mode",
|
||||||
|
"checkmode": "R",
|
||||||
|
"checkmsg": "Password accepted - you are now recognized.",
|
||||||
|
"checkmsg2": "You are logged in as",
|
||||||
|
"checknegativemsg": "has \u0002NOT COMPLETED\u0002 registration verification",
|
||||||
|
"checkendnegative": "End of Info"
|
||||||
|
},
|
||||||
|
"freenode": {
|
||||||
|
"confirm": "VERIFY REGISTER {nickname} {token}"
|
||||||
|
},
|
||||||
|
"libera": {
|
||||||
|
"confirm": "VERIFY REGISTER {nickname} {token}"
|
||||||
|
},
|
||||||
|
"cyberia": {
|
||||||
|
"setmode": "I"
|
||||||
|
},
|
||||||
|
"ircnow": {
|
||||||
|
"negative": false,
|
||||||
|
"ping": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,946 @@
|
||||||
|
import sys
|
||||||
|
from copy import deepcopy
|
||||||
|
from datetime import datetime
|
||||||
|
from random import randint
|
||||||
|
|
||||||
|
import main
|
||||||
|
from core.relay import sendRelayNotification
|
||||||
|
from modules import chankeep, counters, helpers, monitor, regproc, userinfo
|
||||||
|
from twisted.internet import reactor
|
||||||
|
from twisted.internet.defer import Deferred
|
||||||
|
from twisted.internet.protocol import ReconnectingClientFactory
|
||||||
|
from twisted.internet.task import LoopingCall
|
||||||
|
from twisted.words.protocols.irc import (
|
||||||
|
IRCBadMessage,
|
||||||
|
IRCClient,
|
||||||
|
lowDequote,
|
||||||
|
numeric_to_symbolic,
|
||||||
|
)
|
||||||
|
from utils.dedup import dedup
|
||||||
|
from utils.logging.debug import debug, trace
|
||||||
|
from utils.logging.log import error, log, warn
|
||||||
|
from utils.logging.send import sendAll
|
||||||
|
from utils.parsing import parsen
|
||||||
|
|
||||||
|
|
||||||
|
# Copied from the Twisted source so we can fix a bug
|
||||||
|
def parsemsg(s):
|
||||||
|
"""
|
||||||
|
Breaks a message from an IRC server into its prefix, command, and
|
||||||
|
arguments.
|
||||||
|
@param s: The message to break.
|
||||||
|
@type s: L{bytes}
|
||||||
|
@return: A tuple of (prefix, command, args).
|
||||||
|
@rtype: L{tuple}
|
||||||
|
"""
|
||||||
|
prefix = ""
|
||||||
|
trailing = []
|
||||||
|
if not s:
|
||||||
|
raise IRCBadMessage("Empty line.")
|
||||||
|
if s[0:1] == ":":
|
||||||
|
prefix, s = s[1:].split(" ", 1)
|
||||||
|
if s.find(" :") != -1:
|
||||||
|
s, trailing = s.split(" :", 1)
|
||||||
|
args = s.split(" ") # Twisted bug fixed by adding an argument to split()
|
||||||
|
args.append(trailing)
|
||||||
|
else:
|
||||||
|
args = s.split(" ") # And again
|
||||||
|
command = args.pop(0)
|
||||||
|
return prefix, command, args
|
||||||
|
|
||||||
|
|
||||||
|
class IRCBot(IRCClient):
|
||||||
|
def __init__(self, net, num):
|
||||||
|
self.isconnected = False
|
||||||
|
self.channels = []
|
||||||
|
self.net = net
|
||||||
|
self.authenticated = not regproc.needToAuth(self.net)
|
||||||
|
self.num = num
|
||||||
|
self.buffer = ""
|
||||||
|
self.name = net + str(num)
|
||||||
|
alias = main.alias[num]
|
||||||
|
relay = main.network[self.net].relays[num]
|
||||||
|
self.netinst = main.network[self.net]
|
||||||
|
self.nickname = alias["nick"]
|
||||||
|
self.realname = alias["realname"]
|
||||||
|
self.username = alias["nick"].lower() + "/" + relay["net"]
|
||||||
|
self.password = main.config["Relay"]["Password"]
|
||||||
|
self.userinfo = None #
|
||||||
|
self.fingerReply = None #
|
||||||
|
self.versionName = None # Don't give out information
|
||||||
|
self.versionNum = None #
|
||||||
|
self.versionEnv = None #
|
||||||
|
self.sourceURL = None #
|
||||||
|
|
||||||
|
self._getWho = {} # LoopingCall objects -- needed to be able to stop them
|
||||||
|
|
||||||
|
self._tempWho = {} # temporary storage for gathering WHO info
|
||||||
|
self._tempNames = {} # temporary storage for gathering NAMES info
|
||||||
|
self._tempList = ([], []) # temporary storage for gathering LIST info
|
||||||
|
self.listOngoing = False # we are currently receiving a LIST
|
||||||
|
self.listRetried = False # we asked and got nothing so asked again
|
||||||
|
self.listAttempted = False # we asked for a list
|
||||||
|
self.listSimple = False # after asking again we got the list, so use the simple
|
||||||
|
# syntax from now on
|
||||||
|
self.wantList = (
|
||||||
|
False # we want to send a LIST, but not all relays are active yet
|
||||||
|
)
|
||||||
|
self.chanlimit = 0
|
||||||
|
self.prefix = {}
|
||||||
|
self.servername = None
|
||||||
|
|
||||||
|
self._regAttempt = None
|
||||||
|
self._negativePass = None
|
||||||
|
|
||||||
|
def lineReceived(self, line):
|
||||||
|
if bytes != str and isinstance(line, bytes):
|
||||||
|
# decode bytes from transport to unicode
|
||||||
|
line = line.decode("utf-8", "replace")
|
||||||
|
|
||||||
|
line = lowDequote(line)
|
||||||
|
try:
|
||||||
|
prefix, command, params = parsemsg(line)
|
||||||
|
if command in numeric_to_symbolic:
|
||||||
|
command = numeric_to_symbolic[command]
|
||||||
|
try:
|
||||||
|
self.handleCommand(command, prefix, params)
|
||||||
|
except Exception as err:
|
||||||
|
error(err)
|
||||||
|
except IRCBadMessage:
|
||||||
|
self.badMessage(line, *sys.exc_info())
|
||||||
|
|
||||||
|
def joinChannels(self, channels):
|
||||||
|
sleeptime = 0.0
|
||||||
|
increment = 0.8
|
||||||
|
for i in channels:
|
||||||
|
if i not in self.channels:
|
||||||
|
if self.net in main.blacklist.keys():
|
||||||
|
if i in main.blacklist[self.net]:
|
||||||
|
debug(
|
||||||
|
"Not joining blacklisted channel %s on %s - %i"
|
||||||
|
% (i, self.net, self.num)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
trace(
|
||||||
|
self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds"
|
||||||
|
)
|
||||||
|
reactor.callLater(sleeptime, self.join, i)
|
||||||
|
sleeptime += increment
|
||||||
|
if sleeptime == 10:
|
||||||
|
sleeptime = 0.0
|
||||||
|
increment = 0.7
|
||||||
|
increment += 0.1
|
||||||
|
else:
|
||||||
|
error(
|
||||||
|
"%s - Cannot join channel we are already on: %s - %i"
|
||||||
|
% (i, self.net, self.num)
|
||||||
|
)
|
||||||
|
|
||||||
|
def checkChannels(self):
|
||||||
|
if chankeep.allRelaysActive(self.net):
|
||||||
|
debug(f"checkChannels() all relays active for {self.net}")
|
||||||
|
else:
|
||||||
|
debug(
|
||||||
|
"checkChannels() skipping channel check as we have inactive relays: %s - %i"
|
||||||
|
% (self.net, self.num)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if self.net in main.TempChan.keys():
|
||||||
|
if self.num in main.TempChan[self.net].keys():
|
||||||
|
self.joinChannels(main.TempChan[self.net][self.num])
|
||||||
|
del main.TempChan[self.net][self.num]
|
||||||
|
if not main.TempChan[self.net]:
|
||||||
|
del main.TempChan[self.net]
|
||||||
|
|
||||||
|
def event(self, **cast):
|
||||||
|
if "ts" not in cast.keys():
|
||||||
|
cast["ts"] = int(datetime.now().timestamp())
|
||||||
|
|
||||||
|
# remove odd stuff
|
||||||
|
for i in list(
|
||||||
|
cast.keys()
|
||||||
|
): # Make a copy of the .keys() as Python 3 cannot handle iterating over
|
||||||
|
if cast[i] == "": # a dictionary that changes length with each iteration
|
||||||
|
del cast[i]
|
||||||
|
# remove server stuff
|
||||||
|
if "muser" in cast.keys():
|
||||||
|
if cast["muser"] == self.servername:
|
||||||
|
cast["type"] = "conn"
|
||||||
|
if "channel" in cast.keys():
|
||||||
|
if cast["channel"] == "*":
|
||||||
|
cast["type"] = "conn"
|
||||||
|
##
|
||||||
|
|
||||||
|
# expand out the hostmask
|
||||||
|
if not {"nick", "ident", "host"}.issubset(set(cast.keys())):
|
||||||
|
if "muser" in cast.keys():
|
||||||
|
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
|
||||||
|
|
||||||
|
# handle ZNC stuff
|
||||||
|
if {"nick", "ident", "host", "msg"}.issubset(set(cast)):
|
||||||
|
if cast["ident"] == "znc" and cast["host"] == "znc.in":
|
||||||
|
cast["type"] = "znc"
|
||||||
|
cast["num"] = self.num
|
||||||
|
del cast["nick"]
|
||||||
|
del cast["ident"]
|
||||||
|
del cast["host"]
|
||||||
|
del cast["channel"]
|
||||||
|
del cast["muser"]
|
||||||
|
if "Disconnected from IRC" in cast["msg"]:
|
||||||
|
log("ZNC disconnected on %s - %i" % (self.net, self.num))
|
||||||
|
self.isconnected = False
|
||||||
|
self.authenticated = False
|
||||||
|
if "Connected!" in cast["msg"]:
|
||||||
|
log("ZNC connected on %s - %i" % (self.net, self.num))
|
||||||
|
self.isconnected = True
|
||||||
|
if "could not be joined, disabling it" in cast["msg"]:
|
||||||
|
error(cast["msg"])
|
||||||
|
#
|
||||||
|
|
||||||
|
# don't reprocess the same message twice
|
||||||
|
# if the type is in that list, it's already been here, don't run it again
|
||||||
|
if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
|
||||||
|
cast["num"] = self.num
|
||||||
|
if "channel" in cast.keys():
|
||||||
|
if cast["type"] == "mode":
|
||||||
|
if cast["channel"].lower() == self.nickname.lower():
|
||||||
|
# castDup = deepcopy(cast)
|
||||||
|
cast["mtype"] = cast["type"]
|
||||||
|
cast["type"] = "self"
|
||||||
|
# self.event(**castDup)
|
||||||
|
if cast["modearg"]: # check if modearg is non-NoneType
|
||||||
|
if self.nickname.lower() == cast["modearg"].lower():
|
||||||
|
castDup = deepcopy(cast)
|
||||||
|
castDup["mtype"] = cast["type"]
|
||||||
|
castDup["type"] = "highlight"
|
||||||
|
self.event(**castDup)
|
||||||
|
else:
|
||||||
|
if cast["channel"].lower() == self.nickname.lower() or not cast[
|
||||||
|
"channel"
|
||||||
|
].startswith("#"):
|
||||||
|
cast["mtype"] = cast["type"]
|
||||||
|
cast["type"] = "query"
|
||||||
|
# self.event(**castDup)
|
||||||
|
# Don't call self.event for this one because queries are not events on a
|
||||||
|
# channel, but we still want to see them
|
||||||
|
if cast["channel"] == "AUTH":
|
||||||
|
cast["type"] = "conn"
|
||||||
|
cast["mtype"] = cast["type"]
|
||||||
|
|
||||||
|
# TODO: better way to do this
|
||||||
|
# as we changed the types above, check again
|
||||||
|
if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}:
|
||||||
|
# we have been kicked
|
||||||
|
if "user" in cast.keys():
|
||||||
|
if cast["user"].lower() == self.nickname.lower():
|
||||||
|
castDup = deepcopy(cast)
|
||||||
|
castDup["mtype"] = cast["type"]
|
||||||
|
castDup["type"] = "self"
|
||||||
|
if self.net in main.blacklist.keys():
|
||||||
|
if cast["channel"] not in main.blacklist[self.net]:
|
||||||
|
main.blacklist[self.net].append(cast["channel"])
|
||||||
|
else:
|
||||||
|
main.blacklist[self.net] = [cast["channel"]]
|
||||||
|
main.saveConf("blacklist")
|
||||||
|
self.event(**castDup)
|
||||||
|
|
||||||
|
# we sent a message/left/joined/kick someone/quit
|
||||||
|
if "nick" in cast.keys():
|
||||||
|
if cast["nick"].lower() == self.nickname.lower():
|
||||||
|
castDup = deepcopy(cast)
|
||||||
|
castDup["mtype"] = cast["type"]
|
||||||
|
castDup["type"] = "self"
|
||||||
|
|
||||||
|
# we have been mentioned in a msg/notice/action/part/quit/topic message
|
||||||
|
if "msg" in cast.keys(): # Don't highlight queries
|
||||||
|
if cast["msg"] is not None:
|
||||||
|
if self.nickname.lower() in cast["msg"].lower():
|
||||||
|
castDup = deepcopy(cast)
|
||||||
|
castDup["mtype"] = cast["type"]
|
||||||
|
castDup["type"] = "highlight"
|
||||||
|
self.event(**castDup)
|
||||||
|
|
||||||
|
if "net" not in cast.keys():
|
||||||
|
cast["net"] = self.net
|
||||||
|
if "num" not in cast.keys():
|
||||||
|
cast["num"] = self.num
|
||||||
|
if not self.authenticated:
|
||||||
|
regproc.registerTest(cast)
|
||||||
|
counters.event(self.net, cast["type"])
|
||||||
|
monitor.event(self.net, cast)
|
||||||
|
|
||||||
|
def privmsg(self, user, channel, msg):
|
||||||
|
self.event(type="msg", muser=user, channel=channel, msg=msg)
|
||||||
|
|
||||||
|
def noticed(self, user, channel, msg):
|
||||||
|
self.event(type="notice", muser=user, channel=channel, msg=msg)
|
||||||
|
|
||||||
|
def action(self, user, channel, msg):
|
||||||
|
self.event(type="action", muser=user, channel=channel, msg=msg)
|
||||||
|
|
||||||
|
def sendmsg(self, channel, msg, in_query=False):
|
||||||
|
query = f"{self.nickname}!*@*"
|
||||||
|
res = userinfo.getWhoSingle(self.net, query)
|
||||||
|
if not res:
|
||||||
|
res = []
|
||||||
|
us = list(res)
|
||||||
|
if len(us) > 0:
|
||||||
|
hostmask = us[0]
|
||||||
|
else:
|
||||||
|
# Close enough...
|
||||||
|
hostmask = f"{self.nickname}!{self.nickname}@{self.servername}"
|
||||||
|
warn(f"Could not get a hostname, using {hostmask}")
|
||||||
|
nick, ident, host = parsen(hostmask)
|
||||||
|
# We sent someone a query reply
|
||||||
|
if in_query:
|
||||||
|
self.event(
|
||||||
|
type="self",
|
||||||
|
mtype="msg",
|
||||||
|
channel=self.nickname,
|
||||||
|
nick=channel,
|
||||||
|
ident=ident,
|
||||||
|
host=host,
|
||||||
|
msg=msg,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.event(
|
||||||
|
type="self",
|
||||||
|
mtype="msg",
|
||||||
|
channel=channel,
|
||||||
|
nick=self.nickname,
|
||||||
|
ident=ident,
|
||||||
|
host=host,
|
||||||
|
msg=msg,
|
||||||
|
)
|
||||||
|
self.event(
|
||||||
|
type="msg",
|
||||||
|
channel=channel,
|
||||||
|
nick=self.nickname,
|
||||||
|
ident=ident,
|
||||||
|
host=host,
|
||||||
|
msg=msg,
|
||||||
|
)
|
||||||
|
self.msg(channel, msg)
|
||||||
|
|
||||||
|
def get(self, var):
|
||||||
|
try:
|
||||||
|
result = getattr(self, var)
|
||||||
|
except AttributeError:
|
||||||
|
result = None
|
||||||
|
return result
|
||||||
|
|
||||||
|
def setNick(self, nickname):
|
||||||
|
self._attemptedNick = nickname
|
||||||
|
self.sendLine("NICK %s" % nickname)
|
||||||
|
self.nickname = nickname
|
||||||
|
|
||||||
|
def alterCollidedNick(self, nickname):
|
||||||
|
newnick = nickname + "_"
|
||||||
|
return newnick
|
||||||
|
|
||||||
|
def nickChanged(self, olduser, newnick):
|
||||||
|
self.nickname = newnick
|
||||||
|
self.event(type="self", mtype="nick", muser=olduser, user=newnick)
|
||||||
|
|
||||||
|
def irc_ERR_NICKNAMEINUSE(self, prefix, params):
|
||||||
|
self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
|
||||||
|
self.setNick(self._attemptedNick)
|
||||||
|
|
||||||
|
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
|
||||||
|
log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username))
|
||||||
|
sendAll(
|
||||||
|
"%s - %i: password mismatch as %s" % (self.net, self.num, self.username)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _who(self, channel):
|
||||||
|
d = Deferred()
|
||||||
|
if channel not in self._tempWho:
|
||||||
|
self._tempWho[channel] = ([], [])
|
||||||
|
self._tempWho[channel][0].append(d)
|
||||||
|
self.sendLine("WHO %s" % channel)
|
||||||
|
return d
|
||||||
|
|
||||||
|
def who(self, channel):
|
||||||
|
self._who(channel).addCallback(self.got_who)
|
||||||
|
|
||||||
|
def irc_RPL_WHOREPLY(self, prefix, params):
|
||||||
|
channel = params[1]
|
||||||
|
ident = params[2]
|
||||||
|
host = params[3]
|
||||||
|
server = params[4]
|
||||||
|
nick = params[5]
|
||||||
|
status = params[6]
|
||||||
|
realname = params[7]
|
||||||
|
if channel not in self._tempWho:
|
||||||
|
return
|
||||||
|
n = self._tempWho[channel][1]
|
||||||
|
n.append([nick, nick, host, server, status, realname])
|
||||||
|
self.event(
|
||||||
|
type="who",
|
||||||
|
nick=nick,
|
||||||
|
ident=ident,
|
||||||
|
host=host,
|
||||||
|
realname=realname,
|
||||||
|
channel=channel,
|
||||||
|
server=server,
|
||||||
|
status=status,
|
||||||
|
)
|
||||||
|
|
||||||
|
def irc_RPL_ENDOFWHO(self, prefix, params):
|
||||||
|
channel = params[1]
|
||||||
|
if channel not in self._tempWho:
|
||||||
|
return
|
||||||
|
callbacks, info = self._tempWho[channel]
|
||||||
|
for cb in callbacks:
|
||||||
|
cb.callback((channel, info))
|
||||||
|
del self._tempWho[channel]
|
||||||
|
|
||||||
|
def got_who(self, whoinfo):
|
||||||
|
userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
|
||||||
|
|
||||||
|
def sanit(self, data):
|
||||||
|
if len(data) >= 1:
|
||||||
|
if data[0] in self.prefix.keys():
|
||||||
|
return (
|
||||||
|
self.prefix[data[0]],
|
||||||
|
data[1:],
|
||||||
|
) # would use a set but it's possible these are the same
|
||||||
|
return (None, data)
|
||||||
|
else:
|
||||||
|
return (None, False)
|
||||||
|
|
||||||
|
def names(self, channel):
|
||||||
|
d = Deferred()
|
||||||
|
if channel not in self._tempNames:
|
||||||
|
self._tempNames[channel] = ([], [])
|
||||||
|
self._tempNames[channel][0].append(d)
|
||||||
|
self.sendLine("NAMES %s" % channel)
|
||||||
|
return d
|
||||||
|
|
||||||
|
def irc_RPL_NAMREPLY(self, prefix, params):
|
||||||
|
channel = params[2]
|
||||||
|
nicklist = params[3].split(" ")
|
||||||
|
if channel not in self._tempNames:
|
||||||
|
return
|
||||||
|
n = self._tempNames[channel][1]
|
||||||
|
n.append(nicklist)
|
||||||
|
|
||||||
|
def irc_RPL_ENDOFNAMES(self, prefix, params):
|
||||||
|
channel = params[1]
|
||||||
|
if channel not in self._tempNames:
|
||||||
|
return
|
||||||
|
callbacks, namelist = self._tempNames[channel]
|
||||||
|
for cb in callbacks:
|
||||||
|
cb.callback((channel, namelist))
|
||||||
|
del self._tempNames[channel]
|
||||||
|
|
||||||
|
def got_names(self, nicklist):
|
||||||
|
newNicklist = []
|
||||||
|
for i in nicklist[1]:
|
||||||
|
for x in i:
|
||||||
|
mode, nick = self.sanit(x)
|
||||||
|
if nick:
|
||||||
|
newNicklist.append((mode, nick))
|
||||||
|
userinfo.initialNames(self.net, nicklist[0], newNicklist)
|
||||||
|
|
||||||
|
def myInfo(self, servername, version, umodes, cmodes):
|
||||||
|
self.servername = servername
|
||||||
|
|
||||||
|
def _list(self, noargs):
|
||||||
|
d = Deferred()
|
||||||
|
self._tempList = ([], [])
|
||||||
|
self._tempList[0].append(d)
|
||||||
|
if self.listSimple:
|
||||||
|
self.sendLine("LIST")
|
||||||
|
return d # return early if we know what to do
|
||||||
|
|
||||||
|
if noargs:
|
||||||
|
self.sendLine("LIST")
|
||||||
|
else:
|
||||||
|
self.sendLine("LIST >0")
|
||||||
|
return d
|
||||||
|
|
||||||
|
def list(self, noargs=False, nocheck=False):
|
||||||
|
if not self.authenticated:
|
||||||
|
debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num))
|
||||||
|
return
|
||||||
|
if self.listAttempted:
|
||||||
|
debug(
|
||||||
|
"List request dropped, already asked for LIST - %s - %i"
|
||||||
|
% (self.net, self.num)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
self.listAttempted = True
|
||||||
|
if self.listOngoing:
|
||||||
|
debug(
|
||||||
|
"LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
if nocheck:
|
||||||
|
allRelays = True # override the system - if this is
|
||||||
|
else: # specified, we already did this
|
||||||
|
allRelays = chankeep.allRelaysActive(self.net)
|
||||||
|
if not allRelays:
|
||||||
|
self.wantList = True
|
||||||
|
debug("Not all relays were active for LIST request")
|
||||||
|
return
|
||||||
|
self._list(noargs).addCallback(self.got_list)
|
||||||
|
|
||||||
|
def irc_RPL_LISTSTART(self, prefix, params):
|
||||||
|
self.listAttempted = False
|
||||||
|
self.listOngoing = True
|
||||||
|
self.wantList = False
|
||||||
|
|
||||||
|
def irc_RPL_LIST(self, prefix, params):
|
||||||
|
channel = params[1]
|
||||||
|
users = params[2]
|
||||||
|
topic = params[3]
|
||||||
|
self._tempList[1].append([channel, users, topic])
|
||||||
|
|
||||||
|
def irc_RPL_LISTEND(self, prefix, params):
|
||||||
|
if (
|
||||||
|
not len(self._tempList[0]) > 0
|
||||||
|
): # there are no callbacks, can't do anything there
|
||||||
|
debug("We didn't ask for this LIST, discarding")
|
||||||
|
self._tempList[0].clear()
|
||||||
|
self._tempList[1].clear()
|
||||||
|
return
|
||||||
|
callbacks, info = self._tempList
|
||||||
|
self.listOngoing = False
|
||||||
|
for cb in callbacks:
|
||||||
|
cb.callback((info))
|
||||||
|
noResults = False
|
||||||
|
if len(self._tempList[1]) == 0:
|
||||||
|
noResults = True
|
||||||
|
self._tempList[0].clear()
|
||||||
|
self._tempList[1].clear()
|
||||||
|
if noResults:
|
||||||
|
if self.listRetried:
|
||||||
|
warn("LIST still empty after retry: %s - %i" % (self.net, self.num))
|
||||||
|
self.listRetried = False
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
self.list(True)
|
||||||
|
self.listRetried = True
|
||||||
|
else:
|
||||||
|
if self.listRetried:
|
||||||
|
self.listRetried = False
|
||||||
|
debug(
|
||||||
|
"List received after retry - defaulting to simple list syntax: %s - %i"
|
||||||
|
% (self.net, self.num)
|
||||||
|
)
|
||||||
|
self.listSimple = True
|
||||||
|
|
||||||
|
def got_list(self, listinfo):
|
||||||
|
if len(listinfo) == 0: # probably ngircd not supporting LIST >0
|
||||||
|
return
|
||||||
|
if main.config["ChanKeep"]["Enabled"]:
|
||||||
|
chankeep.initialList(self.net, self.num, listinfo)
|
||||||
|
|
||||||
|
def recheckList(self):
|
||||||
|
if not main.config["ChanKeep"]["Enabled"]:
|
||||||
|
return
|
||||||
|
allRelays = chankeep.allRelaysActive(self.net)
|
||||||
|
debug(f"recheckList() all relays for {self.net} {allRelays}")
|
||||||
|
if allRelays:
|
||||||
|
debug(f"recheckList() all relays active for {self.net}")
|
||||||
|
first_relay = helpers.get_first_relay(self.net)
|
||||||
|
debug(f"recheckList() first relay for {self.net}: {first_relay.num}")
|
||||||
|
if first_relay:
|
||||||
|
if first_relay.wantList is True:
|
||||||
|
first_relay.list(nocheck=True)
|
||||||
|
debug(
|
||||||
|
f"recheckList() asking for a list for {self.net} after final relay {self.num} connected"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
debug(
|
||||||
|
f"recheckList() first relay wantList is False for {self.net} ({first_relay.num})"
|
||||||
|
)
|
||||||
|
# name = self.net + "1"
|
||||||
|
|
||||||
|
# if self.num == 1: # Only one instance should do a list
|
||||||
|
if helpers.is_first_relay(self.net, self.num):
|
||||||
|
debug(f"recheckList() we are the first relay for {self.net} ({self.num})")
|
||||||
|
if self.chanlimit:
|
||||||
|
if allRelays:
|
||||||
|
self.list()
|
||||||
|
debug(
|
||||||
|
f"recheckList() requested a list for {self.net} from {self.num}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
debug(f"recheckList() not all relays active for {self.net}")
|
||||||
|
self.wantList = True
|
||||||
|
else:
|
||||||
|
debug("recheckList() aborting LIST due to bad chanlimit")
|
||||||
|
self.checkChannels()
|
||||||
|
|
||||||
|
def seed_chanlimit(self, chanlimit):
|
||||||
|
if not main.network[self.net].relays[self.num][
|
||||||
|
"registered"
|
||||||
|
]: # TODO: add check for register request sent, only send it once
|
||||||
|
if main.config["AutoReg"]:
|
||||||
|
if not self.authenticated:
|
||||||
|
self._regAttempt = reactor.callLater(
|
||||||
|
5, regproc.registerAccount, self.net, self.num
|
||||||
|
)
|
||||||
|
# regproc.registerAccount(self.net, self.num)
|
||||||
|
try:
|
||||||
|
self.chanlimit = int(chanlimit)
|
||||||
|
except TypeError:
|
||||||
|
warn("Invalid chanlimit: %s" % chanlimit)
|
||||||
|
if self.chanlimit == 0:
|
||||||
|
self.chanlimit = 200 # don't take the piss if it's not limited
|
||||||
|
net_inst_chanlimit = self.netinst.chanlimit
|
||||||
|
if net_inst_chanlimit:
|
||||||
|
if self.chanlimit > net_inst_chanlimit:
|
||||||
|
self.chanlimit = net_inst_chanlimit
|
||||||
|
# warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}")
|
||||||
|
|
||||||
|
if not regproc.needToRegister(
|
||||||
|
self.net
|
||||||
|
): # if we need to register, only recheck on auth confirmation
|
||||||
|
if main.config["ChanKeep"]["Enabled"]:
|
||||||
|
self.recheckList()
|
||||||
|
|
||||||
|
def seed_prefix(self, prefix):
|
||||||
|
prefix = prefix.replace(")", "")
|
||||||
|
prefix = prefix.replace("(", "")
|
||||||
|
length = len(prefix)
|
||||||
|
half = int(length / 2)
|
||||||
|
prefixToMode = dict(zip(prefix[half:], prefix[:half]))
|
||||||
|
self.prefix = prefixToMode
|
||||||
|
|
||||||
|
def isupport(self, options):
|
||||||
|
interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"}
|
||||||
|
newOptions = {x for x in options if any(y in x for y in interested)}
|
||||||
|
if len(newOptions) == 0:
|
||||||
|
return
|
||||||
|
if not self.isconnected:
|
||||||
|
log("endpoint connected: %s - %i" % (self.net, self.num))
|
||||||
|
self.isconnected = True
|
||||||
|
for i in newOptions:
|
||||||
|
if i.startswith("PREFIX"):
|
||||||
|
if "=" in i:
|
||||||
|
split = i.split("=")
|
||||||
|
if len(split) == 2:
|
||||||
|
prefix = split[1]
|
||||||
|
self.seed_prefix(prefix)
|
||||||
|
elif i.startswith("CHANLIMIT"):
|
||||||
|
if ":" in i:
|
||||||
|
split = i.split(":")
|
||||||
|
if len(split) >= 2:
|
||||||
|
chanlimit = split[1]
|
||||||
|
self.seed_chanlimit(chanlimit)
|
||||||
|
elif i.startswith("MAXCHANNELS"):
|
||||||
|
if "=" in i:
|
||||||
|
split = i.split("=")
|
||||||
|
if len(split) == 2:
|
||||||
|
chanlimit = split[1]
|
||||||
|
self.seed_chanlimit(chanlimit)
|
||||||
|
|
||||||
|
# We need to override these functions as Twisted discards
|
||||||
|
# the hostname and other useful information in the functions
|
||||||
|
# that these call by default
|
||||||
|
def irc_JOIN(self, prefix, params):
|
||||||
|
nick = prefix.split("!")[0]
|
||||||
|
channel = params[-1]
|
||||||
|
if nick == self.nickname:
|
||||||
|
self.joined(channel)
|
||||||
|
else:
|
||||||
|
self.userJoined(prefix, channel)
|
||||||
|
|
||||||
|
def irc_PART(self, prefix, params):
|
||||||
|
nick = prefix.split("!")[0]
|
||||||
|
channel = params[0]
|
||||||
|
if len(params) >= 2:
|
||||||
|
message = params[1]
|
||||||
|
else:
|
||||||
|
message = None
|
||||||
|
if nick == self.nickname:
|
||||||
|
self.left(prefix, channel, message)
|
||||||
|
else:
|
||||||
|
self.userLeft(prefix, channel, message)
|
||||||
|
|
||||||
|
def irc_QUIT(self, prefix, params):
|
||||||
|
# nick = prefix.split("!")[0]
|
||||||
|
self.userQuit(prefix, params[0])
|
||||||
|
|
||||||
|
def irc_NICK(self, prefix, params):
|
||||||
|
nick = prefix.split("!", 1)[0]
|
||||||
|
if nick == self.nickname:
|
||||||
|
self.nickChanged(prefix, params[0])
|
||||||
|
else:
|
||||||
|
self.userRenamed(prefix, params[0])
|
||||||
|
|
||||||
|
def irc_KICK(self, prefix, params):
|
||||||
|
channel = params[0]
|
||||||
|
kicked = params[1]
|
||||||
|
message = params[-1]
|
||||||
|
# Checks on whether it was us that was kicked are done in userKicked
|
||||||
|
self.userKicked(kicked, channel, prefix, message)
|
||||||
|
|
||||||
|
def irc_TOPIC(self, prefix, params):
|
||||||
|
channel = params[0]
|
||||||
|
newtopic = params[1]
|
||||||
|
self.topicUpdated(prefix, channel, newtopic)
|
||||||
|
|
||||||
|
# End of Twisted hackery
|
||||||
|
|
||||||
|
def regPing(self, negativepass=None, reset=True):
|
||||||
|
if not main.config["AutoReg"]:
|
||||||
|
return
|
||||||
|
if self.authenticated:
|
||||||
|
return
|
||||||
|
if not regproc.needToAuth(self.net):
|
||||||
|
self.authenticated = True
|
||||||
|
return
|
||||||
|
sinst = regproc.substitute(self.net, self.num)
|
||||||
|
if not sinst:
|
||||||
|
error(f"regPing() {self.net}: registration ping failed for {self.num}")
|
||||||
|
return
|
||||||
|
if reset:
|
||||||
|
self._negativePass = None
|
||||||
|
|
||||||
|
debug(
|
||||||
|
f"regPing() {self.net} - {self.num}: _negativePass:{self._negativePass} negativepass:{negativepass}"
|
||||||
|
)
|
||||||
|
if self._negativePass is None:
|
||||||
|
# We have failed, the blacklisted message has been found
|
||||||
|
if negativepass is False:
|
||||||
|
self._negativePass = False
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"regPing() {self.net} - {self.num} not passed negative:checknegativemsg "
|
||||||
|
f"check, {sinst['checknegativemsg']} present in message"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
# End of negative output reached with no blacklisted message
|
||||||
|
elif negativepass is True:
|
||||||
|
if self._negativePass is None: # check if it's not failed yet
|
||||||
|
self._negativePass = True
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"regPing() {self.net} - {self.num} passed negative:checkendnegative "
|
||||||
|
f"check, {sinst['checkendnegative']} present in message"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
debug(
|
||||||
|
f"regPing() {self.net}: negative registration check - {self.num}"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
if sinst["negative"]:
|
||||||
|
self.msg(sinst["entity"], sinst["negativemsg"])
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"regPing() {self.net}: sent negativemsg "
|
||||||
|
f"'{sinst['negativemsg']}' to {sinst['entity']} - {self.num}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
self._negativePass = (
|
||||||
|
True # if it's disabled, we still want the next block to run
|
||||||
|
)
|
||||||
|
|
||||||
|
# Only run if negativepass has completed, or exempted as above
|
||||||
|
if self._negativePass:
|
||||||
|
if sinst["check"]:
|
||||||
|
if sinst["ping"]:
|
||||||
|
self.msg(sinst["entity"], sinst["pingmsg"])
|
||||||
|
debug(
|
||||||
|
f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
self.authenticated = True
|
||||||
|
|
||||||
|
def signedOn(self):
|
||||||
|
log("signed on: %s - %i" % (self.net, self.num))
|
||||||
|
ctime = str(datetime.now().isoformat())
|
||||||
|
sendRelayNotification(
|
||||||
|
{
|
||||||
|
"type": "conn",
|
||||||
|
"net": self.net,
|
||||||
|
"num": self.num,
|
||||||
|
"status": "signedon",
|
||||||
|
"ts": ctime,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not self.authenticated:
|
||||||
|
reactor.callLater(10, self.regPing)
|
||||||
|
|
||||||
|
def setup_who_loop(self, channel):
|
||||||
|
# if main.config["Toggles"]["Who"]:
|
||||||
|
lc = LoopingCall(self.who, channel)
|
||||||
|
self._getWho[channel] = lc
|
||||||
|
intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
|
||||||
|
minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
|
||||||
|
interval = randint(minint, minint + intrange)
|
||||||
|
lc.start(interval)
|
||||||
|
|
||||||
|
def joined(self, channel):
|
||||||
|
if channel not in self.channels:
|
||||||
|
self.channels.append(channel)
|
||||||
|
self.names(channel).addCallback(self.got_names)
|
||||||
|
if main.config["Toggles"]["Who"]:
|
||||||
|
reactor.callLater(
|
||||||
|
main.config["Tweaks"]["Delays"]["WhoDelay"],
|
||||||
|
self.setup_who_loop,
|
||||||
|
channel,
|
||||||
|
)
|
||||||
|
# lc = LoopingCall(self.who, channel)
|
||||||
|
# self._getWho[channel] = lc
|
||||||
|
# intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
|
||||||
|
# minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
|
||||||
|
# interval = randint(minint, minint + intrange)
|
||||||
|
# lc.start(interval)
|
||||||
|
|
||||||
|
def botLeft(self, channel):
|
||||||
|
if channel in self.channels:
|
||||||
|
self.channels.remove(channel)
|
||||||
|
if channel in self._getWho.keys():
|
||||||
|
lc = self._getWho[channel]
|
||||||
|
lc.stop()
|
||||||
|
del self._getWho[channel]
|
||||||
|
userinfo.delChannels(
|
||||||
|
self.net, [channel]
|
||||||
|
) # < we do not need to deduplicate this
|
||||||
|
# log("Can no longer cover %s, removing records" % channel)# as it will only be matched once --
|
||||||
|
# other bots have different nicknames so
|
||||||
|
|
||||||
|
def left(self, user, channel, message): # even if they saw it, they wouldn't react
|
||||||
|
self.event(type="part", muser=user, channel=channel, msg=message)
|
||||||
|
self.botLeft(channel)
|
||||||
|
|
||||||
|
def userJoined(self, user, channel):
|
||||||
|
self.event(type="join", muser=user, channel=channel)
|
||||||
|
|
||||||
|
def userLeft(self, user, channel, message):
|
||||||
|
self.event(type="part", muser=user, channel=channel, msg=message)
|
||||||
|
|
||||||
|
def userQuit(self, user, quitMessage):
|
||||||
|
self.chanlessEvent({"type": "quit", "muser": user, "msg": quitMessage})
|
||||||
|
|
||||||
|
def userKicked(self, kickee, channel, kicker, message):
|
||||||
|
if kickee.lower() == self.nickname.lower():
|
||||||
|
self.botLeft(channel)
|
||||||
|
self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee)
|
||||||
|
|
||||||
|
def chanlessEvent(self, cast):
|
||||||
|
cast["ts"] = int(datetime.now().timestamp())
|
||||||
|
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
|
||||||
|
if dedup(self.name, cast): # Needs to be kept self.name until the dedup
|
||||||
|
# function is converted to the new net, num
|
||||||
|
# format
|
||||||
|
return # stop right there sir!
|
||||||
|
chans = userinfo.getChanList(self.net, cast["nick"])
|
||||||
|
if chans is None:
|
||||||
|
error("No channels returned for chanless event: %s" % cast)
|
||||||
|
# self.event(**cast) -- no, should NEVER happen
|
||||||
|
return
|
||||||
|
# getChansSingle returns all channels of the user, we only want to use
|
||||||
|
# ones we have common with them
|
||||||
|
realChans = set(chans).intersection(set(self.channels))
|
||||||
|
for i in realChans:
|
||||||
|
cast["channel"] = i
|
||||||
|
self.event(**cast)
|
||||||
|
|
||||||
|
def userRenamed(self, oldname, newname):
|
||||||
|
self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
|
||||||
|
|
||||||
|
def topicUpdated(self, user, channel, newTopic):
|
||||||
|
self.event(type="topic", muser=user, channel=channel, msg=newTopic)
|
||||||
|
|
||||||
|
def modeChanged(self, user, channel, toset, modes, args):
|
||||||
|
argList = list(args)
|
||||||
|
modeList = [i for i in modes]
|
||||||
|
for a, m in zip(argList, modeList):
|
||||||
|
self.event(
|
||||||
|
type="mode",
|
||||||
|
muser=user,
|
||||||
|
channel=channel,
|
||||||
|
mode=m,
|
||||||
|
status=toset,
|
||||||
|
modearg=a,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: strip out relay functionality
|
||||||
|
class IRCBotFactory(ReconnectingClientFactory):
|
||||||
|
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
|
||||||
|
if net is None:
|
||||||
|
self.num = num
|
||||||
|
self.net = None
|
||||||
|
self.name = "relay - %i" % num
|
||||||
|
self.relay = True
|
||||||
|
else:
|
||||||
|
self.name = net + str(num)
|
||||||
|
self.num = num
|
||||||
|
self.net = net
|
||||||
|
self.relay = False
|
||||||
|
self.client = None
|
||||||
|
self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
|
||||||
|
self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
|
||||||
|
self.factor = main.config["Tweaks"]["Delays"]["Factor"]
|
||||||
|
self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
|
||||||
|
|
||||||
|
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
|
||||||
|
|
||||||
|
def buildProtocol(self, addr):
|
||||||
|
entry = IRCBot(self.net, self.num)
|
||||||
|
main.IRCPool[self.name] = entry
|
||||||
|
|
||||||
|
self.client = entry
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def clientConnectionLost(self, connector, reason):
|
||||||
|
if not self.relay:
|
||||||
|
userinfo.delChannels(self.net, self.client.channels)
|
||||||
|
if self.client is not None:
|
||||||
|
self.client.isconnected = False
|
||||||
|
self.client.authenticated = False
|
||||||
|
self.client.channels = []
|
||||||
|
error = reason.getErrorMessage()
|
||||||
|
log("%s - %i: connection lost: %s" % (self.net, self.num, error))
|
||||||
|
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
|
||||||
|
ctime = str(datetime.now().isoformat())
|
||||||
|
sendRelayNotification(
|
||||||
|
{
|
||||||
|
"type": "conn",
|
||||||
|
"net": self.net,
|
||||||
|
"num": self.num,
|
||||||
|
"status": "lost",
|
||||||
|
"message": error,
|
||||||
|
"ts": ctime,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.retry(connector)
|
||||||
|
# ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
|
||||||
|
|
||||||
|
def clientConnectionFailed(self, connector, reason):
|
||||||
|
if self.client is not None:
|
||||||
|
self.client.isconnected = False
|
||||||
|
self.client.authenticated = False
|
||||||
|
self.client.channels = []
|
||||||
|
error = reason.getErrorMessage()
|
||||||
|
log("%s - %i: connection failed: %s" % (self.net, self.num, error))
|
||||||
|
if not self.relay:
|
||||||
|
sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
|
||||||
|
ctime = str(datetime.now().isoformat())
|
||||||
|
sendRelayNotification(
|
||||||
|
{
|
||||||
|
"type": "conn",
|
||||||
|
"net": self.net,
|
||||||
|
"num": self.num,
|
||||||
|
"status": "failed",
|
||||||
|
"message": error,
|
||||||
|
"ts": ctime,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.retry(connector)
|
||||||
|
# ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
import main
|
||||||
|
from utils.logging.log import warn
|
||||||
|
from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess
|
||||||
|
|
||||||
|
|
||||||
|
def parseCommand(addr, authed, data):
|
||||||
|
# call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
|
||||||
|
spl = data.split()
|
||||||
|
if addr in main.connections.keys():
|
||||||
|
obj = main.connections[addr]
|
||||||
|
else:
|
||||||
|
warn("Got connection object with no instance in the address pool")
|
||||||
|
return
|
||||||
|
|
||||||
|
success = lambda data: sendSuccess(addr, data) # noqa: E731
|
||||||
|
failure = lambda data: sendFailure(addr, data) # noqa: E731
|
||||||
|
info = lambda data: sendInfo(addr, data) # noqa: E731
|
||||||
|
|
||||||
|
incUsage = lambda mode: incorrectUsage(addr, mode) # noqa: E731
|
||||||
|
length = len(spl)
|
||||||
|
if spl[0] in main.CommandMap.keys():
|
||||||
|
main.CommandMap[spl[0]](
|
||||||
|
addr, authed, data, obj, spl, success, failure, info, incUsage, length
|
||||||
|
)
|
||||||
|
return
|
||||||
|
incUsage(None)
|
||||||
|
return
|
||||||
|
|
@ -1,12 +1,28 @@
|
||||||
from twisted.internet.protocol import Protocol, Factory, ClientFactory
|
|
||||||
from json import dumps, loads
|
from json import dumps, loads
|
||||||
from copy import deepcopy
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
import main
|
import main
|
||||||
from utils.logging.log import *
|
from twisted.internet.protocol import Factory, Protocol
|
||||||
|
from utils.logging.log import log, warn
|
||||||
|
|
||||||
|
validTypes = [
|
||||||
|
"msg",
|
||||||
|
"notice",
|
||||||
|
"action",
|
||||||
|
"who",
|
||||||
|
"part",
|
||||||
|
"join",
|
||||||
|
"kick",
|
||||||
|
"quit",
|
||||||
|
"nick",
|
||||||
|
"topic",
|
||||||
|
"mode",
|
||||||
|
"conn",
|
||||||
|
"znc",
|
||||||
|
"query",
|
||||||
|
"self",
|
||||||
|
"highlight",
|
||||||
|
]
|
||||||
|
|
||||||
validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"]
|
|
||||||
|
|
||||||
class Relay(Protocol):
|
class Relay(Protocol):
|
||||||
def __init__(self, addr):
|
def __init__(self, addr):
|
||||||
|
|
@ -30,10 +46,10 @@ class Relay(Protocol):
|
||||||
data = data.decode("utf-8", "replace")
|
data = data.decode("utf-8", "replace")
|
||||||
try:
|
try:
|
||||||
parsed = loads(data)
|
parsed = loads(data)
|
||||||
except:
|
except: # noqa: E722
|
||||||
self.sendErr("MALFORMED")
|
self.sendErr("MALFORMED")
|
||||||
return
|
return
|
||||||
if not "type" in parsed.keys():
|
if "type" not in parsed.keys():
|
||||||
self.sendErr("NOTYPE")
|
self.sendErr("NOTYPE")
|
||||||
return
|
return
|
||||||
if parsed["type"] == "hello":
|
if parsed["type"] == "hello":
|
||||||
|
|
@ -70,7 +86,7 @@ class Relay(Protocol):
|
||||||
self.sendErr("NOTLIST")
|
self.sendErr("NOTLIST")
|
||||||
return
|
return
|
||||||
for i in lst:
|
for i in lst:
|
||||||
if not i in validTypes:
|
if i not in validTypes:
|
||||||
self.sendErr("NONEXISTANT")
|
self.sendErr("NONEXISTANT")
|
||||||
return
|
return
|
||||||
if i in self.subscriptions:
|
if i in self.subscriptions:
|
||||||
|
|
@ -85,10 +101,10 @@ class Relay(Protocol):
|
||||||
self.sendErr("NOTLIST")
|
self.sendErr("NOTLIST")
|
||||||
return
|
return
|
||||||
for i in lst:
|
for i in lst:
|
||||||
if not i in validTypes:
|
if i not in validTypes:
|
||||||
self.sendErr("NONEXISTANT")
|
self.sendErr("NONEXISTANT")
|
||||||
return
|
return
|
||||||
if not i in self.subscriptions:
|
if i not in self.subscriptions:
|
||||||
self.sendErr("NOTSUBSCRIBED")
|
self.sendErr("NOTSUBSCRIBED")
|
||||||
return
|
return
|
||||||
del self.subscriptions[i]
|
del self.subscriptions[i]
|
||||||
|
|
@ -97,8 +113,13 @@ class Relay(Protocol):
|
||||||
|
|
||||||
def handleHello(self, parsed):
|
def handleHello(self, parsed):
|
||||||
if parsed["key"] in main.tokens.keys():
|
if parsed["key"] in main.tokens.keys():
|
||||||
if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay":
|
if (
|
||||||
self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]})
|
parsed["hello"] == main.tokens[parsed["key"]]["hello"]
|
||||||
|
and main.tokens[parsed["key"]]["usage"] == "relay"
|
||||||
|
):
|
||||||
|
self.sendMsg(
|
||||||
|
{"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}
|
||||||
|
)
|
||||||
self.authed = True
|
self.authed = True
|
||||||
else:
|
else:
|
||||||
self.transport.loseConnection()
|
self.transport.loseConnection()
|
||||||
|
|
@ -113,12 +134,16 @@ class Relay(Protocol):
|
||||||
|
|
||||||
def connectionLost(self, reason):
|
def connectionLost(self, reason):
|
||||||
self.authed = False
|
self.authed = False
|
||||||
log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
|
log(
|
||||||
|
"Relay connection lost from %s:%s -- %s"
|
||||||
|
% (self.addr.host, self.addr.port, reason.getErrorMessage())
|
||||||
|
)
|
||||||
if self.addr in main.relayConnections.keys():
|
if self.addr in main.relayConnections.keys():
|
||||||
del main.relayConnections[self.addr]
|
del main.relayConnections[self.addr]
|
||||||
else:
|
else:
|
||||||
warn("Tried to remove a non-existant relay connection.")
|
warn("Tried to remove a non-existant relay connection.")
|
||||||
|
|
||||||
|
|
||||||
class RelayFactory(Factory):
|
class RelayFactory(Factory):
|
||||||
def buildProtocol(self, addr):
|
def buildProtocol(self, addr):
|
||||||
entry = Relay(addr)
|
entry = Relay(addr)
|
||||||
|
|
@ -132,10 +157,9 @@ class RelayFactory(Factory):
|
||||||
else:
|
else:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def sendRelayNotification(cast):
|
def sendRelayNotification(cast):
|
||||||
for i in main.relayConnections.keys():
|
for i in main.relayConnections.keys():
|
||||||
if main.relayConnections[i].authed:
|
if main.relayConnections[i].authed:
|
||||||
if cast["type"] in main.relayConnections[i].subscriptions:
|
if cast["type"] in main.relayConnections[i].subscriptions:
|
||||||
newCast = deepcopy(cast)
|
main.relayConnections[i].send(dumps(cast))
|
||||||
newCast["time"] = str(datetime.now().isoformat())
|
|
||||||
main.relayConnections[i].send(dumps(newCast))
|
|
||||||
|
|
@ -1,14 +1,14 @@
|
||||||
from twisted.internet.protocol import Protocol, Factory, ClientFactory
|
|
||||||
import main
|
import main
|
||||||
from utils.logging.log import *
|
|
||||||
|
|
||||||
from core.parser import parseCommand
|
from core.parser import parseCommand
|
||||||
|
from twisted.internet.protocol import Factory, Protocol
|
||||||
|
from utils.logging.log import log, warn
|
||||||
|
|
||||||
|
|
||||||
class Server(Protocol):
|
class Server(Protocol):
|
||||||
def __init__(self, addr):
|
def __init__(self, addr):
|
||||||
self.addr = addr
|
self.addr = addr
|
||||||
self.authed = False
|
self.authed = False
|
||||||
if main.config["UsePassword"] == False:
|
if main.config["UsePassword"] is False:
|
||||||
self.authed = True
|
self.authed = True
|
||||||
|
|
||||||
def send(self, data):
|
def send(self, data):
|
||||||
|
|
@ -33,13 +33,15 @@ class Server(Protocol):
|
||||||
|
|
||||||
def connectionLost(self, reason):
|
def connectionLost(self, reason):
|
||||||
self.authed = False
|
self.authed = False
|
||||||
log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage()))
|
log(
|
||||||
|
"Connection lost from %s:%s -- %s"
|
||||||
|
% (self.addr.host, self.addr.port, reason.getErrorMessage())
|
||||||
|
)
|
||||||
if self.addr in main.connections.keys():
|
if self.addr in main.connections.keys():
|
||||||
del main.connections[self.addr]
|
del main.connections[self.addr]
|
||||||
else:
|
else:
|
||||||
warn("Tried to remove a non-existant connection.")
|
warn("Tried to remove a non-existant connection.")
|
||||||
if self.addr in main.MonitorPool:
|
|
||||||
main.MonitorPool.remove(self.addr)
|
|
||||||
|
|
||||||
class ServerFactory(Factory):
|
class ServerFactory(Factory):
|
||||||
def buildProtocol(self, addr):
|
def buildProtocol(self, addr):
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
version: "2"
|
||||||
|
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: pathogen/threshold:latest
|
||||||
|
build: ./docker
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}:/code
|
||||||
|
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
|
||||||
|
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
|
||||||
|
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
|
||||||
|
ports:
|
||||||
|
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
|
||||||
|
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
|
||||||
|
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
# for development
|
||||||
|
extra_hosts:
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
|
||||||
|
tmp:
|
||||||
|
image: busybox
|
||||||
|
command: chmod -R 777 /var/run/redis
|
||||||
|
volumes:
|
||||||
|
- /var/run/redis
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis
|
||||||
|
command: redis-server /etc/redis.conf
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
external:
|
||||||
|
name: pathogen
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM python:3
|
||||||
|
|
||||||
|
RUN useradd -d /code pathogen
|
||||||
|
RUN mkdir /code
|
||||||
|
RUN chown pathogen:pathogen /code
|
||||||
|
|
||||||
|
RUN mkdir /venv
|
||||||
|
RUN chown pathogen:pathogen /venv
|
||||||
|
|
||||||
|
USER pathogen
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
WORKDIR /code/legacy
|
||||||
|
COPY requirements.prod.txt /code/legacy
|
||||||
|
RUN python -m venv /venv
|
||||||
|
RUN . /venv/bin/activate && pip install -r /code/legacy/requirements.prod.txt
|
||||||
|
CMD . /venv/bin/activate && exec python /code/legacy/threshold
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
version: "2"
|
||||||
|
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: pathogen/threshold:latest
|
||||||
|
build: ./docker
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}:/code
|
||||||
|
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
|
||||||
|
#- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
|
||||||
|
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
|
||||||
|
ports:
|
||||||
|
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
|
||||||
|
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"
|
||||||
|
- "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}"
|
||||||
|
env_file:
|
||||||
|
- ../stack.env
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
|
||||||
|
tmp:
|
||||||
|
image: busybox
|
||||||
|
command: chmod -R 777 /var/run/redis
|
||||||
|
volumes:
|
||||||
|
- /var/run/redis
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis
|
||||||
|
command: redis-server /etc/redis.conf
|
||||||
|
volumes:
|
||||||
|
- ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
|
||||||
|
volumes_from:
|
||||||
|
- tmp
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
external:
|
||||||
|
name: pathogen
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
wheel
|
||||||
|
twisted
|
||||||
|
pyOpenSSL
|
||||||
|
redis
|
||||||
|
pyYaML
|
||||||
|
service_identity
|
||||||
|
siphashc
|
||||||
|
Klein
|
||||||
|
|
@ -0,0 +1,131 @@
|
||||||
|
import json
|
||||||
|
import pickle
|
||||||
|
from os import urandom
|
||||||
|
from os.path import exists
|
||||||
|
from string import digits
|
||||||
|
|
||||||
|
from redis import StrictRedis
|
||||||
|
|
||||||
|
# List of errors ZNC can give us
|
||||||
|
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
|
||||||
|
|
||||||
|
configPath = "conf/live/"
|
||||||
|
templateConfigPath = "conf/templates/"
|
||||||
|
certPath = "conf/cert/"
|
||||||
|
|
||||||
|
filemap = {
|
||||||
|
# JSON configs
|
||||||
|
"config": ["config.json", "configuration", "json"],
|
||||||
|
"help": ["help.json", "command help", "json"],
|
||||||
|
"counters": ["counters.json", "counters file", "json"],
|
||||||
|
"tokens": ["tokens.json", "authentication tokens", "json"],
|
||||||
|
"aliasdata": ["aliasdata.json", "data for alias generation", "json"],
|
||||||
|
"alias": ["alias.json", "provisioned alias data", "json"],
|
||||||
|
"irc": ["irc.json", "IRC network definitions", "json"],
|
||||||
|
"blacklist": ["blacklist.json", "IRC channel blacklist", "json"],
|
||||||
|
# Binary (pickle) configs
|
||||||
|
"network": ["network.dat", "network list", "pickle"],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Connections to the plain-text interface
|
||||||
|
connections = {}
|
||||||
|
# Connections to the JSON interface
|
||||||
|
relayConnections = {}
|
||||||
|
|
||||||
|
# Mapping of network names to Protocol (IRCClient) instances
|
||||||
|
IRCPool = {}
|
||||||
|
|
||||||
|
# Mapping of network names to Reactor instances
|
||||||
|
# Needed for calling .disconnect()
|
||||||
|
ReactorPool = {}
|
||||||
|
|
||||||
|
# Mapping of network names to Factory instances
|
||||||
|
# Needed for calling .stopTrying()
|
||||||
|
FactoryPool = {}
|
||||||
|
|
||||||
|
# Temporary store for channels allocated after a LIST
|
||||||
|
# Will get purged as the instances fire up and pop() from
|
||||||
|
# their respective keys in here
|
||||||
|
TempChan = {}
|
||||||
|
|
||||||
|
# Mapping of command names to their functions
|
||||||
|
CommandMap = {}
|
||||||
|
|
||||||
|
# Incremented by 1 for each event reaching modules.counters.event()
|
||||||
|
# and cloned into lastMinuteSample every minute
|
||||||
|
runningSample = 0
|
||||||
|
lastMinuteSample = 0
|
||||||
|
|
||||||
|
# Generate 16-byte hex key for message checksums
|
||||||
|
hashKey = urandom(16)
|
||||||
|
lastEvents = {}
|
||||||
|
|
||||||
|
|
||||||
|
# Get networks that are currently online and dedupliate
|
||||||
|
def liveNets():
|
||||||
|
networks = set()
|
||||||
|
for i in IRCPool.keys():
|
||||||
|
networks.add("".join([x for x in i if x not in digits]))
|
||||||
|
return networks
|
||||||
|
|
||||||
|
|
||||||
|
def saveConf(var):
|
||||||
|
if var in ("help", "aliasdata"):
|
||||||
|
return # no need to save this
|
||||||
|
if filemap[var][2] == "json":
|
||||||
|
with open(configPath + filemap[var][0], "w") as f:
|
||||||
|
json.dump(globals()[var], f, indent=4)
|
||||||
|
elif filemap[var][2] == "pickle":
|
||||||
|
with open(configPath + filemap[var][0], "wb") as f:
|
||||||
|
pickle.dump(globals()[var], f)
|
||||||
|
else:
|
||||||
|
raise Exception("invalid format")
|
||||||
|
|
||||||
|
|
||||||
|
def loadConf(var):
|
||||||
|
if filemap[var][2] == "json":
|
||||||
|
filename = configPath + filemap[var][0]
|
||||||
|
# Only take the help from the templates
|
||||||
|
if var in ("help", "aliasdata"):
|
||||||
|
filename = templateConfigPath + filemap[var][0]
|
||||||
|
if not exists(filename):
|
||||||
|
# Load the template config
|
||||||
|
if var == "config":
|
||||||
|
filename = templateConfigPath + filemap[var][0]
|
||||||
|
else:
|
||||||
|
# Everything else should be blank
|
||||||
|
globals()[var] = {}
|
||||||
|
return
|
||||||
|
with open(filename, "r") as f:
|
||||||
|
globals()[var] = json.load(f)
|
||||||
|
if var == "alias":
|
||||||
|
# This is a workaround to convert all the keys into integers since JSON
|
||||||
|
# turns them into strings...
|
||||||
|
# Dammit Jason!
|
||||||
|
global alias
|
||||||
|
alias = {int(x): y for x, y in alias.items()}
|
||||||
|
|
||||||
|
elif filemap[var][2] == "pickle":
|
||||||
|
try:
|
||||||
|
with open(configPath + filemap[var][0], "rb") as f:
|
||||||
|
globals()[var] = pickle.load(f)
|
||||||
|
except FileNotFoundError:
|
||||||
|
globals()[var] = {}
|
||||||
|
else:
|
||||||
|
raise Exception("invalid format")
|
||||||
|
|
||||||
|
|
||||||
|
def initConf():
|
||||||
|
for i in filemap.keys():
|
||||||
|
loadConf(i)
|
||||||
|
|
||||||
|
|
||||||
|
def initMain():
|
||||||
|
global r, g
|
||||||
|
initConf()
|
||||||
|
r = StrictRedis(
|
||||||
|
unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] # noqa
|
||||||
|
) # Ephemeral - flushed on quit
|
||||||
|
g = StrictRedis(
|
||||||
|
unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]
|
||||||
|
) # noqa
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import main
|
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
import main
|
||||||
|
|
||||||
|
|
||||||
def generate_password():
|
def generate_password():
|
||||||
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
|
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
|
||||||
|
|
||||||
|
|
||||||
def generate_alias():
|
def generate_alias():
|
||||||
nick = random.choice(main.aliasdata["stubs"])
|
nick = random.choice(main.aliasdata["stubs"])
|
||||||
rand = random.randint(1, 2)
|
rand = random.randint(1, 2)
|
||||||
|
|
@ -13,7 +16,7 @@ def generate_alias():
|
||||||
rand = random.randint(1, 4)
|
rand = random.randint(1, 4)
|
||||||
while rand == 1:
|
while rand == 1:
|
||||||
split = random.randint(0, len(nick) - 1)
|
split = random.randint(0, len(nick) - 1)
|
||||||
nick = nick[:split] + nick[split+1:]
|
nick = nick[:split] + nick[split + 1 :] # noqa: E203
|
||||||
rand = random.randint(1, 4)
|
rand = random.randint(1, 4)
|
||||||
rand = random.randint(1, 3)
|
rand = random.randint(1, 3)
|
||||||
if rand == 1 or rand == 4:
|
if rand == 1 or rand == 4:
|
||||||
|
|
@ -51,7 +54,7 @@ def generate_alias():
|
||||||
ident = namebase.split(" ")[0]
|
ident = namebase.split(" ")[0]
|
||||||
ident = ident[:10]
|
ident = ident[:10]
|
||||||
elif rand == 6:
|
elif rand == 6:
|
||||||
ident = re.sub("\s", "", namebase).lower()
|
ident = re.sub("\s", "", namebase).lower() # noqa: W605
|
||||||
ident = ident[:10]
|
ident = ident[:10]
|
||||||
|
|
||||||
realname = nick
|
realname = nick
|
||||||
|
|
@ -63,6 +66,10 @@ def generate_alias():
|
||||||
if rand == 3 or rand == 4:
|
if rand == 3 or rand == 4:
|
||||||
realname = realname.capitalize()
|
realname = realname.capitalize()
|
||||||
|
|
||||||
password = generate_password()
|
return {
|
||||||
|
"nick": nick,
|
||||||
return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password}
|
"altnick": altnick,
|
||||||
|
"ident": ident,
|
||||||
|
"realname": realname,
|
||||||
|
"emails": [],
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,600 @@
|
||||||
|
from copy import deepcopy
|
||||||
|
from math import ceil
|
||||||
|
|
||||||
|
import main
|
||||||
|
from modules import helpers
|
||||||
|
from twisted.internet.threads import deferToThread
|
||||||
|
from utils.logging.debug import debug, trace
|
||||||
|
from utils.logging.log import error, log, warn
|
||||||
|
|
||||||
|
|
||||||
|
def getAllChannels(net=None):
|
||||||
|
"""
|
||||||
|
Get a list of all channels on all relays.
|
||||||
|
:return: list of channels
|
||||||
|
"""
|
||||||
|
channels = {}
|
||||||
|
if not net:
|
||||||
|
nets = main.network.keys()
|
||||||
|
else:
|
||||||
|
nets = [net]
|
||||||
|
for net in nets:
|
||||||
|
relays = helpers.get_connected_relays(net)
|
||||||
|
for relay in relays:
|
||||||
|
if net not in channels:
|
||||||
|
channels[net] = {}
|
||||||
|
if relay.num not in channels[net]:
|
||||||
|
channels[net][relay.num] = []
|
||||||
|
for channel in relay.channels:
|
||||||
|
channels[net][relay.num].append(channel)
|
||||||
|
# debug(f"getAllChannels(): {channels}")
|
||||||
|
return channels
|
||||||
|
|
||||||
|
|
||||||
|
def getDuplicateChannels(net=None, total=False):
|
||||||
|
"""
|
||||||
|
Get a list of duplicate channels.
|
||||||
|
:return: list of duplicate channels
|
||||||
|
"""
|
||||||
|
allChans = getAllChannels(net)
|
||||||
|
duplicates = {}
|
||||||
|
for net in allChans.keys():
|
||||||
|
net_chans = []
|
||||||
|
inst = {}
|
||||||
|
# add all the channels from this network to a list
|
||||||
|
for num in allChans[net].keys():
|
||||||
|
net_chans.extend(allChans[net][num])
|
||||||
|
for channel in net_chans:
|
||||||
|
count_chan = net_chans.count(channel)
|
||||||
|
# I don't know why but it works
|
||||||
|
# this is used in userinfo.delChannels
|
||||||
|
set_min = 1
|
||||||
|
if total:
|
||||||
|
set_min = 0
|
||||||
|
if count_chan > set_min:
|
||||||
|
inst[channel] = count_chan
|
||||||
|
if inst:
|
||||||
|
duplicates[net] = inst
|
||||||
|
|
||||||
|
if total:
|
||||||
|
return duplicates
|
||||||
|
|
||||||
|
to_part = {}
|
||||||
|
for net in allChans:
|
||||||
|
if net in duplicates:
|
||||||
|
for num in allChans[net].keys():
|
||||||
|
for channel in allChans[net][num]:
|
||||||
|
if channel in duplicates[net].keys():
|
||||||
|
if duplicates[net][channel] > 1:
|
||||||
|
if net not in to_part:
|
||||||
|
to_part[net] = {}
|
||||||
|
if num not in to_part[net]:
|
||||||
|
to_part[net][num] = []
|
||||||
|
to_part[net][num].append(channel)
|
||||||
|
duplicates[net][channel] -= 1
|
||||||
|
|
||||||
|
return to_part
|
||||||
|
|
||||||
|
|
||||||
|
def partChannels(data):
|
||||||
|
for net in data:
|
||||||
|
for num in data[net]:
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
for channel in data[net][num]:
|
||||||
|
if channel in main.IRCPool[name].channels:
|
||||||
|
main.IRCPool[name].part(channel)
|
||||||
|
log(f"Parted {channel} on {net} - {num}")
|
||||||
|
|
||||||
|
|
||||||
|
def getEnabledRelays(net):
|
||||||
|
"""
|
||||||
|
Get a list of enabled relays for a network.
|
||||||
|
:param net: network
|
||||||
|
:rtype: list of int
|
||||||
|
:return: list of enabled relay numbers
|
||||||
|
"""
|
||||||
|
enabledRelays = [
|
||||||
|
x
|
||||||
|
for x in main.network[net].relays.keys()
|
||||||
|
if main.network[net].relays[x]["enabled"]
|
||||||
|
]
|
||||||
|
# debug(f"getEnabledRelays() {net}: {enabledRelays}")
|
||||||
|
return enabledRelays
|
||||||
|
|
||||||
|
|
||||||
|
def getConnectedRelays(net):
|
||||||
|
"""
|
||||||
|
Get a list of connected relays for a network.
|
||||||
|
:param net: network
|
||||||
|
:rtype: list of int
|
||||||
|
:return: list of relay numbers
|
||||||
|
"""
|
||||||
|
enabledRelays = getEnabledRelays(net)
|
||||||
|
connectedRelays = []
|
||||||
|
for i in enabledRelays:
|
||||||
|
name = net + str(i)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
if main.IRCPool[name].isconnected:
|
||||||
|
connectedRelays.append(i)
|
||||||
|
# debug(f"getConnectedRelays() {net}: {connectedRelays}")
|
||||||
|
return connectedRelays
|
||||||
|
|
||||||
|
|
||||||
|
def getActiveRelays(net):
|
||||||
|
"""
|
||||||
|
Get a list of active relays for a network.
|
||||||
|
:param net: network
|
||||||
|
:rtype: list of int
|
||||||
|
:return: list of relay numbers
|
||||||
|
"""
|
||||||
|
enabledRelays = getEnabledRelays(net)
|
||||||
|
activeRelays = []
|
||||||
|
for i in enabledRelays:
|
||||||
|
name = net + str(i)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
# debug(
|
||||||
|
# (
|
||||||
|
# f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
|
||||||
|
# f"conn:{main.IRCPool[name].isconnected}"
|
||||||
|
# )
|
||||||
|
# )
|
||||||
|
if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
|
||||||
|
activeRelays.append(i)
|
||||||
|
debug(f"getActiveRelays() {net}: {activeRelays}")
|
||||||
|
return activeRelays
|
||||||
|
|
||||||
|
|
||||||
|
def relayIsActive(net, num):
|
||||||
|
"""
|
||||||
|
Check if a relay is active.
|
||||||
|
:param net: network
|
||||||
|
:param num: relay number
|
||||||
|
:rtype: bool
|
||||||
|
:return: True if relay is active, False otherwise
|
||||||
|
"""
|
||||||
|
activeRelays = getActiveRelays(net)
|
||||||
|
return num in activeRelays
|
||||||
|
|
||||||
|
|
||||||
|
def allRelaysActive(net):
|
||||||
|
"""
|
||||||
|
Check if all enabled relays are active and authenticated.
|
||||||
|
:param net: network
|
||||||
|
:rtype: bool
|
||||||
|
:return: True if all relays are active and authenticated, False otherwise
|
||||||
|
"""
|
||||||
|
activeRelays = getActiveRelays(net)
|
||||||
|
enabledRelays = getEnabledRelays(net)
|
||||||
|
relaysActive = len(activeRelays) == len(enabledRelays)
|
||||||
|
# debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
|
||||||
|
return relaysActive
|
||||||
|
|
||||||
|
|
||||||
|
def getAverageChanlimit(net):
|
||||||
|
"""
|
||||||
|
Get the average channel limit for a network.
|
||||||
|
:param net: network
|
||||||
|
:rtype: int
|
||||||
|
:return: average channel limit
|
||||||
|
"""
|
||||||
|
total = 0
|
||||||
|
for i in getActiveRelays(net):
|
||||||
|
name = net + str(i)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
total += main.IRCPool[name].chanlimit
|
||||||
|
avg_chanlimit = total / len(getActiveRelays(net))
|
||||||
|
debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
|
||||||
|
return avg_chanlimit
|
||||||
|
|
||||||
|
|
||||||
|
def getSumChanlimit(net):
|
||||||
|
"""
|
||||||
|
Get the sum of all channel limits for a network.
|
||||||
|
:param net: network
|
||||||
|
:rtype: int
|
||||||
|
:return: sum of channel limits
|
||||||
|
"""
|
||||||
|
total = 0
|
||||||
|
for i in getActiveRelays(net):
|
||||||
|
name = net + str(i)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
total += main.IRCPool[name].chanlimit
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def getChanFree(net):
|
||||||
|
"""
|
||||||
|
Get a dictionary with the free channel spaces for
|
||||||
|
each relay, and a channel limit.
|
||||||
|
Example return:
|
||||||
|
({1: 99}, 100)
|
||||||
|
:param net: network
|
||||||
|
:return: ({relay: channel spaces}, channel limit)
|
||||||
|
"""
|
||||||
|
chanfree = {}
|
||||||
|
for i in getActiveRelays(net):
|
||||||
|
name = net + str(i)
|
||||||
|
if name not in main.IRCPool.keys():
|
||||||
|
continue
|
||||||
|
if not main.IRCPool[name].isconnected:
|
||||||
|
continue
|
||||||
|
chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
|
||||||
|
|
||||||
|
return chanfree
|
||||||
|
|
||||||
|
|
||||||
|
def getTotalChans(net):
|
||||||
|
"""
|
||||||
|
Get the total number of channels on all relays for a network.
|
||||||
|
:param net: network
|
||||||
|
:rtype: int
|
||||||
|
:return: total number of channels
|
||||||
|
"""
|
||||||
|
total = 0
|
||||||
|
for i in getActiveRelays(net):
|
||||||
|
name = net + str(i)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
total += len(main.IRCPool[name].channels)
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def emptyChanAllocate(net, flist):
|
||||||
|
"""
|
||||||
|
Allocate channels to relays.
|
||||||
|
:param net: network
|
||||||
|
:param flist: list of channels to allocate
|
||||||
|
:param new: list of newly provisioned relays to account for
|
||||||
|
:rtype: dict
|
||||||
|
:return: dictionary of {relay: list of channels}"""
|
||||||
|
|
||||||
|
# Get the free channel spaces for each relay
|
||||||
|
chanfree = getChanFree(net)
|
||||||
|
if not chanfree:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Pretend the newly provisioned relays are already on the network
|
||||||
|
# for i in new:
|
||||||
|
# chanfree[0][i] = chanfree[1]
|
||||||
|
allocated = {}
|
||||||
|
|
||||||
|
newlist = list(flist)
|
||||||
|
chan_slots_used = getTotalChans(net)
|
||||||
|
max_chans = getSumChanlimit(net) - chan_slots_used
|
||||||
|
trunc_list = newlist[:max_chans]
|
||||||
|
debug(
|
||||||
|
f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for i in chanfree.keys():
|
||||||
|
for x in range(chanfree[i]):
|
||||||
|
if not len(trunc_list):
|
||||||
|
break
|
||||||
|
if i in allocated.keys():
|
||||||
|
allocated[i].append(trunc_list.pop())
|
||||||
|
else:
|
||||||
|
allocated[i] = [trunc_list.pop()]
|
||||||
|
return allocated
|
||||||
|
|
||||||
|
|
||||||
|
def populateChans(net, clist):
|
||||||
|
"""
|
||||||
|
Populate channels on relays.
|
||||||
|
Stores channels to join in a list in main.TempChan[net][num]
|
||||||
|
:param net: network
|
||||||
|
:param clist: list of channels to join
|
||||||
|
:param new: list of newly provisioned relays to account for"""
|
||||||
|
# divided = array_split(clist, relay)
|
||||||
|
allocated = emptyChanAllocate(net, clist)
|
||||||
|
trace(f"populateChans() allocated:{allocated}")
|
||||||
|
if not allocated:
|
||||||
|
return
|
||||||
|
for i in allocated.keys():
|
||||||
|
if net in main.TempChan.keys():
|
||||||
|
main.TempChan[net][i] = allocated[i]
|
||||||
|
else:
|
||||||
|
main.TempChan[net] = {i: allocated[i]}
|
||||||
|
trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
|
||||||
|
|
||||||
|
|
||||||
|
def notifyJoin(net):
|
||||||
|
"""
|
||||||
|
Notify relays to join channels.
|
||||||
|
They will pull from main.TempChan and remove channels they join.
|
||||||
|
:param net: network
|
||||||
|
"""
|
||||||
|
for i in getActiveRelays(net):
|
||||||
|
name = net + str(i)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
trace(f"notifyJoin() {name}")
|
||||||
|
main.IRCPool[name].checkChannels()
|
||||||
|
|
||||||
|
|
||||||
|
def minifyChans(net, listinfo, as_list=False):
|
||||||
|
"""
|
||||||
|
Remove channels from listinfo that are already covered by a relay.
|
||||||
|
:param net: network
|
||||||
|
:param listinfo: list of channels to check
|
||||||
|
:type listinfo: list of [channel, num_users]
|
||||||
|
:return: list of channels with joined channels removed
|
||||||
|
:rtype: list of [channel, num_users]
|
||||||
|
"""
|
||||||
|
# We want to make this reusable for joining a bunch of channels.
|
||||||
|
if as_list:
|
||||||
|
channel_list = listinfo
|
||||||
|
|
||||||
|
if not allRelaysActive(net):
|
||||||
|
error("All relays for %s are not active, cannot minify list" % net)
|
||||||
|
return False
|
||||||
|
for i in getConnectedRelays(net):
|
||||||
|
name = net + str(i)
|
||||||
|
for x in main.IRCPool[name].channels:
|
||||||
|
if as_list:
|
||||||
|
for y in channel_list:
|
||||||
|
if y == x:
|
||||||
|
channel_list.remove(y)
|
||||||
|
else:
|
||||||
|
for y in listinfo:
|
||||||
|
if y[0] == x:
|
||||||
|
listinfo.remove(y)
|
||||||
|
if not as_list:
|
||||||
|
if not listinfo:
|
||||||
|
log("We're on all the channels we want to be on, dropping LIST")
|
||||||
|
return False
|
||||||
|
if as_list:
|
||||||
|
return channel_list
|
||||||
|
else:
|
||||||
|
return listinfo
|
||||||
|
|
||||||
|
|
||||||
|
def keepChannels(net, listinfo, mean, sigrelay, relay):
|
||||||
|
"""
|
||||||
|
Minify channels, determine whether we can cover all the channels
|
||||||
|
on the network, or need to use 'significant' mode.
|
||||||
|
Truncate the channel list to available channel spaces.
|
||||||
|
Allocate these channels to relays.
|
||||||
|
Notify relays that they should pull from TempChan to join.
|
||||||
|
:param net: network
|
||||||
|
:param listinfo: list of [channel, num_users] lists
|
||||||
|
:param mean: mean of channel population
|
||||||
|
:param sigrelay: number of relays needed to cover significant channels
|
||||||
|
:param relay: number of relays needed to cover all channels
|
||||||
|
:param chanlimit: maximum number of channels to allocate to a relay
|
||||||
|
"""
|
||||||
|
listinfo = minifyChans(net, listinfo)
|
||||||
|
if not listinfo:
|
||||||
|
return
|
||||||
|
if (
|
||||||
|
relay <= main.config["ChanKeep"]["SigSwitch"]
|
||||||
|
): # we can cover all of the channels
|
||||||
|
coverAll = True
|
||||||
|
elif (
|
||||||
|
relay > main.config["ChanKeep"]["SigSwitch"]
|
||||||
|
): # we cannot cover all of the channels
|
||||||
|
coverAll = False
|
||||||
|
# if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
|
||||||
|
# error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
|
||||||
|
# return
|
||||||
|
num_instances = len(getActiveRelays(net))
|
||||||
|
debug(f"keepChannels() {net} instances:{num_instances}")
|
||||||
|
chan_slots_used = getTotalChans(net)
|
||||||
|
debug(f"keepChannels() slots_used:{chan_slots_used}")
|
||||||
|
# max_chans = (chanlimit * num_instances) - chan_slots_used
|
||||||
|
max_chans = getSumChanlimit(net) - chan_slots_used
|
||||||
|
if max_chans < 0:
|
||||||
|
max_chans = 0
|
||||||
|
debug(f"keepChannels() max_chans:{max_chans}")
|
||||||
|
if coverAll:
|
||||||
|
# needed = relay - len(getActiveRelays(net))
|
||||||
|
# if needed:
|
||||||
|
# debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
|
||||||
|
# newNums = modules.provision.provisionMultipleRelays(net, needed)
|
||||||
|
# else:
|
||||||
|
# newNums = []
|
||||||
|
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
|
||||||
|
if len(listinfo_sort) > max_chans:
|
||||||
|
max_chans = len(listinfo_sort) - 1
|
||||||
|
|
||||||
|
flist = [i[0] for i in listinfo_sort]
|
||||||
|
|
||||||
|
flist = flist[:max_chans]
|
||||||
|
debug(
|
||||||
|
f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels"
|
||||||
|
)
|
||||||
|
trace(f"keepChannels() {net}: joining:{flist}")
|
||||||
|
populateChans(net, flist)
|
||||||
|
else:
|
||||||
|
# needed = sigrelay - len(getActiveRelays(net))
|
||||||
|
# if needed:
|
||||||
|
# debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
|
||||||
|
# newNums = modules.provision.provisionMultipleRelays(net, needed)
|
||||||
|
# else:
|
||||||
|
# newNums = []
|
||||||
|
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
|
||||||
|
trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
|
||||||
|
if len(listinfo_sort) > max_chans:
|
||||||
|
max_chans = len(listinfo_sort) - 1
|
||||||
|
debug(f"keepChannels() {net}: new max_chans:{max_chans}")
|
||||||
|
|
||||||
|
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
|
||||||
|
trace(f"keepChannels() {net}: new siglist:{siglist}")
|
||||||
|
|
||||||
|
siglist = siglist[:max_chans]
|
||||||
|
trace(f"keepChannels() {net}: truncated siglist:{siglist}")
|
||||||
|
|
||||||
|
trace(
|
||||||
|
f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}"
|
||||||
|
)
|
||||||
|
debug(
|
||||||
|
f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels"
|
||||||
|
)
|
||||||
|
trace(f"keepChannels() {net}: joining:{siglist}")
|
||||||
|
populateChans(net, siglist)
|
||||||
|
notifyJoin(net)
|
||||||
|
|
||||||
|
|
||||||
|
def joinSingle(net, channel):
|
||||||
|
"""
|
||||||
|
Join a channel on a relay.
|
||||||
|
Use ECA to determine which relay to join on.
|
||||||
|
:param net: network
|
||||||
|
:param channel: channel to join
|
||||||
|
:return: relay number that joined the channel
|
||||||
|
:rtype: int
|
||||||
|
"""
|
||||||
|
if "," in channel:
|
||||||
|
channels = channel.split(",")
|
||||||
|
channels = minifyChans(net, channels, as_list=True)
|
||||||
|
|
||||||
|
else:
|
||||||
|
channels = [channel]
|
||||||
|
populateChans(net, channels)
|
||||||
|
notifyJoin(net)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def partSingle(net, channel):
|
||||||
|
"""
|
||||||
|
Iterate over all the relays of net and part channels matching channel.
|
||||||
|
:param net: network
|
||||||
|
:param channel: channel to part
|
||||||
|
:return: list of relays that parted the channel
|
||||||
|
:rtype: list of str
|
||||||
|
"""
|
||||||
|
parted = []
|
||||||
|
for i in getConnectedRelays(net):
|
||||||
|
name = f"{net}{i}"
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
if channel in main.IRCPool[name].channels:
|
||||||
|
main.IRCPool[name].part(channel)
|
||||||
|
parted.append(str(i))
|
||||||
|
return parted
|
||||||
|
|
||||||
|
|
||||||
|
def nukeNetwork(net):
|
||||||
|
"""
|
||||||
|
Remove network records.
|
||||||
|
:param net: network"""
|
||||||
|
# purgeRecords(net)
|
||||||
|
# p = main.g.pipeline()
|
||||||
|
main.g.delete("analytics.list." + net)
|
||||||
|
# p.delete("list."+net)
|
||||||
|
# p.execute()
|
||||||
|
|
||||||
|
|
||||||
|
# def nukeNetwork(net):
|
||||||
|
# deferToThread(_nukeNetwork, net)
|
||||||
|
|
||||||
|
|
||||||
|
def _initialList(net, num, listinfo):
|
||||||
|
"""
|
||||||
|
Called when a relay receives a full LIST response.
|
||||||
|
Run statistics to determine how many channels are significant.
|
||||||
|
This is done by adding all the numbers of users on the channels together,
|
||||||
|
then dividing by the number of channels.
|
||||||
|
* cumul - cumulative sum of all channel membership
|
||||||
|
* siglength - number of significant channels
|
||||||
|
* listlength - number of channels in the list
|
||||||
|
* sigrelay - number of relays needed to cover siglength
|
||||||
|
* relay - number of relays needed to cover all channels
|
||||||
|
:param net: network
|
||||||
|
:param num: relay number
|
||||||
|
:param listinfo: list of [channel, num_users] lists
|
||||||
|
:param chanlimit: maximum number of channels the relay can join
|
||||||
|
"""
|
||||||
|
listlength = len(listinfo)
|
||||||
|
cumul = 0
|
||||||
|
try:
|
||||||
|
cumul += sum(int(i[1]) for i in listinfo)
|
||||||
|
except TypeError:
|
||||||
|
warn("Bad LIST data received from %s - %i" % (net, num))
|
||||||
|
return
|
||||||
|
mean = round(cumul / listlength, 2)
|
||||||
|
siglength = 0
|
||||||
|
insiglength = 0
|
||||||
|
sigcumul = 0
|
||||||
|
insigcumul = 0
|
||||||
|
for i in listinfo:
|
||||||
|
if int(i[1]) > mean:
|
||||||
|
siglength += 1
|
||||||
|
sigcumul += int(i[1])
|
||||||
|
elif int(i[1]) < mean:
|
||||||
|
insiglength += 1
|
||||||
|
insigcumul += int(i[1])
|
||||||
|
|
||||||
|
avg_chanlimit = getAverageChanlimit(net)
|
||||||
|
sigrelay = ceil(siglength / avg_chanlimit)
|
||||||
|
relay = ceil(listlength / avg_chanlimit)
|
||||||
|
|
||||||
|
cur_relays = len(getActiveRelays(net))
|
||||||
|
sig_relays_missing = sigrelay - cur_relays
|
||||||
|
all_relays_missing = relay - cur_relays
|
||||||
|
|
||||||
|
abase = "analytics.list.%s" % net
|
||||||
|
main.g.delete(abase)
|
||||||
|
p = main.g.pipeline()
|
||||||
|
|
||||||
|
# See docstring for meanings
|
||||||
|
p.hset(abase, "mean", mean)
|
||||||
|
p.hset(abase, "total_chans", listlength)
|
||||||
|
p.hset(abase, "big_chans", siglength)
|
||||||
|
p.hset(abase, "small_chans", insiglength)
|
||||||
|
p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
|
||||||
|
p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
|
||||||
|
p.hset(abase, "total_cumul_mem", cumul)
|
||||||
|
p.hset(abase, "big_chan_cumul_mem", sigcumul)
|
||||||
|
p.hset(abase, "small_chan_cumul_mem", insigcumul)
|
||||||
|
p.hset(abase, "relays_for_all_chans", relay)
|
||||||
|
p.hset(abase, "relays_for_big_chans", sigrelay)
|
||||||
|
p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
|
||||||
|
p.hset(abase, "sig_relays_missing", sig_relays_missing)
|
||||||
|
p.hset(abase, "all_relays_missing", all_relays_missing)
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"_initialList() net:{net} num:{num} listlength:{listlength} "
|
||||||
|
f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
|
||||||
|
f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Purge existing records before writing
|
||||||
|
# purgeRecords(net)
|
||||||
|
# for i in listinfo:
|
||||||
|
# p.rpush(netbase+"."+i[0], i[1])
|
||||||
|
# p.rpush(netbase+"."+i[0], i[2])
|
||||||
|
# p.sadd(netbase, i[0])
|
||||||
|
|
||||||
|
p.execute()
|
||||||
|
debug("List parsing completed on %s" % net)
|
||||||
|
keepChannels(net, listinfo, mean, sigrelay, relay)
|
||||||
|
|
||||||
|
# return (listinfo, mean, sigrelay, relay)
|
||||||
|
|
||||||
|
|
||||||
|
def convert(data):
|
||||||
|
"""
|
||||||
|
Recursively convert a dictionary.
|
||||||
|
"""
|
||||||
|
if isinstance(data, bytes):
|
||||||
|
return data.decode("ascii")
|
||||||
|
if isinstance(data, dict):
|
||||||
|
return dict(map(convert, data.items()))
|
||||||
|
if isinstance(data, tuple):
|
||||||
|
return map(convert, data)
|
||||||
|
if isinstance(data, list):
|
||||||
|
return list(map(convert, data))
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def getListInfo(net):
|
||||||
|
abase = f"analytics.list.{net}"
|
||||||
|
info = main.g.hgetall(abase)
|
||||||
|
return convert(info)
|
||||||
|
|
||||||
|
|
||||||
|
def initialList(net, num, listinfo):
|
||||||
|
"""
|
||||||
|
Run _initialList in a thread.
|
||||||
|
See above docstring.
|
||||||
|
"""
|
||||||
|
deferToThread(_initialList, net, num, deepcopy(listinfo))
|
||||||
|
|
@ -1,12 +1,13 @@
|
||||||
import main
|
import main
|
||||||
from twisted.internet.task import LoopingCall
|
from twisted.internet.task import LoopingCall
|
||||||
|
|
||||||
|
|
||||||
def event(name, eventType):
|
def event(name, eventType):
|
||||||
if not "local" in main.counters.keys():
|
if "local" not in main.counters.keys():
|
||||||
main.counters["local"] = {}
|
main.counters["local"] = {}
|
||||||
if not "global" in main.counters.keys():
|
if "global" not in main.counters.keys():
|
||||||
main.counters["global"] = {}
|
main.counters["global"] = {}
|
||||||
if not name in main.counters["local"].keys():
|
if name not in main.counters["local"].keys():
|
||||||
main.counters["local"][name] = {}
|
main.counters["local"][name] = {}
|
||||||
if eventType not in main.counters["local"][name].keys():
|
if eventType not in main.counters["local"][name].keys():
|
||||||
main.counters["local"][name][eventType] = 0
|
main.counters["local"][name][eventType] = 0
|
||||||
|
|
@ -18,8 +19,9 @@ def event(name, eventType):
|
||||||
main.counters["global"][eventType] += 1
|
main.counters["global"][eventType] += 1
|
||||||
main.runningSample += 1
|
main.runningSample += 1
|
||||||
|
|
||||||
|
|
||||||
def getEvents(name=None):
|
def getEvents(name=None):
|
||||||
if name == None:
|
if name is None:
|
||||||
if "global" in main.counters.keys():
|
if "global" in main.counters.keys():
|
||||||
return main.counters["global"]
|
return main.counters["global"]
|
||||||
else:
|
else:
|
||||||
|
|
@ -30,10 +32,12 @@ def getEvents(name=None):
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def takeSample():
|
def takeSample():
|
||||||
main.lastMinuteSample = main.runningSample
|
main.lastMinuteSample = main.runningSample
|
||||||
main.runningSample = 0
|
main.runningSample = 0
|
||||||
|
|
||||||
|
|
||||||
def setupCounterLoop():
|
def setupCounterLoop():
|
||||||
lc = LoopingCall(takeSample)
|
lc = LoopingCall(takeSample)
|
||||||
lc.start(60)
|
lc.start(60)
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
import main
|
||||||
|
from modules import chankeep
|
||||||
|
from utils.logging.debug import debug
|
||||||
|
|
||||||
|
|
||||||
|
def get_first_relay(net):
|
||||||
|
"""
|
||||||
|
Get the first relay in the network.
|
||||||
|
:param net: the network
|
||||||
|
:param num: number or relay
|
||||||
|
:return: IRCPool instance for the IRC bot
|
||||||
|
"""
|
||||||
|
cur_relay = 0
|
||||||
|
max_relay = max(main.network[net].relays.keys())
|
||||||
|
debug(f"get_first_relay() {net}: max_relay:{max_relay}")
|
||||||
|
activeRelays = chankeep.getActiveRelays(net)
|
||||||
|
|
||||||
|
debug(f"get_first_relay() {net}: activeRelays:{activeRelays}")
|
||||||
|
while cur_relay != max_relay:
|
||||||
|
cur_relay += 1
|
||||||
|
if cur_relay not in activeRelays:
|
||||||
|
continue
|
||||||
|
name = net + str(cur_relay)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
# debug(f"get_first_relay() {net}: found relay {name}")
|
||||||
|
return main.IRCPool[name]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def is_first_relay(net, num):
|
||||||
|
"""
|
||||||
|
Determine if we are the first relay for the network.
|
||||||
|
:param net: the network
|
||||||
|
:param num: number or relay
|
||||||
|
:return: True if we are the first relay, False otherwise
|
||||||
|
"""
|
||||||
|
first_relay = get_first_relay(net)
|
||||||
|
if not first_relay:
|
||||||
|
return False
|
||||||
|
return first_relay.num == num
|
||||||
|
|
||||||
|
|
||||||
|
def get_active_relays(net):
|
||||||
|
"""
|
||||||
|
Get all active instances for the network.
|
||||||
|
:param net: the network
|
||||||
|
:return: list of active instances
|
||||||
|
:rtype: list of IRCPool instances
|
||||||
|
"""
|
||||||
|
active_nums = chankeep.getActiveRelays(net)
|
||||||
|
active_insts = []
|
||||||
|
for num in active_nums:
|
||||||
|
name = net + str(num)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
active_insts.append(main.IRCPool[name])
|
||||||
|
return active_insts
|
||||||
|
|
||||||
|
|
||||||
|
def get_connected_relays(net):
|
||||||
|
"""
|
||||||
|
Get all connected instances for the network.
|
||||||
|
:param net: the network
|
||||||
|
:return: list of active instances
|
||||||
|
:rtype: list of IRCPool instances
|
||||||
|
"""
|
||||||
|
active_nums = chankeep.getConnectedRelays(net)
|
||||||
|
active_insts = []
|
||||||
|
for num in active_nums:
|
||||||
|
name = net + str(num)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
active_insts.append(main.IRCPool[name])
|
||||||
|
return active_insts
|
||||||
|
|
@ -0,0 +1,91 @@
|
||||||
|
import json
|
||||||
|
|
||||||
|
import main
|
||||||
|
from core.relay import sendRelayNotification
|
||||||
|
from modules import userinfo
|
||||||
|
from utils.dedup import dedup
|
||||||
|
|
||||||
|
order = [
|
||||||
|
"type",
|
||||||
|
"net",
|
||||||
|
"num",
|
||||||
|
"channel",
|
||||||
|
"msg",
|
||||||
|
"nick",
|
||||||
|
"ident",
|
||||||
|
"host",
|
||||||
|
"mtype",
|
||||||
|
"user",
|
||||||
|
"mode",
|
||||||
|
"modearg",
|
||||||
|
"realname",
|
||||||
|
"server",
|
||||||
|
"status",
|
||||||
|
"ts",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def parsemeta(numName, c):
|
||||||
|
if "channel" not in c.keys():
|
||||||
|
c["channel"] = None
|
||||||
|
# metadata scraping
|
||||||
|
# need to check if this was received from a relay
|
||||||
|
# in which case, do not do this
|
||||||
|
if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
|
||||||
|
if "muser" in c.keys():
|
||||||
|
userinfo.editUser(c["net"], c["muser"])
|
||||||
|
# if c["type"] == "mode":
|
||||||
|
# userinfo.updateMode(c)
|
||||||
|
elif c["type"] == "nick":
|
||||||
|
userinfo.renameUser(
|
||||||
|
c["net"],
|
||||||
|
c["nick"],
|
||||||
|
c["muser"],
|
||||||
|
c["user"],
|
||||||
|
c["user"] + "!" + c["ident"] + "@" + c["host"],
|
||||||
|
)
|
||||||
|
elif c["type"] == "kick":
|
||||||
|
userinfo.editUser(c["net"], c["muser"])
|
||||||
|
userinfo.delUserByNick(c["net"], c["channel"], c["user"])
|
||||||
|
elif c["type"] == "quit":
|
||||||
|
userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"])
|
||||||
|
elif c["type"] == "join":
|
||||||
|
userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"])
|
||||||
|
elif c["type"] == "part":
|
||||||
|
userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"])
|
||||||
|
|
||||||
|
if "mtype" in c.keys():
|
||||||
|
if c["mtype"] == "nick":
|
||||||
|
userinfo.renameUser(
|
||||||
|
c["net"],
|
||||||
|
c["nick"],
|
||||||
|
c["muser"],
|
||||||
|
c["user"],
|
||||||
|
c["user"] + "!" + c["ident"] + "@" + c["host"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def queue_message(c):
|
||||||
|
message = json.dumps(c)
|
||||||
|
print("APPENDING MESSAGE", message)
|
||||||
|
print("KEY", main.config["Ingest"]["Key"])
|
||||||
|
main.g.sadd(main.config["Ingest"]["Key"], message)
|
||||||
|
|
||||||
|
|
||||||
|
def event(
|
||||||
|
numName, c
|
||||||
|
): # yes I'm using a short variable because otherwise it goes off the screen
|
||||||
|
if dedup(numName, c):
|
||||||
|
return
|
||||||
|
|
||||||
|
# make a copy of the object with dict() to prevent sending notifications with channel of None
|
||||||
|
parsemeta(numName, dict(c))
|
||||||
|
|
||||||
|
if "muser" in c.keys():
|
||||||
|
del c["muser"]
|
||||||
|
# sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order
|
||||||
|
# sortedKeys["src"] = "irc"
|
||||||
|
c["src"] = "irc"
|
||||||
|
if main.config["Ingest"]["Enabled"]:
|
||||||
|
queue_message(c)
|
||||||
|
sendRelayNotification(c)
|
||||||
|
|
@ -0,0 +1,163 @@
|
||||||
|
from copy import deepcopy
|
||||||
|
|
||||||
|
import main
|
||||||
|
from core.bot import IRCBotFactory
|
||||||
|
from modules import alias
|
||||||
|
from modules.chankeep import nukeNetwork
|
||||||
|
from modules.provision import provisionRelay
|
||||||
|
from modules.regproc import needToRegister
|
||||||
|
from twisted.internet import reactor
|
||||||
|
from twisted.internet.ssl import DefaultOpenSSLContextFactory
|
||||||
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
from utils.get import getRelay
|
||||||
|
from utils.logging.log import log
|
||||||
|
|
||||||
|
|
||||||
|
def migrate():
|
||||||
|
existing = deepcopy(main.network)
|
||||||
|
log("Migrating network configuration")
|
||||||
|
log(f"Existing network configuration: {existing}")
|
||||||
|
for net, net_inst in existing.items():
|
||||||
|
log(f"Migrating network {net}")
|
||||||
|
net = net_inst.net
|
||||||
|
host = net_inst.host
|
||||||
|
port = net_inst.port
|
||||||
|
security = net_inst.security
|
||||||
|
auth = net_inst.auth
|
||||||
|
last = net_inst.last
|
||||||
|
relays = net_inst.relays
|
||||||
|
aliases = net_inst.aliases
|
||||||
|
|
||||||
|
new_net = Network(net, host, port, security, auth)
|
||||||
|
log(f"New network for {net}: {new_net}")
|
||||||
|
new_net.last = last
|
||||||
|
new_net.relays = relays
|
||||||
|
new_net.aliases = aliases
|
||||||
|
main.network[net] = new_net
|
||||||
|
main.saveConf("network")
|
||||||
|
log("Finished migrating network configuration")
|
||||||
|
|
||||||
|
|
||||||
|
class Network:
|
||||||
|
def __init__(self, net, host, port, security, auth):
|
||||||
|
self.net = net
|
||||||
|
self.host = host
|
||||||
|
self.port = port
|
||||||
|
self.security = security
|
||||||
|
self.auth = auth
|
||||||
|
self.chanlimit = None
|
||||||
|
|
||||||
|
self.last = 1
|
||||||
|
self.relays = {}
|
||||||
|
self.aliases = {}
|
||||||
|
|
||||||
|
def add_relay(self, num=None):
|
||||||
|
# Grrrrrrrrrr
|
||||||
|
self.last = int(self.last)
|
||||||
|
if not num:
|
||||||
|
num = self.last
|
||||||
|
self.last += 1
|
||||||
|
elif num == self.last:
|
||||||
|
self.last += 1
|
||||||
|
registered = False
|
||||||
|
if not needToRegister(self.net):
|
||||||
|
registered = True
|
||||||
|
# Don't need to register if it's been disabled in definitions,
|
||||||
|
# so we'll pretend we already did
|
||||||
|
self.relays[num] = {
|
||||||
|
"enabled": main.config["ConnectOnCreate"],
|
||||||
|
"net": self.net,
|
||||||
|
"id": num,
|
||||||
|
"registered": registered,
|
||||||
|
}
|
||||||
|
password = alias.generate_password()
|
||||||
|
if num not in main.alias.keys():
|
||||||
|
main.alias[num] = alias.generate_alias()
|
||||||
|
main.saveConf("alias")
|
||||||
|
self.aliases[num] = {"password": password}
|
||||||
|
# if main.config["ConnectOnCreate"]: -- Done in provision
|
||||||
|
# self.start_bot(num)
|
||||||
|
provisionRelay(num, self.net)
|
||||||
|
return num, main.alias[num]["nick"]
|
||||||
|
|
||||||
|
def enable_relay(self, num):
|
||||||
|
"""
|
||||||
|
Enable a relay for this network.
|
||||||
|
Send a command to ZNC to connect.
|
||||||
|
"""
|
||||||
|
self.relays[num]["enabled"] = True
|
||||||
|
user = main.alias[num]["nick"]
|
||||||
|
commands = {"status": ["Connect"]}
|
||||||
|
name = f"{self.net}{num}"
|
||||||
|
deliverRelayCommands(num, commands, user=user + "/" + self.net)
|
||||||
|
main.saveConf("network")
|
||||||
|
if name not in main.IRCPool.keys():
|
||||||
|
self.start_bot(num)
|
||||||
|
|
||||||
|
def disable_relay(self, num):
|
||||||
|
"""
|
||||||
|
Disable a relay for this network.
|
||||||
|
Send a command to ZNC to disconnect.
|
||||||
|
Stop trying to connect to the relay.
|
||||||
|
"""
|
||||||
|
self.relays[num]["enabled"] = False
|
||||||
|
user = main.alias[num]["nick"]
|
||||||
|
# relay = main.network[spl[1]].relays[relayNum]
|
||||||
|
commands = {"status": ["Disconnect"]}
|
||||||
|
name = f"{self.net}{num}"
|
||||||
|
deliverRelayCommands(num, commands, user=user + "/" + self.net)
|
||||||
|
main.saveConf("network")
|
||||||
|
if name in main.ReactorPool.keys():
|
||||||
|
if name in main.FactoryPool.keys():
|
||||||
|
main.FactoryPool[name].stopTrying()
|
||||||
|
main.ReactorPool[name].disconnect()
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
del main.IRCPool[name]
|
||||||
|
del main.ReactorPool[name]
|
||||||
|
del main.FactoryPool[name]
|
||||||
|
|
||||||
|
def killAliases(self, aliasList):
|
||||||
|
for i in aliasList:
|
||||||
|
name = self.net + str(i)
|
||||||
|
if name in main.ReactorPool.keys():
|
||||||
|
if name in main.FactoryPool.keys():
|
||||||
|
main.FactoryPool[name].stopTrying()
|
||||||
|
main.ReactorPool[name].disconnect()
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
del main.IRCPool[name]
|
||||||
|
del main.ReactorPool[name]
|
||||||
|
del main.FactoryPool[name]
|
||||||
|
|
||||||
|
def delete_relay(self, id):
|
||||||
|
del self.relays[id]
|
||||||
|
del self.aliases[id]
|
||||||
|
# del main.alias[id] - Aliases are global per num, so don't delete them!
|
||||||
|
self.killAliases([id])
|
||||||
|
|
||||||
|
def seppuku(self):
|
||||||
|
# Removes all bots in preperation for deletion
|
||||||
|
self.killAliases(self.relays.keys())
|
||||||
|
nukeNetwork(self.net)
|
||||||
|
|
||||||
|
def start_bot(self, num):
|
||||||
|
# a single name is given to relays in the backend
|
||||||
|
# e.g. freenode1 for the first relay on freenode network
|
||||||
|
keyFN = main.certPath + main.config["Key"]
|
||||||
|
certFN = main.certPath + main.config["Certificate"]
|
||||||
|
contextFactory = DefaultOpenSSLContextFactory(
|
||||||
|
keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
|
||||||
|
)
|
||||||
|
bot = IRCBotFactory(self.net, num)
|
||||||
|
# host, port = self.relays[num]["host"], self.relays[num]["port"]
|
||||||
|
host, port = getRelay(num)
|
||||||
|
rct = reactor.connectSSL(host, port, bot, contextFactory)
|
||||||
|
name = self.net + str(num)
|
||||||
|
main.ReactorPool[name] = rct
|
||||||
|
main.FactoryPool[name] = bot
|
||||||
|
|
||||||
|
log("Started bot on relay %s on %s" % (num, self.host))
|
||||||
|
|
||||||
|
def start_bots(self):
|
||||||
|
for num in self.relays.keys():
|
||||||
|
if self.relays[num]["enabled"]:
|
||||||
|
self.start_bot(num)
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
import main
|
||||||
|
import modules.regproc
|
||||||
|
from twisted.internet import reactor
|
||||||
|
from utils.deliver_relay_commands import deliverRelayCommands
|
||||||
|
from utils.logging.log import warn
|
||||||
|
|
||||||
|
|
||||||
|
def provisionUserNetworkData(
|
||||||
|
num, nick, altnick, ident, realname, network, host, port, security
|
||||||
|
):
|
||||||
|
commands = {}
|
||||||
|
stage2commands = {}
|
||||||
|
stage2commands["status"] = []
|
||||||
|
commands["controlpanel"] = []
|
||||||
|
user = nick.lower()
|
||||||
|
commands["controlpanel"].append(
|
||||||
|
"AddUser %s %s" % (user, main.config["Relay"]["Password"])
|
||||||
|
)
|
||||||
|
commands["controlpanel"].append("AddNetwork %s %s" % (user, network))
|
||||||
|
commands["controlpanel"].append("Set Nick %s %s" % (user, nick))
|
||||||
|
commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick))
|
||||||
|
commands["controlpanel"].append("Set Ident %s %s" % (user, ident))
|
||||||
|
commands["controlpanel"].append("Set RealName %s %s" % (user, realname))
|
||||||
|
if security == "ssl":
|
||||||
|
commands["controlpanel"].append(
|
||||||
|
"SetNetwork TrustAllCerts %s %s true" % (user, network)
|
||||||
|
) # Don't judge me
|
||||||
|
commands["controlpanel"].append(
|
||||||
|
"AddServer %s %s %s +%s" % (user, network, host, port)
|
||||||
|
)
|
||||||
|
elif security == "plain":
|
||||||
|
commands["controlpanel"].append(
|
||||||
|
"AddServer %s %s %s %s" % (user, network, host, port)
|
||||||
|
)
|
||||||
|
if not main.config["ConnectOnCreate"]:
|
||||||
|
stage2commands["status"].append("Disconnect")
|
||||||
|
if main.config["Toggles"]["CycleChans"]:
|
||||||
|
stage2commands["status"].append("LoadMod disconkick")
|
||||||
|
stage2commands["status"].append("LoadMod chansaver")
|
||||||
|
inst = modules.regproc.selectInst(network)
|
||||||
|
if "setmode" in inst.keys():
|
||||||
|
stage2commands["status"].append("LoadMod perform")
|
||||||
|
# stage2commands["perform"].append("add mode %nick% +"+inst["setmode"])
|
||||||
|
deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]])
|
||||||
|
|
||||||
|
|
||||||
|
def provisionAuthenticationData(num, nick, network, auth, password):
|
||||||
|
commands = {}
|
||||||
|
commands["status"] = []
|
||||||
|
user = nick.lower()
|
||||||
|
if auth == "sasl":
|
||||||
|
commands["sasl"] = []
|
||||||
|
commands["status"].append("UnloadMod nickserv")
|
||||||
|
commands["status"].append("LoadMod sasl")
|
||||||
|
commands["sasl"].append("Mechanism plain")
|
||||||
|
commands["sasl"].append("Set %s %s" % (nick, password))
|
||||||
|
elif auth == "ns":
|
||||||
|
commands["nickserv"] = []
|
||||||
|
commands["status"].append("UnloadMod sasl")
|
||||||
|
commands["status"].append("LoadMod nickserv")
|
||||||
|
commands["nickserv"].append("Set %s" % password)
|
||||||
|
inst = modules.regproc.selectInst(network)
|
||||||
|
if "setmode" in inst.keys():
|
||||||
|
# perform is loaded above
|
||||||
|
# commands["status"].append("LoadMod perform")
|
||||||
|
commands["perform"] = ["add mode %nick% +" + inst["setmode"]]
|
||||||
|
deliverRelayCommands(num, commands, user=user + "/" + network)
|
||||||
|
|
||||||
|
|
||||||
|
def provisionRelay(num, network): # provision user and network data
|
||||||
|
aliasObj = main.alias[num]
|
||||||
|
# alias = aliasObj["nick"]
|
||||||
|
nick = aliasObj["nick"]
|
||||||
|
altnick = aliasObj["altnick"]
|
||||||
|
ident = aliasObj["ident"]
|
||||||
|
realname = aliasObj["realname"]
|
||||||
|
provisionUserNetworkData(
|
||||||
|
num,
|
||||||
|
nick,
|
||||||
|
altnick,
|
||||||
|
ident,
|
||||||
|
realname,
|
||||||
|
network,
|
||||||
|
main.network[network].host,
|
||||||
|
main.network[network].port,
|
||||||
|
main.network[network].security,
|
||||||
|
)
|
||||||
|
if main.config["ConnectOnCreate"]:
|
||||||
|
reactor.callLater(10, main.network[network].start_bot, num)
|
||||||
|
|
||||||
|
|
||||||
|
def provisionMultipleRelays(net, relaysNeeded):
|
||||||
|
if not relaysNeeded:
|
||||||
|
return []
|
||||||
|
if not main.config["ChanKeep"]["Provision"]:
|
||||||
|
warn(
|
||||||
|
f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled"
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
numsProvisioned = []
|
||||||
|
for i in range(relaysNeeded):
|
||||||
|
num, alias = main.network[net].add_relay()
|
||||||
|
numsProvisioned.append(num)
|
||||||
|
provisionRelay(num, net)
|
||||||
|
main.saveConf("network")
|
||||||
|
return numsProvisioned
|
||||||
|
|
@ -0,0 +1,237 @@
|
||||||
|
from copy import deepcopy
|
||||||
|
from random import choice
|
||||||
|
|
||||||
|
import main
|
||||||
|
from modules import provision
|
||||||
|
from utils.logging.debug import debug
|
||||||
|
from utils.logging.log import error
|
||||||
|
|
||||||
|
|
||||||
|
def needToRegister(net):
|
||||||
|
# Check if the network does not support authentication
|
||||||
|
networkObj = main.network[net]
|
||||||
|
if networkObj.auth == "none":
|
||||||
|
return False
|
||||||
|
# Check if the IRC network definition has registration disabled
|
||||||
|
inst = selectInst(net)
|
||||||
|
if "register" in inst.keys():
|
||||||
|
if inst["register"]:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def needToAuth(net):
|
||||||
|
networkObj = main.network[net]
|
||||||
|
if networkObj.auth == "none":
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def selectInst(net):
|
||||||
|
if net in main.irc.keys():
|
||||||
|
inst = deepcopy(main.irc[net])
|
||||||
|
for i in main.irc["_"].keys():
|
||||||
|
if i not in inst:
|
||||||
|
inst[i] = main.irc["_"][i]
|
||||||
|
else:
|
||||||
|
inst = deepcopy(main.irc["_"])
|
||||||
|
return inst
|
||||||
|
|
||||||
|
|
||||||
|
def substitute(net, num, token=None):
|
||||||
|
inst = selectInst(net)
|
||||||
|
alias = main.alias[num]
|
||||||
|
gotemail = False
|
||||||
|
if "emails" in alias:
|
||||||
|
# First priority is explicit email lists
|
||||||
|
if alias["emails"]:
|
||||||
|
email = choice(alias["emails"])
|
||||||
|
gotemail = True
|
||||||
|
if "domains" in inst:
|
||||||
|
if inst["domains"]:
|
||||||
|
if not gotemail:
|
||||||
|
domain = choice(inst["domains"])
|
||||||
|
email = f"{alias['nickname']}@{domain}"
|
||||||
|
gotemail = True
|
||||||
|
if not gotemail:
|
||||||
|
inst["email"] = False
|
||||||
|
nickname = alias["nick"]
|
||||||
|
# username = nickname + "/" + net
|
||||||
|
password = main.network[net].aliases[num]["password"]
|
||||||
|
# inst["email"] = inst["email"].replace("{nickname}", nickname)
|
||||||
|
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name in main.IRCPool:
|
||||||
|
curnick = main.IRCPool[name].nickname
|
||||||
|
else:
|
||||||
|
curnick = nickname
|
||||||
|
for i in inst.keys():
|
||||||
|
if not isinstance(inst[i], str):
|
||||||
|
continue
|
||||||
|
inst[i] = inst[i].replace("{nickname}", nickname)
|
||||||
|
inst[i] = inst[i].replace("{curnick}", curnick)
|
||||||
|
inst[i] = inst[i].replace("{password}", password)
|
||||||
|
if gotemail:
|
||||||
|
inst[i] = inst[i].replace("{email}", email)
|
||||||
|
if token:
|
||||||
|
inst[i] = inst[i].replace("{token}", token)
|
||||||
|
return inst
|
||||||
|
|
||||||
|
|
||||||
|
def registerAccount(net, num):
|
||||||
|
debug("Attempting to register: %s - %i" % (net, num))
|
||||||
|
sinst = substitute(net, num)
|
||||||
|
if not sinst:
|
||||||
|
error(f"Register account failed for {net} - {num}")
|
||||||
|
return
|
||||||
|
if not sinst["email"]:
|
||||||
|
error(f"Could not get email for {net} - {num}")
|
||||||
|
return
|
||||||
|
if not sinst["register"]:
|
||||||
|
error("Cannot register for %s: function disabled" % (net))
|
||||||
|
return False
|
||||||
|
name = net + str(num)
|
||||||
|
if not main.IRCPool[name].authenticated:
|
||||||
|
main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"])
|
||||||
|
|
||||||
|
|
||||||
|
def confirmAccount(net, num, token):
|
||||||
|
sinst = substitute(net, num, token=token)
|
||||||
|
name = net + str(num)
|
||||||
|
if name in main.IRCPool:
|
||||||
|
main.IRCPool[name].msg(sinst["entity"], sinst["confirm"])
|
||||||
|
enableAuthentication(net, num)
|
||||||
|
|
||||||
|
|
||||||
|
def confirmRegistration(net, num, negativepass=None):
|
||||||
|
obj = main.network[net]
|
||||||
|
name = net + str(num)
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
if negativepass is not None:
|
||||||
|
main.IRCPool[name].regPing(negativepass=negativepass)
|
||||||
|
return
|
||||||
|
debug("Relay authenticated: %s - %i" % (net, num))
|
||||||
|
main.IRCPool[name].authenticated = True
|
||||||
|
main.IRCPool[name].recheckList()
|
||||||
|
if obj.relays[num]["registered"]:
|
||||||
|
return
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
if main.IRCPool[name]._regAttempt:
|
||||||
|
try:
|
||||||
|
main.IRCPool[name]._regAttempt.cancel()
|
||||||
|
except: # noqa
|
||||||
|
pass
|
||||||
|
obj.relays[num]["registered"] = True
|
||||||
|
main.saveConf("network")
|
||||||
|
|
||||||
|
|
||||||
|
def attemptManualAuthentication(net, num):
|
||||||
|
sinst = substitute(net, num)
|
||||||
|
identifymsg = sinst["identifymsg"]
|
||||||
|
entity = sinst["entity"]
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name not in main.IRCPool:
|
||||||
|
return
|
||||||
|
main.IRCPool[name].sendmsg(entity, identifymsg, in_query=True)
|
||||||
|
|
||||||
|
|
||||||
|
def enableAuthentication(net, num, jump=True, run_now=False):
|
||||||
|
obj = main.network[net]
|
||||||
|
nick = main.alias[num]["nick"]
|
||||||
|
auth = obj.auth
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name not in main.IRCPool:
|
||||||
|
return
|
||||||
|
# uname = main.alias[num]["nick"] + "/" + net
|
||||||
|
password = main.network[net].aliases[num]["password"]
|
||||||
|
provision.provisionAuthenticationData(
|
||||||
|
num, nick, net, auth, password
|
||||||
|
) # Set up for auth
|
||||||
|
if jump:
|
||||||
|
main.IRCPool[name].msg(
|
||||||
|
main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump"
|
||||||
|
)
|
||||||
|
if run_now:
|
||||||
|
attemptManualAuthentication(net, num)
|
||||||
|
if selectInst(net)["check"] is False:
|
||||||
|
confirmRegistration(net, num)
|
||||||
|
|
||||||
|
|
||||||
|
def get_unregistered_relays(net=None):
|
||||||
|
"""
|
||||||
|
Get a dict of unregistereed relays, either globally or
|
||||||
|
for a network.
|
||||||
|
Returns:
|
||||||
|
{"net": [["nick1", 1], ["nick2", 2], ...]}
|
||||||
|
"""
|
||||||
|
unreg = {}
|
||||||
|
if net:
|
||||||
|
nets = [net]
|
||||||
|
else:
|
||||||
|
nets = main.network.keys()
|
||||||
|
for i in nets:
|
||||||
|
for num in main.network[i].relays.keys():
|
||||||
|
if not main.network[i].relays[num]["registered"]:
|
||||||
|
nick = main.alias[num]["nick"]
|
||||||
|
if i in unreg:
|
||||||
|
unreg[i].append([nick, num])
|
||||||
|
else:
|
||||||
|
unreg[i] = [[nick, num]]
|
||||||
|
return unreg
|
||||||
|
|
||||||
|
|
||||||
|
def registerTest(c):
|
||||||
|
sinst = substitute(c["net"], c["num"])
|
||||||
|
name = c["net"] + str(c["num"])
|
||||||
|
net = c["net"]
|
||||||
|
num = c["num"]
|
||||||
|
if sinst["check"] is False:
|
||||||
|
return
|
||||||
|
if "msg" in c.keys() and not c["msg"] is None:
|
||||||
|
if sinst["negative"]:
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
if main.IRCPool[name]._negativePass is not True:
|
||||||
|
if c["type"] == "query" and c["nick"] == sinst["entity"]:
|
||||||
|
if sinst["checknegativemsg"] in c["msg"]:
|
||||||
|
confirmRegistration(
|
||||||
|
c["net"], c["num"], negativepass=False
|
||||||
|
) # Not passed negative check, report back
|
||||||
|
return
|
||||||
|
if sinst["checkendnegative"] in c["msg"]:
|
||||||
|
confirmRegistration(
|
||||||
|
c["net"], c["num"], negativepass=True
|
||||||
|
) # Passed the negative check, report back
|
||||||
|
return
|
||||||
|
if sinst["ping"]:
|
||||||
|
if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]:
|
||||||
|
confirmRegistration(c["net"], c["num"])
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"registerTest() {net} - {num} passed ping:checkmsg2 "
|
||||||
|
f"check, {sinst['checkmsg2']} present in message"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if sinst["checktype"] == "msg":
|
||||||
|
if sinst["checkmsg"] in c["msg"]:
|
||||||
|
confirmRegistration(c["net"], c["num"])
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, "
|
||||||
|
f"{sinst['checkmsg']} present in message"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
elif sinst["checktype"] == "mode":
|
||||||
|
if c["type"] == "self":
|
||||||
|
if c["mtype"] == "mode":
|
||||||
|
if sinst["checkmode"] in c["mode"] and c["status"] is True:
|
||||||
|
confirmRegistration(c["net"], c["num"])
|
||||||
|
debug(
|
||||||
|
(
|
||||||
|
f"registerTest() {net} - {num} passed checktype:mode:checkmost check, "
|
||||||
|
f"{sinst['checkmode']} present in mode"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
@ -1,17 +1,18 @@
|
||||||
from twisted.internet.threads import deferToThread
|
|
||||||
from string import digits
|
|
||||||
|
|
||||||
import main
|
import main
|
||||||
from utils.logging.log import *
|
from modules import chankeep
|
||||||
from utils.logging.debug import debug
|
from twisted.internet.threads import deferToThread
|
||||||
|
from utils.logging.debug import debug, trace
|
||||||
|
from utils.logging.log import warn
|
||||||
from utils.parsing import parsen
|
from utils.parsing import parsen
|
||||||
|
|
||||||
|
|
||||||
def getWhoSingle(name, query):
|
def getWhoSingle(name, query):
|
||||||
result = main.r.sscan("live.who." + name, 0, query, count=999999)
|
result = main.r.sscan("live.who." + name, 0, query, count=999999)
|
||||||
if result[1] == []:
|
if result[1] == []:
|
||||||
return None
|
return None
|
||||||
return (i.decode() for i in result[1])
|
return (i.decode() for i in result[1])
|
||||||
|
|
||||||
|
|
||||||
def getWho(query):
|
def getWho(query):
|
||||||
result = {}
|
result = {}
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
|
|
@ -20,13 +21,15 @@ def getWho(query):
|
||||||
result[i] = f
|
result[i] = f
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def getChansSingle(name, nick):
|
def getChansSingle(name, nick):
|
||||||
nick = ["live.chan."+name+"."+i for i in nick]
|
nick = ("live.chan." + name + "." + i for i in nick)
|
||||||
result = main.r.sinter(*nick)
|
result = main.r.sinter(*nick)
|
||||||
if len(result) == 0:
|
if len(result) == 0:
|
||||||
return None
|
return None
|
||||||
return (i.decode() for i in result)
|
return (i.decode() for i in result)
|
||||||
|
|
||||||
|
|
||||||
def getChanList(name, nick):
|
def getChanList(name, nick):
|
||||||
chanspace = "live.chan." + name + "." + nick
|
chanspace = "live.chan." + name + "." + nick
|
||||||
result = main.r.smembers(chanspace)
|
result = main.r.smembers(chanspace)
|
||||||
|
|
@ -34,6 +37,37 @@ def getChanList(name, nick):
|
||||||
return None
|
return None
|
||||||
return (i.decode() for i in result)
|
return (i.decode() for i in result)
|
||||||
|
|
||||||
|
|
||||||
|
def getTotalChanNum(net):
|
||||||
|
"""
|
||||||
|
Get the number of channels a network has.
|
||||||
|
"""
|
||||||
|
chans = main.r.keys(f"live.who.{net}.*")
|
||||||
|
return len(chans)
|
||||||
|
|
||||||
|
|
||||||
|
def getUserNum(name, channels):
|
||||||
|
"""
|
||||||
|
Get the number of users on a list of channels.
|
||||||
|
"""
|
||||||
|
chanspace = ("live.who." + name + "." + i for i in channels)
|
||||||
|
results = {}
|
||||||
|
for channel, space in zip(channels, chanspace):
|
||||||
|
results[channel] = main.r.scard(space)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def getChanNum(name, nicks):
|
||||||
|
"""
|
||||||
|
Get the number of channels a list of users are on.
|
||||||
|
"""
|
||||||
|
nickspace = ("live.chan." + name + "." + i for i in nicks)
|
||||||
|
results = {}
|
||||||
|
for nick, space in zip(nicks, nickspace):
|
||||||
|
results[nick] = main.r.scard(space)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
def getChans(nick):
|
def getChans(nick):
|
||||||
result = {}
|
result = {}
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
|
|
@ -42,6 +76,7 @@ def getChans(nick):
|
||||||
result[i] = f
|
result[i] = f
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def getUsersSingle(name, nick):
|
def getUsersSingle(name, nick):
|
||||||
nick = ("live.who." + name + "." + i for i in nick)
|
nick = ("live.who." + name + "." + i for i in nick)
|
||||||
result = main.r.sinter(*nick)
|
result = main.r.sinter(*nick)
|
||||||
|
|
@ -49,6 +84,7 @@ def getUsersSingle(name, nick):
|
||||||
return None
|
return None
|
||||||
return (i.decode() for i in result)
|
return (i.decode() for i in result)
|
||||||
|
|
||||||
|
|
||||||
def getUsers(nick):
|
def getUsers(nick):
|
||||||
result = {}
|
result = {}
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
|
|
@ -57,15 +93,18 @@ def getUsers(nick):
|
||||||
result[i] = f
|
result[i] = f
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def getNumWhoEntries(name):
|
def getNumWhoEntries(name):
|
||||||
return main.r.scard("live.who." + name)
|
return main.r.scard("live.who." + name)
|
||||||
|
|
||||||
|
|
||||||
def getNumTotalWhoEntries():
|
def getNumTotalWhoEntries():
|
||||||
total = 0
|
total = 0
|
||||||
for i in main.network.keys():
|
for i in main.network.keys():
|
||||||
total += getNumWhoEntries(i)
|
total += getNumWhoEntries(i)
|
||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
||||||
def getNamespace(name, channel, nick):
|
def getNamespace(name, channel, nick):
|
||||||
gnamespace = "live.who.%s" % name
|
gnamespace = "live.who.%s" % name
|
||||||
namespace = "live.who.%s.%s" % (name, channel)
|
namespace = "live.who.%s.%s" % (name, channel)
|
||||||
|
|
@ -73,6 +112,7 @@ def getNamespace(name, channel, nick):
|
||||||
mapspace = "live.map.%s" % name
|
mapspace = "live.map.%s" % name
|
||||||
return (gnamespace, namespace, chanspace, mapspace)
|
return (gnamespace, namespace, chanspace, mapspace)
|
||||||
|
|
||||||
|
|
||||||
def _initialUsers(name, channel, users):
|
def _initialUsers(name, channel, users):
|
||||||
gnamespace = "live.who.%s" % name
|
gnamespace = "live.who.%s" % name
|
||||||
mapspace = "live.map.%s" % name
|
mapspace = "live.map.%s" % name
|
||||||
|
|
@ -83,24 +123,30 @@ def _initialUsers(name, channel, users):
|
||||||
p.sadd(gnamespace, user)
|
p.sadd(gnamespace, user)
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def initialUsers(name, channel, users):
|
def initialUsers(name, channel, users):
|
||||||
debug("Initialising WHO records for %s on %s" % (channel, name))
|
trace("Initialising WHO records for %s on %s" % (channel, name))
|
||||||
d = deferToThread(_initialUsers, name, channel, users)
|
deferToThread(_initialUsers, name, channel, users)
|
||||||
# d.addCallback(testCallback)
|
# d.addCallback(testCallback)
|
||||||
|
|
||||||
|
|
||||||
def _initialNames(name, channel, names):
|
def _initialNames(name, channel, names):
|
||||||
namespace = "live.who.%s.%s" % (name, channel)
|
namespace = "live.who.%s.%s" % (name, channel)
|
||||||
p = main.r.pipeline()
|
p = main.r.pipeline()
|
||||||
for i in names:
|
for mode, nick in names:
|
||||||
p.sadd(namespace, i)
|
p.sadd(namespace, nick)
|
||||||
p.sadd("live.chan."+name+"."+i, channel)
|
p.sadd("live.chan." + name + "." + nick, channel)
|
||||||
|
if mode:
|
||||||
|
p.hset("live.prefix." + name + "." + channel, nick, mode)
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def initialNames(name, channel, names):
|
def initialNames(name, channel, names):
|
||||||
debug("Initialising NAMES records for %s on %s" % (channel, name))
|
trace("Initialising NAMES records for %s on %s" % (channel, name))
|
||||||
d = deferToThread(_initialNames, name, channel, names)
|
deferToThread(_initialNames, name, channel, names)
|
||||||
# d.addCallback(testCallback)
|
# d.addCallback(testCallback)
|
||||||
|
|
||||||
|
|
||||||
def editUser(name, user):
|
def editUser(name, user):
|
||||||
gnamespace = "live.who.%s" % name
|
gnamespace = "live.who.%s" % name
|
||||||
mapspace = "live.map.%s" % name
|
mapspace = "live.map.%s" % name
|
||||||
|
|
@ -110,6 +156,7 @@ def editUser(name, user):
|
||||||
p.hset(mapspace, parsed[0], user) # add nick -> user mapping
|
p.hset(mapspace, parsed[0], user) # add nick -> user mapping
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def addUser(name, channel, nick, user):
|
def addUser(name, channel, nick, user):
|
||||||
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
|
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
|
||||||
p = main.r.pipeline()
|
p = main.r.pipeline()
|
||||||
|
|
@ -119,6 +166,7 @@ def addUser(name, channel, nick, user):
|
||||||
p.hset(mapspace, nick, user)
|
p.hset(mapspace, nick, user)
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def delUser(name, channel, nick, user):
|
def delUser(name, channel, nick, user):
|
||||||
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
|
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
|
||||||
p = main.r.pipeline()
|
p = main.r.pipeline()
|
||||||
|
|
@ -126,6 +174,9 @@ def delUser(name, channel, nick, user):
|
||||||
p.srem(namespace, nick)
|
p.srem(namespace, nick)
|
||||||
if channels == {channel.encode()}: # can we only see them on this channel?
|
if channels == {channel.encode()}: # can we only see them on this channel?
|
||||||
p.delete(chanspace) # remove channel tracking entry
|
p.delete(chanspace) # remove channel tracking entry
|
||||||
|
p.hdel(
|
||||||
|
"live.prefix." + name + "." + channel, nick
|
||||||
|
) # remove prefix tracking entry
|
||||||
p.hdel(mapspace, nick) # remove nick mapping entry
|
p.hdel(mapspace, nick) # remove nick mapping entry
|
||||||
if user:
|
if user:
|
||||||
p.srem(gnamespace, user) # remove global userinfo entry
|
p.srem(gnamespace, user) # remove global userinfo entry
|
||||||
|
|
@ -135,6 +186,7 @@ def delUser(name, channel, nick, user):
|
||||||
p.srem(chanspace, channel) # keep up - remove the channel from their list
|
p.srem(chanspace, channel) # keep up - remove the channel from their list
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def escape(text):
|
def escape(text):
|
||||||
chars = ["[", "]", "^", "-", "*", "?"]
|
chars = ["[", "]", "^", "-", "*", "?"]
|
||||||
text = text.replace("\\", "\\\\")
|
text = text.replace("\\", "\\\\")
|
||||||
|
|
@ -142,17 +194,22 @@ def escape(text):
|
||||||
text = text.replace(i, "\\" + i)
|
text = text.replace(i, "\\" + i)
|
||||||
return text
|
return text
|
||||||
|
|
||||||
|
|
||||||
def getUserByNick(name, nick):
|
def getUserByNick(name, nick):
|
||||||
gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
|
gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
|
||||||
mapspace = "live.map.%s" % name
|
mapspace = "live.map.%s" % name
|
||||||
if main.r.hexists(mapspace, nick):
|
if main.r.hexists(mapspace, nick):
|
||||||
return main.r.hget(mapspace, nick)
|
return main.r.hget(mapspace, nick)
|
||||||
else:
|
else:
|
||||||
warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace))
|
warn(
|
||||||
#return Falsedd
|
"Entry doesn't exist: %s on %s - attempting auxiliary lookup"
|
||||||
|
% (nick, mapspace)
|
||||||
|
)
|
||||||
|
# return False
|
||||||
# legacy code below - remove when map is reliable
|
# legacy code below - remove when map is reliable
|
||||||
usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1)
|
usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999)
|
||||||
if usermatch[1] == []:
|
if usermatch[1] == []:
|
||||||
|
warn("No matches found for user query: %s on %s" % (nick, name))
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
if len(usermatch[1]) == 1:
|
if len(usermatch[1]) == 1:
|
||||||
|
|
@ -162,6 +219,7 @@ def getUserByNick(name, nick):
|
||||||
warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
|
warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def renameUser(name, oldnick, olduser, newnick, newuser):
|
def renameUser(name, oldnick, olduser, newnick, newuser):
|
||||||
gnamespace = "live.who.%s" % name
|
gnamespace = "live.who.%s" % name
|
||||||
chanspace = "live.chan.%s.%s" % (name, oldnick)
|
chanspace = "live.chan.%s.%s" % (name, oldnick)
|
||||||
|
|
@ -176,16 +234,32 @@ def renameUser(name, oldnick, olduser, newnick, newuser):
|
||||||
p.sadd("live.who." + name + "." + i, newnick)
|
p.sadd("live.who." + name + "." + i, newnick)
|
||||||
p.hdel(mapspace, oldnick)
|
p.hdel(mapspace, oldnick)
|
||||||
p.hset(mapspace, newnick, newuser)
|
p.hset(mapspace, newnick, newuser)
|
||||||
|
if main.r.exists(
|
||||||
|
"live.prefix." + name + "." + i
|
||||||
|
): # if there's a prefix entry for the channel
|
||||||
|
if main.r.hexists(
|
||||||
|
"live.prefix." + name + "." + i, oldnick
|
||||||
|
): # if the old nick is in it
|
||||||
|
mode = main.r.hget(
|
||||||
|
"live.prefix." + name + "." + i, oldnick
|
||||||
|
) # retrieve old modes
|
||||||
|
p.hset(
|
||||||
|
"live.prefix." + name + "." + i, newnick, mode
|
||||||
|
) # set old modes to new nickname
|
||||||
if main.r.exists(chanspace):
|
if main.r.exists(chanspace):
|
||||||
p.rename(chanspace, newchanspace)
|
p.rename(chanspace, newchanspace)
|
||||||
else:
|
else:
|
||||||
warn("Key doesn't exist: %s" % chanspace)
|
warn("Key doesn't exist: %s" % chanspace)
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def delUserByNick(name, channel, nick): # kick
|
def delUserByNick(name, channel, nick): # kick
|
||||||
user = getUserByNick(name, nick)
|
user = getUserByNick(name, nick)
|
||||||
|
if not user:
|
||||||
|
return
|
||||||
delUser(name, channel, nick, user)
|
delUser(name, channel, nick, user)
|
||||||
|
|
||||||
|
|
||||||
def delUserByNetwork(name, nick, user): # quit
|
def delUserByNetwork(name, nick, user): # quit
|
||||||
gnamespace = "live.who.%s" % name
|
gnamespace = "live.who.%s" % name
|
||||||
chanspace = "live.chan.%s.%s" % (name, nick)
|
chanspace = "live.chan.%s.%s" % (name, nick)
|
||||||
|
|
@ -194,10 +268,13 @@ def delUserByNetwork(name, nick, user): # quit
|
||||||
p.srem(gnamespace, user)
|
p.srem(gnamespace, user)
|
||||||
for i in main.r.smembers(chanspace):
|
for i in main.r.smembers(chanspace):
|
||||||
p.srem("live.who." + name + "." + i.decode(), nick)
|
p.srem("live.who." + name + "." + i.decode(), nick)
|
||||||
|
p.hdel("live.prefix." + name + "." + i.decode(), nick)
|
||||||
|
|
||||||
p.delete(chanspace)
|
p.delete(chanspace)
|
||||||
p.hdel(mapspace, nick)
|
p.hdel(mapspace, nick)
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
|
|
||||||
def _delChannels(net, channels):
|
def _delChannels(net, channels):
|
||||||
gnamespace = "live.who.%s" % net
|
gnamespace = "live.who.%s" % net
|
||||||
mapspace = "live.map.%s" % net
|
mapspace = "live.map.%s" % net
|
||||||
|
|
@ -213,14 +290,29 @@ def _delChannels(net, channels):
|
||||||
if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}:
|
if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}:
|
||||||
if user:
|
if user:
|
||||||
p.srem(gnamespace, user)
|
p.srem(gnamespace, user)
|
||||||
|
|
||||||
p.delete("live.chan." + net + "." + nick)
|
p.delete("live.chan." + net + "." + nick)
|
||||||
p.hdel(mapspace, nick) # remove map entry
|
p.hdel(mapspace, nick) # remove map entry
|
||||||
else:
|
else:
|
||||||
p.srem("live.chan." + net + "." + nick, channel)
|
p.srem("live.chan." + net + "." + nick, channel)
|
||||||
p.delete(namespace)
|
p.delete(namespace)
|
||||||
|
p.delete("live.prefix." + net + "." + channel)
|
||||||
p.execute()
|
p.execute()
|
||||||
|
|
||||||
def delChannels(net, channels):
|
|
||||||
debug("Purging channel %s for %s" % (", ".join(channels), net))
|
def delChannels(net, channels): # we have left a channel
|
||||||
d = deferToThread(_delChannels, net, channels)
|
trace("Purging channel %s for %s" % (", ".join(channels), net))
|
||||||
|
dupes = chankeep.getDuplicateChannels(net, total=True)
|
||||||
|
print("dupes: %s" % dupes)
|
||||||
|
if not dupes:
|
||||||
|
deferToThread(_delChannels, net, channels)
|
||||||
|
else:
|
||||||
|
for channel in channels:
|
||||||
|
if channel in dupes[net]:
|
||||||
|
if dupes[net][channel] != 0:
|
||||||
|
channels.remove(channel)
|
||||||
|
debug(
|
||||||
|
f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it"
|
||||||
|
)
|
||||||
|
deferToThread(_delChannels, net, channels)
|
||||||
# d.addCallback(testCallback)
|
# d.addCallback(testCallback)
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
wheel
|
||||||
|
pre-commit
|
||||||
|
twisted
|
||||||
|
pyOpenSSL
|
||||||
|
redis
|
||||||
|
pyYaML
|
||||||
|
service_identity
|
||||||
|
siphashc
|
||||||
|
Klein
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
#!/bin/sh
|
||||||
|
#pre-commit run -a
|
||||||
|
python -m unittest discover -s tests -p 'test_*.py'
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
THRESHOLD_LISTENER_HOST=0.0.0.0
|
||||||
|
THRESHOLD_LISTENER_PORT=13867
|
||||||
|
THRESHOLD_LISTENER_SSL=1
|
||||||
|
|
||||||
|
THRESHOLD_RELAY_ENABLED=0
|
||||||
|
THRESHOLD_RELAY_HOST=0.0.0.0
|
||||||
|
THRESHOLD_RELAY_PORT=13868
|
||||||
|
THRESHOLD_RELAY_SSL=1
|
||||||
|
|
||||||
|
THRESHOLD_API_ENABLED=1
|
||||||
|
THRESHOLD_API_HOST=0.0.0.0
|
||||||
|
THRESHOLD_API_PORT=13869
|
||||||
|
PORTAINER_GIT_DIR=..
|
||||||
|
|
||||||
|
THRESHOLD_CONFIG_DIR=../conf/live/
|
||||||
|
THRESHOLD_TEMPLATE_DIR=../conf/example/
|
||||||
|
THRESHOLD_CERT_DIR=../conf/cert/
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
from math import ceil
|
||||||
|
from random import randint
|
||||||
|
from unittest import TestCase
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from modules import chankeep
|
||||||
|
|
||||||
|
|
||||||
|
class TestChanKeep(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.net = "testnet"
|
||||||
|
self.num = 1
|
||||||
|
self.chanlimit = 100
|
||||||
|
chankeep.main.initConf()
|
||||||
|
chankeep.main.r = MagicMock()
|
||||||
|
chankeep.main.g = MagicMock()
|
||||||
|
chankeep.main.g.pipeline = MagicMock()
|
||||||
|
chankeep.main.config["ChanKeep"]["Provision"] = False
|
||||||
|
chankeep.getAverageChanlimit = MagicMock()
|
||||||
|
chankeep.getAverageChanlimit.return_value = self.chanlimit
|
||||||
|
|
||||||
|
self.listinfo = self.generate_listinfo()
|
||||||
|
self.chan_name_list = [x[0] for x in self.listinfo]
|
||||||
|
self.chan_member_list = [x[1] for x in self.listinfo]
|
||||||
|
|
||||||
|
def generate_listinfo(self, ranges=None):
|
||||||
|
"""
|
||||||
|
Create a fake listinfo.
|
||||||
|
Where #channel has 192 users, and #channel2 has 188 users.
|
||||||
|
listinfo = [["#channel", 192], ["#channel2", 188]]
|
||||||
|
"""
|
||||||
|
if not ranges:
|
||||||
|
ranges = [[100, 5, 10], [400, 100, 200], [2, 500, 1000]]
|
||||||
|
listinfo = []
|
||||||
|
for channum, min, max in ranges:
|
||||||
|
for i in range(channum):
|
||||||
|
chan_name = f"#num-{channum}-{i}"
|
||||||
|
chan_users = randint(min, max)
|
||||||
|
listinfo.append([chan_name, chan_users])
|
||||||
|
return listinfo
|
||||||
|
|
||||||
|
def percent_diff(self, a, b):
|
||||||
|
return (abs(b - a) / a) * 100.0
|
||||||
|
|
||||||
|
def test_alt_listinfo(self):
|
||||||
|
# We're looking for a perc of 1000-1100
|
||||||
|
# And a sigrelay of 2
|
||||||
|
# We only want those 10 big channels
|
||||||
|
instances = 1
|
||||||
|
chanlimit = 5
|
||||||
|
max_chans = instances * chanlimit
|
||||||
|
listinfo = self.generate_listinfo(
|
||||||
|
ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]]
|
||||||
|
)
|
||||||
|
# listinfo_num = [x[1] for x in listinfo]
|
||||||
|
|
||||||
|
listlength = len(listinfo)
|
||||||
|
cumul = 0
|
||||||
|
try:
|
||||||
|
cumul += sum(int(i[1]) for i in listinfo)
|
||||||
|
except TypeError:
|
||||||
|
return
|
||||||
|
mean = round(cumul / listlength, 2)
|
||||||
|
siglength = 0
|
||||||
|
insiglength = 0
|
||||||
|
sigcumul = 0
|
||||||
|
insigcumul = 0
|
||||||
|
for i in listinfo:
|
||||||
|
if int(i[1]) > mean:
|
||||||
|
siglength += 1
|
||||||
|
sigcumul += int(i[1])
|
||||||
|
elif int(i[1]) < mean:
|
||||||
|
insiglength += 1
|
||||||
|
insigcumul += int(i[1])
|
||||||
|
|
||||||
|
sigrelay = ceil(siglength / chanlimit)
|
||||||
|
relay = ceil(listlength / chanlimit)
|
||||||
|
print(
|
||||||
|
(
|
||||||
|
f"len:{listlength} cumul:{cumul} mean:{mean} "
|
||||||
|
f"siglength:{siglength} insiglength:{insiglength} "
|
||||||
|
f"sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} "
|
||||||
|
f"insigcumul:{insigcumul}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# We want a return between 1000 and 1100
|
||||||
|
|
||||||
|
# list_insig = [x for x in listinfo_num if x < mean]
|
||||||
|
list_sig = [x for x in listinfo if x[1] > mean]
|
||||||
|
chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans]
|
||||||
|
self.assertEqual(len(chosen), 5)
|
||||||
|
|
||||||
|
@patch("modules.chankeep.keepChannels")
|
||||||
|
def test__initialList(self, keepchannels):
|
||||||
|
chankeep._initialList(self.net, self.num, self.listinfo)
|
||||||
|
net, passed_list, mean, sigrelay, relay = keepchannels.call_args_list[0][0]
|
||||||
|
self.assertEqual(net, self.net)
|
||||||
|
self.assertEqual(passed_list, self.listinfo)
|
||||||
|
# self.assertEqual(chanlimit, self.chanlimit)
|
||||||
|
# print(net, mean, sigrelay, relay)
|
||||||
|
|
||||||
|
@patch("modules.chankeep.getChanFree")
|
||||||
|
def test_empty_chan_allocate(self, getchanfree):
|
||||||
|
getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free
|
||||||
|
eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list)
|
||||||
|
self.assertEqual(len(eca), 1)
|
||||||
|
num = list(eca.keys())[0]
|
||||||
|
chans = eca[list(eca.keys())[0]]
|
||||||
|
self.assertEqual(num, self.num)
|
||||||
|
self.assertCountEqual(chans, self.chan_name_list)
|
||||||
|
|
||||||
|
getchanfree.return_value = ({1: 100}, 10)
|
||||||
|
eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list)
|
||||||
|
self.assertEqual(len(eca), 1)
|
||||||
|
num = list(eca.keys())[0]
|
||||||
|
chans = eca[list(eca.keys())[0]]
|
||||||
|
self.assertEqual(num, self.num)
|
||||||
|
self.assertEqual(len(chans), 100)
|
||||||
|
# self.assertCountEqual(chans, self.chan_name_list)
|
||||||
|
|
@ -0,0 +1,125 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
import sys
|
||||||
|
from codecs import getwriter
|
||||||
|
from os import getenv
|
||||||
|
from signal import SIGINT, signal
|
||||||
|
from sys import stderr, stdout
|
||||||
|
|
||||||
|
import main
|
||||||
|
import modules.counters
|
||||||
|
from api.views import API
|
||||||
|
from core.relay import RelayFactory
|
||||||
|
from core.server import ServerFactory
|
||||||
|
from twisted.internet import reactor
|
||||||
|
|
||||||
|
# Webapp stuff
|
||||||
|
from twisted.internet.protocol import Factory
|
||||||
|
from twisted.internet.ssl import DefaultOpenSSLContextFactory
|
||||||
|
from utils.cleanup import handler
|
||||||
|
from utils.loaders.command_loader import loadCommands
|
||||||
|
from utils.logging.log import log
|
||||||
|
|
||||||
|
Factory.noisy = False
|
||||||
|
|
||||||
|
main.initMain()
|
||||||
|
|
||||||
|
if "--debug" in sys.argv: # yes really
|
||||||
|
main.config["Debug"] = True
|
||||||
|
if "--trace" in sys.argv:
|
||||||
|
main.config["Trace"] = True
|
||||||
|
if "--migrate" in sys.argv:
|
||||||
|
from modules.network import migrate
|
||||||
|
|
||||||
|
migrate()
|
||||||
|
exit()
|
||||||
|
loadCommands()
|
||||||
|
|
||||||
|
|
||||||
|
# core.logstash.init_logstash()
|
||||||
|
signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine
|
||||||
|
stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know
|
||||||
|
stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using
|
||||||
|
# unicode quit messages for no reason
|
||||||
|
|
||||||
|
trues = ("true", "1", "t", True)
|
||||||
|
|
||||||
|
# Main listener
|
||||||
|
listener_address = getenv("THRESHOLD_LISTENER_HOST", main.config["Listener"]["Address"])
|
||||||
|
listener_port = int(getenv("THRESHOLD_LISTENER_PORT", main.config["Listener"]["Port"]))
|
||||||
|
listener_ssl = (
|
||||||
|
getenv("THRESHOLD_LISTENER_SSL", main.config["Listener"]["UseSSL"]) in trues
|
||||||
|
)
|
||||||
|
|
||||||
|
# RelayAPI
|
||||||
|
relay_enabled = (
|
||||||
|
getenv("THRESHOLD_RELAY_ENABLED", main.config["RelayAPI"]["Enabled"]) in trues
|
||||||
|
)
|
||||||
|
relay_address = getenv("THRESHOLD_RELAY_HOST", main.config["RelayAPI"]["Address"])
|
||||||
|
relay_port = int(getenv("THRESHOLD_RELAY_PORT", main.config["RelayAPI"]["Port"]))
|
||||||
|
relay_ssl = getenv("THRESHOLD_RELAY_SSL", main.config["RelayAPI"]["UseSSL"]) in trues
|
||||||
|
|
||||||
|
# Web API
|
||||||
|
api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in trues
|
||||||
|
api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"])
|
||||||
|
api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"]))
|
||||||
|
|
||||||
|
# Debugging
|
||||||
|
debug_enabled = getenv("THRESHOLD_DEBUG", main.config["Debug"]) in trues
|
||||||
|
trace_enabled = getenv("THRESHOLD_TRACE", main.config["Trace"]) in trues
|
||||||
|
if debug_enabled:
|
||||||
|
main.config["Debug"] = True
|
||||||
|
if trace_enabled:
|
||||||
|
main.config["Trace"] = True
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
listener = ServerFactory()
|
||||||
|
if listener_ssl is True:
|
||||||
|
reactor.listenSSL(
|
||||||
|
listener_port,
|
||||||
|
listener,
|
||||||
|
DefaultOpenSSLContextFactory(
|
||||||
|
main.certPath + main.config["Key"],
|
||||||
|
main.certPath + main.config["Certificate"],
|
||||||
|
),
|
||||||
|
interface=listener_address,
|
||||||
|
)
|
||||||
|
log("Threshold running with SSL on %s:%s" % (listener_address, listener_port))
|
||||||
|
else:
|
||||||
|
reactor.listenTCP(
|
||||||
|
listener_port,
|
||||||
|
listener,
|
||||||
|
interface=listener_address,
|
||||||
|
)
|
||||||
|
log("Threshold running on %s:%s" % (listener_address, listener_port))
|
||||||
|
if relay_enabled:
|
||||||
|
relay = RelayFactory()
|
||||||
|
if relay_ssl is True:
|
||||||
|
reactor.listenSSL(
|
||||||
|
relay_port,
|
||||||
|
relay,
|
||||||
|
DefaultOpenSSLContextFactory(
|
||||||
|
main.certPath + main.config["Key"],
|
||||||
|
main.certPath + main.config["Certificate"],
|
||||||
|
),
|
||||||
|
interface=relay_address,
|
||||||
|
)
|
||||||
|
log(
|
||||||
|
"Threshold relay running with SSL on %s:%s"
|
||||||
|
% (relay_address, relay_port)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
reactor.listenTCP(
|
||||||
|
relay_port,
|
||||||
|
relay,
|
||||||
|
interface=relay_address,
|
||||||
|
)
|
||||||
|
log("Threshold relay running on %s:%s" % (relay_address, relay_port))
|
||||||
|
for net in main.network.keys():
|
||||||
|
main.network[net].start_bots()
|
||||||
|
modules.counters.setupCounterLoop()
|
||||||
|
if api_enabled:
|
||||||
|
api = API()
|
||||||
|
api.app.run(api_address, api_port)
|
||||||
|
else:
|
||||||
|
reactor.run()
|
||||||
|
|
@ -1,13 +1,14 @@
|
||||||
import main
|
import main
|
||||||
from twisted.internet import reactor
|
from twisted.internet import reactor
|
||||||
from utils.logging.debug import debug
|
from utils.logging.debug import debug
|
||||||
from utils.logging.log import *
|
from utils.logging.log import log
|
||||||
import sys
|
|
||||||
|
|
||||||
def handler(sig, frame):
|
def handler(sig, frame):
|
||||||
log("Received SIGINT, cleaning up")
|
log("Received SIGINT, cleaning up")
|
||||||
cleanup()
|
cleanup()
|
||||||
|
|
||||||
|
|
||||||
def cleanup():
|
def cleanup():
|
||||||
debug("Flushing Redis database")
|
debug("Flushing Redis database")
|
||||||
main.r.flushdb()
|
main.r.flushdb()
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
from copy import deepcopy
|
||||||
|
from datetime import datetime
|
||||||
|
from json import dumps
|
||||||
|
|
||||||
|
import main
|
||||||
|
from siphashc import siphash
|
||||||
|
from utils.logging.debug import debug
|
||||||
|
|
||||||
|
|
||||||
|
def dedup(numName, b):
|
||||||
|
c = deepcopy(b)
|
||||||
|
if "ts" in c.keys():
|
||||||
|
del c["ts"]
|
||||||
|
c["approxtime"] = str(datetime.utcnow().timestamp())[
|
||||||
|
: main.config["Tweaks"]["DedupPrecision"]
|
||||||
|
]
|
||||||
|
castHash = siphash(main.hashKey, dumps(c, sort_keys=True))
|
||||||
|
del c["approxtime"]
|
||||||
|
isDuplicate = any(
|
||||||
|
castHash in main.lastEvents[x]
|
||||||
|
for x in main.lastEvents.keys()
|
||||||
|
if not x == numName
|
||||||
|
)
|
||||||
|
if isDuplicate:
|
||||||
|
debug("Duplicate: %s" % (c))
|
||||||
|
return True
|
||||||
|
if numName in main.lastEvents.keys():
|
||||||
|
main.lastEvents[numName].insert(0, castHash)
|
||||||
|
main.lastEvents[numName] = main.lastEvents[numName][
|
||||||
|
0 : main.config["Tweaks"]["MaxHash"]
|
||||||
|
] # noqa
|
||||||
|
else:
|
||||||
|
main.lastEvents[numName] = [castHash]
|
||||||
|
return False
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue