XF/monolith
2
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Moved files to subdirectory

Browse Source
This commit is contained in:
Mark Veidemanis
2022-09-06 12:50:09 +01:00
parent 2731713ede
commit 122fdca5db
83 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
legacy/modules/__init__.py Normal file
View File

75
legacy/modules/alias.py Normal file
View File

@@ -0,0 +1,75 @@
import random
import re
import main
def generate_password():
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
def generate_alias():
nick = random.choice(main.aliasdata["stubs"])
rand = random.randint(1, 2)
if rand == 1:
nick = nick.capitalize()
rand = random.randint(1, 4)
while rand == 1:
split = random.randint(0, len(nick) - 1)
nick = nick[:split] + nick[split + 1 :] # noqa: E203
rand = random.randint(1, 4)
rand = random.randint(1, 3)
if rand == 1 or rand == 4:
nick = random.choice(main.aliasdata["stubs"]) + nick
if rand == 2 or rand == 5:
nick = random.choice(main.aliasdata["stubs"]).capitalize() + nick
if rand > 2:
nick = nick + str(random.randint(0, 100))
nick = nick[:11]
altnick = nick
rand = random.randint(1, 3)
if rand == 1:
altnick += "_"
elif rand == 2:
altnick += "1"
else:
altnick = "_" + altnick
namebase = random.choice(main.aliasdata["realnames"])
ident = nick[:10]
rand = random.randint(1, 7)
if rand == 1:
ident = "quassel"
elif rand == 2:
ident = ident.lower()
elif rand == 3:
ident = re.sub("[0-9]", "", nick)
ident = ident[:10]
elif rand == 4:
ident = namebase.split(" ")[0].lower()
ident = ident[:10]
elif rand == 5:
ident = namebase.split(" ")[0]
ident = ident[:10]
elif rand == 6:
ident = re.sub("\s", "", namebase).lower() # noqa: W605
ident = ident[:10]
realname = nick
rand = random.randint(1, 5)
if rand == 1:
realname = namebase
elif rand == 2 or rand == 3:
realname = re.sub("[0-9]", "", realname)
if rand == 3 or rand == 4:
realname = realname.capitalize()
return {
"nick": nick,
"altnick": altnick,
"ident": ident,
"realname": realname,
"emails": [],
}

585
legacy/modules/chankeep.py Normal file
View File

@@ -0,0 +1,585 @@
from copy import deepcopy
from math import ceil
from twisted.internet.threads import deferToThread
import main
from modules import helpers
from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn
def getAllChannels(net=None):
"""
Get a list of all channels on all relays.
:return: list of channels
"""
channels = {}
if not net:
nets = main.network.keys()
else:
nets = [net]
for net in nets:
relays = helpers.get_connected_relays(net)
for relay in relays:
if net not in channels:
channels[net] = {}
if relay.num not in channels[net]:
channels[net][relay.num] = []
for channel in relay.channels:
channels[net][relay.num].append(channel)
# debug(f"getAllChannels(): {channels}")
return channels
def getDuplicateChannels(net=None, total=False):
"""
Get a list of duplicate channels.
:return: list of duplicate channels
"""
allChans = getAllChannels(net)
duplicates = {}
for net in allChans.keys():
net_chans = []
inst = {}
# add all the channels from this network to a list
for num in allChans[net].keys():
net_chans.extend(allChans[net][num])
for channel in net_chans:
count_chan = net_chans.count(channel)
# I don't know why but it works
# this is used in userinfo.delChannels
set_min = 1
if total:
set_min = 0
if count_chan > set_min:
inst[channel] = count_chan
if inst:
duplicates[net] = inst
if total:
return duplicates
to_part = {}
for net in allChans:
if net in duplicates:
for num in allChans[net].keys():
for channel in allChans[net][num]:
if channel in duplicates[net].keys():
if duplicates[net][channel] > 1:
if net not in to_part:
to_part[net] = {}
if num not in to_part[net]:
to_part[net][num] = []
to_part[net][num].append(channel)
duplicates[net][channel] -= 1
return to_part
def partChannels(data):
for net in data:
for num in data[net]:
name = f"{net}{num}"
if name in main.IRCPool.keys():
for channel in data[net][num]:
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
log(f"Parted {channel} on {net} - {num}")
def getEnabledRelays(net):
"""
Get a list of enabled relays for a network.
:param net: network
:rtype: list of int
:return: list of enabled relay numbers
"""
enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]]
# debug(f"getEnabledRelays() {net}: {enabledRelays}")
return enabledRelays
def getConnectedRelays(net):
"""
Get a list of connected relays for a network.
:param net: network
:rtype: list of int
:return: list of relay numbers
"""
enabledRelays = getEnabledRelays(net)
connectedRelays = []
for i in enabledRelays:
name = net + str(i)
if name in main.IRCPool.keys():
if main.IRCPool[name].isconnected:
connectedRelays.append(i)
# debug(f"getConnectedRelays() {net}: {connectedRelays}")
return connectedRelays
def getActiveRelays(net):
"""
Get a list of active relays for a network.
:param net: network
:rtype: list of int
:return: list of relay numbers
"""
enabledRelays = getEnabledRelays(net)
activeRelays = []
for i in enabledRelays:
name = net + str(i)
if name in main.IRCPool.keys():
# debug(
# (
# f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
# f"conn:{main.IRCPool[name].isconnected}"
# )
# )
if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
activeRelays.append(i)
debug(f"getActiveRelays() {net}: {activeRelays}")
return activeRelays
def relayIsActive(net, num):
"""
Check if a relay is active.
:param net: network
:param num: relay number
:rtype: bool
:return: True if relay is active, False otherwise
"""
activeRelays = getActiveRelays(net)
return num in activeRelays
def allRelaysActive(net):
"""
Check if all enabled relays are active and authenticated.
:param net: network
:rtype: bool
:return: True if all relays are active and authenticated, False otherwise
"""
activeRelays = getActiveRelays(net)
enabledRelays = getEnabledRelays(net)
relaysActive = len(activeRelays) == len(enabledRelays)
# debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
return relaysActive
def getAverageChanlimit(net):
"""
Get the average channel limit for a network.
:param net: network
:rtype: int
:return: average channel limit
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += main.IRCPool[name].chanlimit
avg_chanlimit = total / len(getActiveRelays(net))
debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
return avg_chanlimit
def getSumChanlimit(net):
"""
Get the sum of all channel limits for a network.
:param net: network
:rtype: int
:return: sum of channel limits
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += main.IRCPool[name].chanlimit
return total
def getChanFree(net):
"""
Get a dictionary with the free channel spaces for
each relay, and a channel limit.
Example return:
({1: 99}, 100)
:param net: network
:return: ({relay: channel spaces}, channel limit)
"""
chanfree = {}
for i in getActiveRelays(net):
name = net + str(i)
if name not in main.IRCPool.keys():
continue
if not main.IRCPool[name].isconnected:
continue
chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
return chanfree
def getTotalChans(net):
"""
Get the total number of channels on all relays for a network.
:param net: network
:rtype: int
:return: total number of channels
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += len(main.IRCPool[name].channels)
return total
def emptyChanAllocate(net, flist):
"""
Allocate channels to relays.
:param net: network
:param flist: list of channels to allocate
:param new: list of newly provisioned relays to account for
:rtype: dict
:return: dictionary of {relay: list of channels}"""
# Get the free channel spaces for each relay
chanfree = getChanFree(net)
if not chanfree:
return
# Pretend the newly provisioned relays are already on the network
# for i in new:
# chanfree[0][i] = chanfree[1]
allocated = {}
newlist = list(flist)
chan_slots_used = getTotalChans(net)
max_chans = getSumChanlimit(net) - chan_slots_used
trunc_list = newlist[:max_chans]
debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}")
for i in chanfree.keys():
for x in range(chanfree[i]):
if not len(trunc_list):
break
if i in allocated.keys():
allocated[i].append(trunc_list.pop())
else:
allocated[i] = [trunc_list.pop()]
return allocated
def populateChans(net, clist):
"""
Populate channels on relays.
Stores channels to join in a list in main.TempChan[net][num]
:param net: network
:param clist: list of channels to join
:param new: list of newly provisioned relays to account for"""
# divided = array_split(clist, relay)
allocated = emptyChanAllocate(net, clist)
trace(f"populateChans() allocated:{allocated}")
if not allocated:
return
for i in allocated.keys():
if net in main.TempChan.keys():
main.TempChan[net][i] = allocated[i]
else:
main.TempChan[net] = {i: allocated[i]}
trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
def notifyJoin(net):
"""
Notify relays to join channels.
They will pull from main.TempChan and remove channels they join.
:param net: network
"""
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
trace(f"notifyJoin() {name}")
main.IRCPool[name].checkChannels()
def minifyChans(net, listinfo, as_list=False):
"""
Remove channels from listinfo that are already covered by a relay.
:param net: network
:param listinfo: list of channels to check
:type listinfo: list of [channel, num_users]
:return: list of channels with joined channels removed
:rtype: list of [channel, num_users]
"""
# We want to make this reusable for joining a bunch of channels.
if as_list:
channel_list = listinfo
if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net)
return False
for i in getConnectedRelays(net):
name = net + str(i)
for x in main.IRCPool[name].channels:
if as_list:
for y in channel_list:
if y == x:
channel_list.remove(y)
else:
for y in listinfo:
if y[0] == x:
listinfo.remove(y)
if not as_list:
if not listinfo:
log("We're on all the channels we want to be on, dropping LIST")
return False
if as_list:
return channel_list
else:
return listinfo
def keepChannels(net, listinfo, mean, sigrelay, relay):
"""
Minify channels, determine whether we can cover all the channels
on the network, or need to use 'significant' mode.
Truncate the channel list to available channel spaces.
Allocate these channels to relays.
Notify relays that they should pull from TempChan to join.
:param net: network
:param listinfo: list of [channel, num_users] lists
:param mean: mean of channel population
:param sigrelay: number of relays needed to cover significant channels
:param relay: number of relays needed to cover all channels
:param chanlimit: maximum number of channels to allocate to a relay
"""
listinfo = minifyChans(net, listinfo)
if not listinfo:
return
if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
coverAll = True
elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
coverAll = False
# if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
# error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
# return
num_instances = len(getActiveRelays(net))
debug(f"keepChannels() {net} instances:{num_instances}")
chan_slots_used = getTotalChans(net)
debug(f"keepChannels() slots_used:{chan_slots_used}")
# max_chans = (chanlimit * num_instances) - chan_slots_used
max_chans = getSumChanlimit(net) - chan_slots_used
if max_chans < 0:
max_chans = 0
debug(f"keepChannels() max_chans:{max_chans}")
if coverAll:
# needed = relay - len(getActiveRelays(net))
# if needed:
# debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
# newNums = modules.provision.provisionMultipleRelays(net, needed)
# else:
# newNums = []
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
if len(listinfo_sort) > max_chans:
max_chans = len(listinfo_sort) - 1
flist = [i[0] for i in listinfo_sort]
flist = flist[:max_chans]
debug(f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels")
trace(f"keepChannels() {net}: joining:{flist}")
populateChans(net, flist)
else:
# needed = sigrelay - len(getActiveRelays(net))
# if needed:
# debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
# newNums = modules.provision.provisionMultipleRelays(net, needed)
# else:
# newNums = []
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
if len(listinfo_sort) > max_chans:
max_chans = len(listinfo_sort) - 1
debug(f"keepChannels() {net}: new max_chans:{max_chans}")
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
trace(f"keepChannels() {net}: new siglist:{siglist}")
siglist = siglist[:max_chans]
trace(f"keepChannels() {net}: truncated siglist:{siglist}")
trace(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}")
debug(f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels")
trace(f"keepChannels() {net}: joining:{siglist}")
populateChans(net, siglist)
notifyJoin(net)
def joinSingle(net, channel):
"""
Join a channel on a relay.
Use ECA to determine which relay to join on.
:param net: network
:param channel: channel to join
:return: relay number that joined the channel
:rtype: int
"""
if "," in channel:
channels = channel.split(",")
channels = minifyChans(net, channels, as_list=True)
else:
channels = [channel]
populateChans(net, channels)
notifyJoin(net)
return True
def partSingle(net, channel):
"""
Iterate over all the relays of net and part channels matching channel.
:param net: network
:param channel: channel to part
:return: list of relays that parted the channel
:rtype: list of str
"""
parted = []
for i in getConnectedRelays(net):
name = f"{net}{i}"
if name in main.IRCPool.keys():
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
parted.append(str(i))
return parted
def nukeNetwork(net):
"""
Remove network records.
:param net: network"""
# purgeRecords(net)
# p = main.g.pipeline()
main.g.delete("analytics.list." + net)
# p.delete("list."+net)
# p.execute()
# def nukeNetwork(net):
# deferToThread(_nukeNetwork, net)
def _initialList(net, num, listinfo):
"""
Called when a relay receives a full LIST response.
Run statistics to determine how many channels are significant.
This is done by adding all the numbers of users on the channels together,
then dividing by the number of channels.
* cumul - cumulative sum of all channel membership
* siglength - number of significant channels
* listlength - number of channels in the list
* sigrelay - number of relays needed to cover siglength
* relay - number of relays needed to cover all channels
:param net: network
:param num: relay number
:param listinfo: list of [channel, num_users] lists
:param chanlimit: maximum number of channels the relay can join
"""
listlength = len(listinfo)
cumul = 0
try:
cumul += sum(int(i[1]) for i in listinfo)
except TypeError:
warn("Bad LIST data received from %s - %i" % (net, num))
return
mean = round(cumul / listlength, 2)
siglength = 0
insiglength = 0
sigcumul = 0
insigcumul = 0
for i in listinfo:
if int(i[1]) > mean:
siglength += 1
sigcumul += int(i[1])
elif int(i[1]) < mean:
insiglength += 1
insigcumul += int(i[1])
avg_chanlimit = getAverageChanlimit(net)
sigrelay = ceil(siglength / avg_chanlimit)
relay = ceil(listlength / avg_chanlimit)
cur_relays = len(getActiveRelays(net))
sig_relays_missing = sigrelay - cur_relays
all_relays_missing = relay - cur_relays
abase = "analytics.list.%s" % net
main.g.delete(abase)
p = main.g.pipeline()
# See docstring for meanings
p.hset(abase, "mean", mean)
p.hset(abase, "total_chans", listlength)
p.hset(abase, "big_chans", siglength)
p.hset(abase, "small_chans", insiglength)
p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
p.hset(abase, "total_cumul_mem", cumul)
p.hset(abase, "big_chan_cumul_mem", sigcumul)
p.hset(abase, "small_chan_cumul_mem", insigcumul)
p.hset(abase, "relays_for_all_chans", relay)
p.hset(abase, "relays_for_big_chans", sigrelay)
p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
p.hset(abase, "sig_relays_missing", sig_relays_missing)
p.hset(abase, "all_relays_missing", all_relays_missing)
debug(
(
f"_initialList() net:{net} num:{num} listlength:{listlength} "
f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
)
)
# Purge existing records before writing
# purgeRecords(net)
# for i in listinfo:
# p.rpush(netbase+"."+i[0], i[1])
# p.rpush(netbase+"."+i[0], i[2])
# p.sadd(netbase, i[0])
p.execute()
debug("List parsing completed on %s" % net)
keepChannels(net, listinfo, mean, sigrelay, relay)
# return (listinfo, mean, sigrelay, relay)
def convert(data):
"""
Recursively convert a dictionary.
"""
if isinstance(data, bytes):
return data.decode("ascii")
if isinstance(data, dict):
return dict(map(convert, data.items()))
if isinstance(data, tuple):
return map(convert, data)
if isinstance(data, list):
return list(map(convert, data))
return data
def getListInfo(net):
abase = f"analytics.list.{net}"
info = main.g.hgetall(abase)
return convert(info)
def initialList(net, num, listinfo):
"""
Run _initialList in a thread.
See above docstring.
"""
deferToThread(_initialList, net, num, deepcopy(listinfo))

44
legacy/modules/counters.py Normal file
View File

@@ -0,0 +1,44 @@
from twisted.internet.task import LoopingCall
import main
def event(name, eventType):
if "local" not in main.counters.keys():
main.counters["local"] = {}
if "global" not in main.counters.keys():
main.counters["global"] = {}
if name not in main.counters["local"].keys():
main.counters["local"][name] = {}
if eventType not in main.counters["local"][name].keys():
main.counters["local"][name][eventType] = 0
if eventType not in main.counters["global"]:
main.counters["global"][eventType] = 0
main.counters["local"][name][eventType] += 1
main.counters["global"][eventType] += 1
main.runningSample += 1
def getEvents(name=None):
if name is None:
if "global" in main.counters.keys():
return main.counters["global"]
else:
return None
else:
if name in main.counters["local"].keys():
return main.counters["local"][name]
else:
return None
def takeSample():
main.lastMinuteSample = main.runningSample
main.runningSample = 0
def setupCounterLoop():
lc = LoopingCall(takeSample)
lc.start(60)

72
legacy/modules/helpers.py Normal file
View File

@@ -0,0 +1,72 @@
import main
from modules import chankeep
from utils.logging.debug import debug
def get_first_relay(net):
"""
Get the first relay in the network.
:param net: the network
:param num: number or relay
:return: IRCPool instance for the IRC bot
"""
cur_relay = 0
max_relay = max(main.network[net].relays.keys())
debug(f"get_first_relay() {net}: max_relay:{max_relay}")
activeRelays = chankeep.getActiveRelays(net)
debug(f"get_first_relay() {net}: activeRelays:{activeRelays}")
while cur_relay != max_relay:
cur_relay += 1
if cur_relay not in activeRelays:
continue
name = net + str(cur_relay)
if name in main.IRCPool.keys():
# debug(f"get_first_relay() {net}: found relay {name}")
return main.IRCPool[name]
return None
def is_first_relay(net, num):
"""
Determine if we are the first relay for the network.
:param net: the network
:param num: number or relay
:return: True if we are the first relay, False otherwise
"""
first_relay = get_first_relay(net)
if not first_relay:
return False
return first_relay.num == num
def get_active_relays(net):
"""
Get all active instances for the network.
:param net: the network
:return: list of active instances
:rtype: list of IRCPool instances
"""
active_nums = chankeep.getActiveRelays(net)
active_insts = []
for num in active_nums:
name = net + str(num)
if name in main.IRCPool.keys():
active_insts.append(main.IRCPool[name])
return active_insts
def get_connected_relays(net):
"""
Get all connected instances for the network.
:param net: the network
:return: list of active instances
:rtype: list of IRCPool instances
"""
active_nums = chankeep.getConnectedRelays(net)
active_insts = []
for num in active_nums:
name = net + str(num)
if name in main.IRCPool.keys():
active_insts.append(main.IRCPool[name])
return active_insts

80
legacy/modules/monitor.py Normal file
View File

@@ -0,0 +1,80 @@
import main
from core.logstash import sendLogstashNotification
from core.relay import sendRelayNotification
from modules import userinfo
from utils.dedup import dedup
order = [
"type",
"net",
"num",
"channel",
"msg",
"nick",
"ident",
"host",
"mtype",
"user",
"mode",
"modearg",
"realname",
"server",
"status",
"ts",
]
def parsemeta(numName, c):
if "channel" not in c.keys():
c["channel"] = None
# metadata scraping
# need to check if this was received from a relay
# in which case, do not do this
if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
if "muser" in c.keys():
userinfo.editUser(c["net"], c["muser"])
# if c["type"] == "mode":
# userinfo.updateMode(c)
elif c["type"] == "nick":
userinfo.renameUser(
c["net"],
c["nick"],
c["muser"],
c["user"],
c["user"] + "!" + c["ident"] + "@" + c["host"],
)
elif c["type"] == "kick":
userinfo.editUser(c["net"], c["muser"])
userinfo.delUserByNick(c["net"], c["channel"], c["user"])
elif c["type"] == "quit":
userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"])
elif c["type"] == "join":
userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"])
elif c["type"] == "part":
userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"])
if "mtype" in c.keys():
if c["mtype"] == "nick":
userinfo.renameUser(
c["net"],
c["nick"],
c["muser"],
c["user"],
c["user"] + "!" + c["ident"] + "@" + c["host"],
)
def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
if dedup(numName, c):
return
# make a copy of the object with dict() to prevent sending notifications with channel of None
parsemeta(numName, dict(c))
if "muser" in c.keys():
del c["muser"]
sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order
sortedKeys["src"] = "irc"
if main.config["Logstash"]["Enabled"]:
sendLogstashNotification(sortedKeys)
sendRelayNotification(sortedKeys)

164
legacy/modules/network.py Normal file
View File

@@ -0,0 +1,164 @@
from copy import deepcopy
from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory
import main
from core.bot import IRCBotFactory
from modules import alias
from modules.chankeep import nukeNetwork
from modules.provision import provisionRelay
from modules.regproc import needToRegister
from utils.deliver_relay_commands import deliverRelayCommands
from utils.get import getRelay
from utils.logging.log import log
def migrate():
existing = deepcopy(main.network)
log("Migrating network configuration")
log(f"Existing network configuration: {existing}")
for net, net_inst in existing.items():
log(f"Migrating network {net}")
net = net_inst.net
host = net_inst.host
port = net_inst.port
security = net_inst.security
auth = net_inst.auth
last = net_inst.last
relays = net_inst.relays
aliases = net_inst.aliases
new_net = Network(net, host, port, security, auth)
log(f"New network for {net}: {new_net}")
new_net.last = last
new_net.relays = relays
new_net.aliases = aliases
main.network[net] = new_net
main.saveConf("network")
log("Finished migrating network configuration")
class Network:
def __init__(self, net, host, port, security, auth):
self.net = net
self.host = host
self.port = port
self.security = security
self.auth = auth
self.chanlimit = None
self.last = 1
self.relays = {}
self.aliases = {}
def add_relay(self, num=None):
# Grrrrrrrrrr
self.last = int(self.last)
if not num:
num = self.last
self.last += 1
elif num == self.last:
self.last += 1
registered = False
if not needToRegister(self.net):
registered = True
# Don't need to register if it's been disabled in definitions,
# so we'll pretend we already did
self.relays[num] = {
"enabled": main.config["ConnectOnCreate"],
"net": self.net,
"id": num,
"registered": registered,
}
password = alias.generate_password()
if num not in main.alias.keys():
main.alias[num] = alias.generate_alias()
main.saveConf("alias")
self.aliases[num] = {"password": password}
# if main.config["ConnectOnCreate"]: -- Done in provision
# self.start_bot(num)
provisionRelay(num, self.net)
return num, main.alias[num]["nick"]
def enable_relay(self, num):
"""
Enable a relay for this network.
Send a command to ZNC to connect.
"""
self.relays[num]["enabled"] = True
user = main.alias[num]["nick"]
commands = {"status": ["Connect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name not in main.IRCPool.keys():
self.start_bot(num)
def disable_relay(self, num):
"""
Disable a relay for this network.
Send a command to ZNC to disconnect.
Stop trying to connect to the relay.
"""
self.relays[num]["enabled"] = False
user = main.alias[num]["nick"]
# relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying()
main.ReactorPool[name].disconnect()
if name in main.IRCPool.keys():
del main.IRCPool[name]
del main.ReactorPool[name]
del main.FactoryPool[name]
def killAliases(self, aliasList):
for i in aliasList:
name = self.net + str(i)
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying()
main.ReactorPool[name].disconnect()
if name in main.IRCPool.keys():
del main.IRCPool[name]
del main.ReactorPool[name]
del main.FactoryPool[name]
def delete_relay(self, id):
del self.relays[id]
del self.aliases[id]
# del main.alias[id] - Aliases are global per num, so don't delete them!
self.killAliases([id])
def seppuku(self):
# Removes all bots in preperation for deletion
self.killAliases(self.relays.keys())
nukeNetwork(self.net)
def start_bot(self, num):
# a single name is given to relays in the backend
# e.g. freenode1 for the first relay on freenode network
keyFN = main.certPath + main.config["Key"]
certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(
keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
)
bot = IRCBotFactory(self.net, num)
# host, port = self.relays[num]["host"], self.relays[num]["port"]
host, port = getRelay(num)
rct = reactor.connectSSL(host, port, bot, contextFactory)
name = self.net + str(num)
main.ReactorPool[name] = rct
main.FactoryPool[name] = bot
log("Started bot on relay %s on %s" % (num, self.host))
def start_bots(self):
for num in self.relays.keys():
if self.relays[num]["enabled"]:
self.start_bot(num)

95
legacy/modules/provision.py Normal file
View File

@@ -0,0 +1,95 @@
from twisted.internet import reactor
import main
import modules.regproc
from utils.deliver_relay_commands import deliverRelayCommands
from utils.logging.log import warn
def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security):
commands = {}
stage2commands = {}
stage2commands["status"] = []
commands["controlpanel"] = []
user = nick.lower()
commands["controlpanel"].append("AddUser %s %s" % (user, main.config["Relay"]["Password"]))
commands["controlpanel"].append("AddNetwork %s %s" % (user, network))
commands["controlpanel"].append("Set Nick %s %s" % (user, nick))
commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick))
commands["controlpanel"].append("Set Ident %s %s" % (user, ident))
commands["controlpanel"].append("Set RealName %s %s" % (user, realname))
if security == "ssl":
commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me
commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port))
elif security == "plain":
commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port))
if not main.config["ConnectOnCreate"]:
stage2commands["status"].append("Disconnect")
if main.config["Toggles"]["CycleChans"]:
stage2commands["status"].append("LoadMod disconkick")
stage2commands["status"].append("LoadMod chansaver")
inst = modules.regproc.selectInst(network)
if "setmode" in inst.keys():
stage2commands["status"].append("LoadMod perform")
# stage2commands["perform"].append("add mode %nick% +"+inst["setmode"])
deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]])
def provisionAuthenticationData(num, nick, network, auth, password):
commands = {}
commands["status"] = []
user = nick.lower()
if auth == "sasl":
commands["sasl"] = []
commands["status"].append("UnloadMod nickserv")
commands["status"].append("LoadMod sasl")
commands["sasl"].append("Mechanism plain")
commands["sasl"].append("Set %s %s" % (nick, password))
elif auth == "ns":
commands["nickserv"] = []
commands["status"].append("UnloadMod sasl")
commands["status"].append("LoadMod nickserv")
commands["nickserv"].append("Set %s" % password)
inst = modules.regproc.selectInst(network)
if "setmode" in inst.keys():
# perform is loaded above
# commands["status"].append("LoadMod perform")
commands["perform"] = ["add mode %nick% +" + inst["setmode"]]
deliverRelayCommands(num, commands, user=user + "/" + network)
def provisionRelay(num, network): # provision user and network data
aliasObj = main.alias[num]
# alias = aliasObj["nick"]
nick = aliasObj["nick"]
altnick = aliasObj["altnick"]
ident = aliasObj["ident"]
realname = aliasObj["realname"]
provisionUserNetworkData(
num,
nick,
altnick,
ident,
realname,
network,
main.network[network].host,
main.network[network].port,
main.network[network].security,
)
if main.config["ConnectOnCreate"]:
reactor.callLater(10, main.network[network].start_bot, num)
def provisionMultipleRelays(net, relaysNeeded):
if not relaysNeeded:
return []
if not main.config["ChanKeep"]["Provision"]:
warn(f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled")
return []
numsProvisioned = []
for i in range(relaysNeeded):
num, alias = main.network[net].add_relay()
numsProvisioned.append(num)
provisionRelay(num, net)
main.saveConf("network")
return numsProvisioned

233
legacy/modules/regproc.py Normal file
View File

@@ -0,0 +1,233 @@
from copy import deepcopy
from random import choice
import main
from modules import provision
from utils.logging.debug import debug
from utils.logging.log import error
def needToRegister(net):
# Check if the network does not support authentication
networkObj = main.network[net]
if networkObj.auth == "none":
return False
# Check if the IRC network definition has registration disabled
inst = selectInst(net)
if "register" in inst.keys():
if inst["register"]:
return True
else:
return False
def needToAuth(net):
networkObj = main.network[net]
if networkObj.auth == "none":
return False
return True
def selectInst(net):
if net in main.irc.keys():
inst = deepcopy(main.irc[net])
for i in main.irc["_"].keys():
if i not in inst:
inst[i] = main.irc["_"][i]
else:
inst = deepcopy(main.irc["_"])
return inst
def substitute(net, num, token=None):
inst = selectInst(net)
alias = main.alias[num]
gotemail = False
if "emails" in alias:
# First priority is explicit email lists
if alias["emails"]:
email = choice(alias["emails"])
gotemail = True
if "domains" in inst:
if inst["domains"]:
if not gotemail:
domain = choice(inst["domains"])
email = f"{alias['nickname']}@{domain}"
gotemail = True
if not gotemail:
inst["email"] = False
nickname = alias["nick"]
# username = nickname + "/" + net
password = main.network[net].aliases[num]["password"]
# inst["email"] = inst["email"].replace("{nickname}", nickname)
name = f"{net}{num}"
if name in main.IRCPool:
curnick = main.IRCPool[name].nickname
else:
curnick = nickname
for i in inst.keys():
if not isinstance(inst[i], str):
continue
inst[i] = inst[i].replace("{nickname}", nickname)
inst[i] = inst[i].replace("{curnick}", curnick)
inst[i] = inst[i].replace("{password}", password)
if gotemail:
inst[i] = inst[i].replace("{email}", email)
if token:
inst[i] = inst[i].replace("{token}", token)
return inst
def registerAccount(net, num):
debug("Attempting to register: %s - %i" % (net, num))
sinst = substitute(net, num)
if not sinst:
error(f"Register account failed for {net} - {num}")
return
if not sinst["email"]:
error(f"Could not get email for {net} - {num}")
return
if not sinst["register"]:
error("Cannot register for %s: function disabled" % (net))
return False
name = net + str(num)
if not main.IRCPool[name].authenticated:
main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"])
def confirmAccount(net, num, token):
sinst = substitute(net, num, token=token)
name = net + str(num)
if name in main.IRCPool:
main.IRCPool[name].msg(sinst["entity"], sinst["confirm"])
enableAuthentication(net, num)
def confirmRegistration(net, num, negativepass=None):
obj = main.network[net]
name = net + str(num)
if name in main.IRCPool.keys():
if negativepass is not None:
main.IRCPool[name].regPing(negativepass=negativepass)
return
debug("Relay authenticated: %s - %i" % (net, num))
main.IRCPool[name].authenticated = True
main.IRCPool[name].recheckList()
if obj.relays[num]["registered"]:
return
if name in main.IRCPool.keys():
if main.IRCPool[name]._regAttempt:
try:
main.IRCPool[name]._regAttempt.cancel()
except: # noqa
pass
obj.relays[num]["registered"] = True
main.saveConf("network")
def attemptManualAuthentication(net, num):
sinst = substitute(net, num)
identifymsg = sinst["identifymsg"]
entity = sinst["entity"]
name = f"{net}{num}"
if name not in main.IRCPool:
return
main.IRCPool[name].sendmsg(entity, identifymsg, in_query=True)
def enableAuthentication(net, num, jump=True, run_now=False):
obj = main.network[net]
nick = main.alias[num]["nick"]
auth = obj.auth
name = f"{net}{num}"
if name not in main.IRCPool:
return
# uname = main.alias[num]["nick"] + "/" + net
password = main.network[net].aliases[num]["password"]
provision.provisionAuthenticationData(num, nick, net, auth, password) # Set up for auth
if jump:
main.IRCPool[name].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump")
if run_now:
attemptManualAuthentication(net, num)
if selectInst(net)["check"] is False:
confirmRegistration(net, num)
def get_unregistered_relays(net=None):
"""
Get a dict of unregistereed relays, either globally or
for a network.
Returns:
{"net": [["nick1", 1], ["nick2", 2], ...]}
"""
unreg = {}
if net:
nets = [net]
else:
nets = main.network.keys()
for i in nets:
for num in main.network[i].relays.keys():
if not main.network[i].relays[num]["registered"]:
nick = main.alias[num]["nick"]
if i in unreg:
unreg[i].append([nick, num])
else:
unreg[i] = [[nick, num]]
return unreg
def registerTest(c):
sinst = substitute(c["net"], c["num"])
name = c["net"] + str(c["num"])
net = c["net"]
num = c["num"]
if sinst["check"] is False:
return
if "msg" in c.keys() and not c["msg"] is None:
if sinst["negative"]:
if name in main.IRCPool.keys():
if main.IRCPool[name]._negativePass is not True:
if c["type"] == "query" and c["nick"] == sinst["entity"]:
if sinst["checknegativemsg"] in c["msg"]:
confirmRegistration(
c["net"], c["num"], negativepass=False
) # Not passed negative check, report back
return
if sinst["checkendnegative"] in c["msg"]:
confirmRegistration(
c["net"], c["num"], negativepass=True
) # Passed the negative check, report back
return
if sinst["ping"]:
if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]:
confirmRegistration(c["net"], c["num"])
debug(
(
f"registerTest() {net} - {num} passed ping:checkmsg2 "
f"check, {sinst['checkmsg2']} present in message"
)
)
return
if sinst["checktype"] == "msg":
if sinst["checkmsg"] in c["msg"]:
confirmRegistration(c["net"], c["num"])
debug(
(
f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, "
f"{sinst['checkmsg']} present in message"
)
)
return
elif sinst["checktype"] == "mode":
if c["type"] == "self":
if c["mtype"] == "mode":
if sinst["checkmode"] in c["mode"] and c["status"] is True:
confirmRegistration(c["net"], c["num"])
debug(
(
f"registerTest() {net} - {num} passed checktype:mode:checkmost check, "
f"{sinst['checkmode']} present in mode"
)
)
return

304
legacy/modules/userinfo.py Normal file
View File

@@ -0,0 +1,304 @@
from twisted.internet.threads import deferToThread
import main
from modules import chankeep
from utils.logging.debug import debug, trace
from utils.logging.log import warn
from utils.parsing import parsen
def getWhoSingle(name, query):
result = main.r.sscan("live.who." + name, 0, query, count=999999)
if result[1] == []:
return None
return (i.decode() for i in result[1])
def getWho(query):
result = {}
for i in main.network.keys():
f = getWhoSingle(i, query)
if f:
result[i] = f
return result
def getChansSingle(name, nick):
nick = ("live.chan." + name + "." + i for i in nick)
result = main.r.sinter(*nick)
if len(result) == 0:
return None
return (i.decode() for i in result)
def getChanList(name, nick):
chanspace = "live.chan." + name + "." + nick
result = main.r.smembers(chanspace)
if len(result) == 0:
return None
return (i.decode() for i in result)
def getTotalChanNum(net):
"""
Get the number of channels a network has.
"""
chans = main.r.keys(f"live.who.{net}.*")
return len(chans)
def getUserNum(name, channels):
"""
Get the number of users on a list of channels.
"""
chanspace = ("live.who." + name + "." + i for i in channels)
results = {}
for channel, space in zip(channels, chanspace):
results[channel] = main.r.scard(space)
return results
def getChanNum(name, nicks):
"""
Get the number of channels a list of users are on.
"""
nickspace = ("live.chan." + name + "." + i for i in nicks)
results = {}
for nick, space in zip(nicks, nickspace):
results[nick] = main.r.scard(space)
return results
def getChans(nick):
result = {}
for i in main.network.keys():
f = getChansSingle(i, nick)
if f:
result[i] = f
return result
def getUsersSingle(name, nick):
nick = ("live.who." + name + "." + i for i in nick)
result = main.r.sinter(*nick)
if len(result) == 0:
return None
return (i.decode() for i in result)
def getUsers(nick):
result = {}
for i in main.network.keys():
f = getUsersSingle(i, nick)
if f:
result[i] = f
return result
def getNumWhoEntries(name):
return main.r.scard("live.who." + name)
def getNumTotalWhoEntries():
total = 0
for i in main.network.keys():
total += getNumWhoEntries(i)
return total
def getNamespace(name, channel, nick):
gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel)
chanspace = "live.chan.%s.%s" % (name, nick)
mapspace = "live.map.%s" % name
return (gnamespace, namespace, chanspace, mapspace)
def _initialUsers(name, channel, users):
gnamespace = "live.who.%s" % name
mapspace = "live.map.%s" % name
p = main.r.pipeline()
for i in users:
user = i[0] + "!" + i[1] + "@" + i[2]
p.hset(mapspace, i[0], user)
p.sadd(gnamespace, user)
p.execute()
def initialUsers(name, channel, users):
trace("Initialising WHO records for %s on %s" % (channel, name))
deferToThread(_initialUsers, name, channel, users)
# d.addCallback(testCallback)
def _initialNames(name, channel, names):
namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline()
for mode, nick in names:
p.sadd(namespace, nick)
p.sadd("live.chan." + name + "." + nick, channel)
if mode:
p.hset("live.prefix." + name + "." + channel, nick, mode)
p.execute()
def initialNames(name, channel, names):
trace("Initialising NAMES records for %s on %s" % (channel, name))
deferToThread(_initialNames, name, channel, names)
# d.addCallback(testCallback)
def editUser(name, user):
gnamespace = "live.who.%s" % name
mapspace = "live.map.%s" % name
parsed = parsen(user)
p = main.r.pipeline()
p.sadd(gnamespace, user)
p.hset(mapspace, parsed[0], user) # add nick -> user mapping
p.execute()
def addUser(name, channel, nick, user):
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
p.sadd(gnamespace, user)
p.sadd(namespace, nick)
p.sadd(chanspace, channel)
p.hset(mapspace, nick, user)
p.execute()
def delUser(name, channel, nick, user):
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
channels = main.r.smembers(chanspace)
p.srem(namespace, nick)
if channels == {channel.encode()}: # can we only see them on this channel?
p.delete(chanspace) # remove channel tracking entry
p.hdel("live.prefix." + name + "." + channel, nick) # remove prefix tracking entry
p.hdel(mapspace, nick) # remove nick mapping entry
if user:
p.srem(gnamespace, user) # remove global userinfo entry
else:
warn("Attempt to delete nonexistent user: %s" % user)
else:
p.srem(chanspace, channel) # keep up - remove the channel from their list
p.execute()
def escape(text):
chars = ["[", "]", "^", "-", "*", "?"]
text = text.replace("\\", "\\\\")
for i in chars:
text = text.replace(i, "\\" + i)
return text
def getUserByNick(name, nick):
gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
mapspace = "live.map.%s" % name
if main.r.hexists(mapspace, nick):
return main.r.hget(mapspace, nick)
else:
warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace))
# return False
# legacy code below - remove when map is reliable
usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999)
if usermatch[1] == []:
warn("No matches found for user query: %s on %s" % (nick, name))
return False
else:
if len(usermatch[1]) == 1:
user = usermatch[1][0]
return user
else:
warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
return False
def renameUser(name, oldnick, olduser, newnick, newuser):
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, oldnick)
mapspace = "live.map.%s" % name
newchanspace = "live.chan.%s.%s" % (name, newnick)
p = main.r.pipeline()
p.srem(gnamespace, olduser)
p.sadd(gnamespace, newuser)
for i in main.r.smembers(chanspace):
i = i.decode()
p.srem("live.who." + name + "." + i, oldnick)
p.sadd("live.who." + name + "." + i, newnick)
p.hdel(mapspace, oldnick)
p.hset(mapspace, newnick, newuser)
if main.r.exists("live.prefix." + name + "." + i): # if there's a prefix entry for the channel
if main.r.hexists("live.prefix." + name + "." + i, oldnick): # if the old nick is in it
mode = main.r.hget("live.prefix." + name + "." + i, oldnick) # retrieve old modes
p.hset("live.prefix." + name + "." + i, newnick, mode) # set old modes to new nickname
if main.r.exists(chanspace):
p.rename(chanspace, newchanspace)
else:
warn("Key doesn't exist: %s" % chanspace)
p.execute()
def delUserByNick(name, channel, nick): # kick
user = getUserByNick(name, nick)
if not user:
return
delUser(name, channel, nick, user)
def delUserByNetwork(name, nick, user): # quit
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, nick)
mapspace = "live.chan.%s" % name
p = main.r.pipeline()
p.srem(gnamespace, user)
for i in main.r.smembers(chanspace):
p.srem("live.who." + name + "." + i.decode(), nick)
p.hdel("live.prefix." + name + "." + i.decode(), nick)
p.delete(chanspace)
p.hdel(mapspace, nick)
p.execute()
def _delChannels(net, channels):
gnamespace = "live.who.%s" % net
mapspace = "live.map.%s" % net
p = main.r.pipeline()
for channel in channels:
namespace = "live.who.%s.%s" % (net, channel)
for i in main.r.smembers(namespace):
nick = i.decode()
# user = getUserByNick(net, nick) -- far too many function calls
user = main.r.hget(mapspace, nick)
if not user:
warn("User lookup failed: %s on %s" % (nick, net))
if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}:
if user:
p.srem(gnamespace, user)
p.delete("live.chan." + net + "." + nick)
p.hdel(mapspace, nick) # remove map entry
else:
p.srem("live.chan." + net + "." + nick, channel)
p.delete(namespace)
p.delete("live.prefix." + net + "." + channel)
p.execute()
def delChannels(net, channels): # we have left a channel
trace("Purging channel %s for %s" % (", ".join(channels), net))
dupes = chankeep.getDuplicateChannels(net, total=True)
print("dupes: %s" % dupes)
if not dupes:
deferToThread(_delChannels, net, channels)
else:
for channel in channels:
if channel in dupes[net]:
if dupes[net][channel] != 0:
channels.remove(channel)
debug(f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it")
deferToThread(_delChannels, net, channels)
# d.addCallback(testCallback)