Moved files to subdirectory
This commit is contained in:
585
legacy/modules/chankeep.py
Normal file
585
legacy/modules/chankeep.py
Normal file
@@ -0,0 +1,585 @@
|
||||
from copy import deepcopy
|
||||
from math import ceil
|
||||
|
||||
from twisted.internet.threads import deferToThread
|
||||
|
||||
import main
|
||||
from modules import helpers
|
||||
from utils.logging.debug import debug, trace
|
||||
from utils.logging.log import error, log, warn
|
||||
|
||||
|
||||
def getAllChannels(net=None):
|
||||
"""
|
||||
Get a list of all channels on all relays.
|
||||
:return: list of channels
|
||||
"""
|
||||
channels = {}
|
||||
if not net:
|
||||
nets = main.network.keys()
|
||||
else:
|
||||
nets = [net]
|
||||
for net in nets:
|
||||
relays = helpers.get_connected_relays(net)
|
||||
for relay in relays:
|
||||
if net not in channels:
|
||||
channels[net] = {}
|
||||
if relay.num not in channels[net]:
|
||||
channels[net][relay.num] = []
|
||||
for channel in relay.channels:
|
||||
channels[net][relay.num].append(channel)
|
||||
# debug(f"getAllChannels(): {channels}")
|
||||
return channels
|
||||
|
||||
|
||||
def getDuplicateChannels(net=None, total=False):
|
||||
"""
|
||||
Get a list of duplicate channels.
|
||||
:return: list of duplicate channels
|
||||
"""
|
||||
allChans = getAllChannels(net)
|
||||
duplicates = {}
|
||||
for net in allChans.keys():
|
||||
net_chans = []
|
||||
inst = {}
|
||||
# add all the channels from this network to a list
|
||||
for num in allChans[net].keys():
|
||||
net_chans.extend(allChans[net][num])
|
||||
for channel in net_chans:
|
||||
count_chan = net_chans.count(channel)
|
||||
# I don't know why but it works
|
||||
# this is used in userinfo.delChannels
|
||||
set_min = 1
|
||||
if total:
|
||||
set_min = 0
|
||||
if count_chan > set_min:
|
||||
inst[channel] = count_chan
|
||||
if inst:
|
||||
duplicates[net] = inst
|
||||
|
||||
if total:
|
||||
return duplicates
|
||||
|
||||
to_part = {}
|
||||
for net in allChans:
|
||||
if net in duplicates:
|
||||
for num in allChans[net].keys():
|
||||
for channel in allChans[net][num]:
|
||||
if channel in duplicates[net].keys():
|
||||
if duplicates[net][channel] > 1:
|
||||
if net not in to_part:
|
||||
to_part[net] = {}
|
||||
if num not in to_part[net]:
|
||||
to_part[net][num] = []
|
||||
to_part[net][num].append(channel)
|
||||
duplicates[net][channel] -= 1
|
||||
|
||||
return to_part
|
||||
|
||||
|
||||
def partChannels(data):
|
||||
for net in data:
|
||||
for num in data[net]:
|
||||
name = f"{net}{num}"
|
||||
if name in main.IRCPool.keys():
|
||||
for channel in data[net][num]:
|
||||
if channel in main.IRCPool[name].channels:
|
||||
main.IRCPool[name].part(channel)
|
||||
log(f"Parted {channel} on {net} - {num}")
|
||||
|
||||
|
||||
def getEnabledRelays(net):
|
||||
"""
|
||||
Get a list of enabled relays for a network.
|
||||
:param net: network
|
||||
:rtype: list of int
|
||||
:return: list of enabled relay numbers
|
||||
"""
|
||||
enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]]
|
||||
# debug(f"getEnabledRelays() {net}: {enabledRelays}")
|
||||
return enabledRelays
|
||||
|
||||
|
||||
def getConnectedRelays(net):
|
||||
"""
|
||||
Get a list of connected relays for a network.
|
||||
:param net: network
|
||||
:rtype: list of int
|
||||
:return: list of relay numbers
|
||||
"""
|
||||
enabledRelays = getEnabledRelays(net)
|
||||
connectedRelays = []
|
||||
for i in enabledRelays:
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
if main.IRCPool[name].isconnected:
|
||||
connectedRelays.append(i)
|
||||
# debug(f"getConnectedRelays() {net}: {connectedRelays}")
|
||||
return connectedRelays
|
||||
|
||||
|
||||
def getActiveRelays(net):
|
||||
"""
|
||||
Get a list of active relays for a network.
|
||||
:param net: network
|
||||
:rtype: list of int
|
||||
:return: list of relay numbers
|
||||
"""
|
||||
enabledRelays = getEnabledRelays(net)
|
||||
activeRelays = []
|
||||
for i in enabledRelays:
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
# debug(
|
||||
# (
|
||||
# f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
|
||||
# f"conn:{main.IRCPool[name].isconnected}"
|
||||
# )
|
||||
# )
|
||||
if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
|
||||
activeRelays.append(i)
|
||||
debug(f"getActiveRelays() {net}: {activeRelays}")
|
||||
return activeRelays
|
||||
|
||||
|
||||
def relayIsActive(net, num):
|
||||
"""
|
||||
Check if a relay is active.
|
||||
:param net: network
|
||||
:param num: relay number
|
||||
:rtype: bool
|
||||
:return: True if relay is active, False otherwise
|
||||
"""
|
||||
activeRelays = getActiveRelays(net)
|
||||
return num in activeRelays
|
||||
|
||||
|
||||
def allRelaysActive(net):
|
||||
"""
|
||||
Check if all enabled relays are active and authenticated.
|
||||
:param net: network
|
||||
:rtype: bool
|
||||
:return: True if all relays are active and authenticated, False otherwise
|
||||
"""
|
||||
activeRelays = getActiveRelays(net)
|
||||
enabledRelays = getEnabledRelays(net)
|
||||
relaysActive = len(activeRelays) == len(enabledRelays)
|
||||
# debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
|
||||
return relaysActive
|
||||
|
||||
|
||||
def getAverageChanlimit(net):
|
||||
"""
|
||||
Get the average channel limit for a network.
|
||||
:param net: network
|
||||
:rtype: int
|
||||
:return: average channel limit
|
||||
"""
|
||||
total = 0
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
total += main.IRCPool[name].chanlimit
|
||||
avg_chanlimit = total / len(getActiveRelays(net))
|
||||
debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
|
||||
return avg_chanlimit
|
||||
|
||||
|
||||
def getSumChanlimit(net):
|
||||
"""
|
||||
Get the sum of all channel limits for a network.
|
||||
:param net: network
|
||||
:rtype: int
|
||||
:return: sum of channel limits
|
||||
"""
|
||||
total = 0
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
total += main.IRCPool[name].chanlimit
|
||||
return total
|
||||
|
||||
|
||||
def getChanFree(net):
|
||||
"""
|
||||
Get a dictionary with the free channel spaces for
|
||||
each relay, and a channel limit.
|
||||
Example return:
|
||||
({1: 99}, 100)
|
||||
:param net: network
|
||||
:return: ({relay: channel spaces}, channel limit)
|
||||
"""
|
||||
chanfree = {}
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
if name not in main.IRCPool.keys():
|
||||
continue
|
||||
if not main.IRCPool[name].isconnected:
|
||||
continue
|
||||
chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
|
||||
|
||||
return chanfree
|
||||
|
||||
|
||||
def getTotalChans(net):
|
||||
"""
|
||||
Get the total number of channels on all relays for a network.
|
||||
:param net: network
|
||||
:rtype: int
|
||||
:return: total number of channels
|
||||
"""
|
||||
total = 0
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
total += len(main.IRCPool[name].channels)
|
||||
return total
|
||||
|
||||
|
||||
def emptyChanAllocate(net, flist):
|
||||
"""
|
||||
Allocate channels to relays.
|
||||
:param net: network
|
||||
:param flist: list of channels to allocate
|
||||
:param new: list of newly provisioned relays to account for
|
||||
:rtype: dict
|
||||
:return: dictionary of {relay: list of channels}"""
|
||||
|
||||
# Get the free channel spaces for each relay
|
||||
chanfree = getChanFree(net)
|
||||
if not chanfree:
|
||||
return
|
||||
|
||||
# Pretend the newly provisioned relays are already on the network
|
||||
# for i in new:
|
||||
# chanfree[0][i] = chanfree[1]
|
||||
allocated = {}
|
||||
|
||||
newlist = list(flist)
|
||||
chan_slots_used = getTotalChans(net)
|
||||
max_chans = getSumChanlimit(net) - chan_slots_used
|
||||
trunc_list = newlist[:max_chans]
|
||||
debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}")
|
||||
|
||||
for i in chanfree.keys():
|
||||
for x in range(chanfree[i]):
|
||||
if not len(trunc_list):
|
||||
break
|
||||
if i in allocated.keys():
|
||||
allocated[i].append(trunc_list.pop())
|
||||
else:
|
||||
allocated[i] = [trunc_list.pop()]
|
||||
return allocated
|
||||
|
||||
|
||||
def populateChans(net, clist):
|
||||
"""
|
||||
Populate channels on relays.
|
||||
Stores channels to join in a list in main.TempChan[net][num]
|
||||
:param net: network
|
||||
:param clist: list of channels to join
|
||||
:param new: list of newly provisioned relays to account for"""
|
||||
# divided = array_split(clist, relay)
|
||||
allocated = emptyChanAllocate(net, clist)
|
||||
trace(f"populateChans() allocated:{allocated}")
|
||||
if not allocated:
|
||||
return
|
||||
for i in allocated.keys():
|
||||
if net in main.TempChan.keys():
|
||||
main.TempChan[net][i] = allocated[i]
|
||||
else:
|
||||
main.TempChan[net] = {i: allocated[i]}
|
||||
trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
|
||||
|
||||
|
||||
def notifyJoin(net):
|
||||
"""
|
||||
Notify relays to join channels.
|
||||
They will pull from main.TempChan and remove channels they join.
|
||||
:param net: network
|
||||
"""
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
trace(f"notifyJoin() {name}")
|
||||
main.IRCPool[name].checkChannels()
|
||||
|
||||
|
||||
def minifyChans(net, listinfo, as_list=False):
|
||||
"""
|
||||
Remove channels from listinfo that are already covered by a relay.
|
||||
:param net: network
|
||||
:param listinfo: list of channels to check
|
||||
:type listinfo: list of [channel, num_users]
|
||||
:return: list of channels with joined channels removed
|
||||
:rtype: list of [channel, num_users]
|
||||
"""
|
||||
# We want to make this reusable for joining a bunch of channels.
|
||||
if as_list:
|
||||
channel_list = listinfo
|
||||
|
||||
if not allRelaysActive(net):
|
||||
error("All relays for %s are not active, cannot minify list" % net)
|
||||
return False
|
||||
for i in getConnectedRelays(net):
|
||||
name = net + str(i)
|
||||
for x in main.IRCPool[name].channels:
|
||||
if as_list:
|
||||
for y in channel_list:
|
||||
if y == x:
|
||||
channel_list.remove(y)
|
||||
else:
|
||||
for y in listinfo:
|
||||
if y[0] == x:
|
||||
listinfo.remove(y)
|
||||
if not as_list:
|
||||
if not listinfo:
|
||||
log("We're on all the channels we want to be on, dropping LIST")
|
||||
return False
|
||||
if as_list:
|
||||
return channel_list
|
||||
else:
|
||||
return listinfo
|
||||
|
||||
|
||||
def keepChannels(net, listinfo, mean, sigrelay, relay):
|
||||
"""
|
||||
Minify channels, determine whether we can cover all the channels
|
||||
on the network, or need to use 'significant' mode.
|
||||
Truncate the channel list to available channel spaces.
|
||||
Allocate these channels to relays.
|
||||
Notify relays that they should pull from TempChan to join.
|
||||
:param net: network
|
||||
:param listinfo: list of [channel, num_users] lists
|
||||
:param mean: mean of channel population
|
||||
:param sigrelay: number of relays needed to cover significant channels
|
||||
:param relay: number of relays needed to cover all channels
|
||||
:param chanlimit: maximum number of channels to allocate to a relay
|
||||
"""
|
||||
listinfo = minifyChans(net, listinfo)
|
||||
if not listinfo:
|
||||
return
|
||||
if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
|
||||
coverAll = True
|
||||
elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
|
||||
coverAll = False
|
||||
# if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
|
||||
# error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
|
||||
# return
|
||||
num_instances = len(getActiveRelays(net))
|
||||
debug(f"keepChannels() {net} instances:{num_instances}")
|
||||
chan_slots_used = getTotalChans(net)
|
||||
debug(f"keepChannels() slots_used:{chan_slots_used}")
|
||||
# max_chans = (chanlimit * num_instances) - chan_slots_used
|
||||
max_chans = getSumChanlimit(net) - chan_slots_used
|
||||
if max_chans < 0:
|
||||
max_chans = 0
|
||||
debug(f"keepChannels() max_chans:{max_chans}")
|
||||
if coverAll:
|
||||
# needed = relay - len(getActiveRelays(net))
|
||||
# if needed:
|
||||
# debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
|
||||
# newNums = modules.provision.provisionMultipleRelays(net, needed)
|
||||
# else:
|
||||
# newNums = []
|
||||
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
|
||||
if len(listinfo_sort) > max_chans:
|
||||
max_chans = len(listinfo_sort) - 1
|
||||
|
||||
flist = [i[0] for i in listinfo_sort]
|
||||
|
||||
flist = flist[:max_chans]
|
||||
debug(f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels")
|
||||
trace(f"keepChannels() {net}: joining:{flist}")
|
||||
populateChans(net, flist)
|
||||
else:
|
||||
# needed = sigrelay - len(getActiveRelays(net))
|
||||
# if needed:
|
||||
# debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
|
||||
# newNums = modules.provision.provisionMultipleRelays(net, needed)
|
||||
# else:
|
||||
# newNums = []
|
||||
listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
|
||||
trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
|
||||
if len(listinfo_sort) > max_chans:
|
||||
max_chans = len(listinfo_sort) - 1
|
||||
debug(f"keepChannels() {net}: new max_chans:{max_chans}")
|
||||
|
||||
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
|
||||
trace(f"keepChannels() {net}: new siglist:{siglist}")
|
||||
|
||||
siglist = siglist[:max_chans]
|
||||
trace(f"keepChannels() {net}: truncated siglist:{siglist}")
|
||||
|
||||
trace(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}")
|
||||
debug(f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels")
|
||||
trace(f"keepChannels() {net}: joining:{siglist}")
|
||||
populateChans(net, siglist)
|
||||
notifyJoin(net)
|
||||
|
||||
|
||||
def joinSingle(net, channel):
|
||||
"""
|
||||
Join a channel on a relay.
|
||||
Use ECA to determine which relay to join on.
|
||||
:param net: network
|
||||
:param channel: channel to join
|
||||
:return: relay number that joined the channel
|
||||
:rtype: int
|
||||
"""
|
||||
if "," in channel:
|
||||
channels = channel.split(",")
|
||||
channels = minifyChans(net, channels, as_list=True)
|
||||
|
||||
else:
|
||||
channels = [channel]
|
||||
populateChans(net, channels)
|
||||
notifyJoin(net)
|
||||
return True
|
||||
|
||||
|
||||
def partSingle(net, channel):
|
||||
"""
|
||||
Iterate over all the relays of net and part channels matching channel.
|
||||
:param net: network
|
||||
:param channel: channel to part
|
||||
:return: list of relays that parted the channel
|
||||
:rtype: list of str
|
||||
"""
|
||||
parted = []
|
||||
for i in getConnectedRelays(net):
|
||||
name = f"{net}{i}"
|
||||
if name in main.IRCPool.keys():
|
||||
if channel in main.IRCPool[name].channels:
|
||||
main.IRCPool[name].part(channel)
|
||||
parted.append(str(i))
|
||||
return parted
|
||||
|
||||
|
||||
def nukeNetwork(net):
|
||||
"""
|
||||
Remove network records.
|
||||
:param net: network"""
|
||||
# purgeRecords(net)
|
||||
# p = main.g.pipeline()
|
||||
main.g.delete("analytics.list." + net)
|
||||
# p.delete("list."+net)
|
||||
# p.execute()
|
||||
|
||||
|
||||
# def nukeNetwork(net):
|
||||
# deferToThread(_nukeNetwork, net)
|
||||
|
||||
|
||||
def _initialList(net, num, listinfo):
|
||||
"""
|
||||
Called when a relay receives a full LIST response.
|
||||
Run statistics to determine how many channels are significant.
|
||||
This is done by adding all the numbers of users on the channels together,
|
||||
then dividing by the number of channels.
|
||||
* cumul - cumulative sum of all channel membership
|
||||
* siglength - number of significant channels
|
||||
* listlength - number of channels in the list
|
||||
* sigrelay - number of relays needed to cover siglength
|
||||
* relay - number of relays needed to cover all channels
|
||||
:param net: network
|
||||
:param num: relay number
|
||||
:param listinfo: list of [channel, num_users] lists
|
||||
:param chanlimit: maximum number of channels the relay can join
|
||||
"""
|
||||
listlength = len(listinfo)
|
||||
cumul = 0
|
||||
try:
|
||||
cumul += sum(int(i[1]) for i in listinfo)
|
||||
except TypeError:
|
||||
warn("Bad LIST data received from %s - %i" % (net, num))
|
||||
return
|
||||
mean = round(cumul / listlength, 2)
|
||||
siglength = 0
|
||||
insiglength = 0
|
||||
sigcumul = 0
|
||||
insigcumul = 0
|
||||
for i in listinfo:
|
||||
if int(i[1]) > mean:
|
||||
siglength += 1
|
||||
sigcumul += int(i[1])
|
||||
elif int(i[1]) < mean:
|
||||
insiglength += 1
|
||||
insigcumul += int(i[1])
|
||||
|
||||
avg_chanlimit = getAverageChanlimit(net)
|
||||
sigrelay = ceil(siglength / avg_chanlimit)
|
||||
relay = ceil(listlength / avg_chanlimit)
|
||||
|
||||
cur_relays = len(getActiveRelays(net))
|
||||
sig_relays_missing = sigrelay - cur_relays
|
||||
all_relays_missing = relay - cur_relays
|
||||
|
||||
abase = "analytics.list.%s" % net
|
||||
main.g.delete(abase)
|
||||
p = main.g.pipeline()
|
||||
|
||||
# See docstring for meanings
|
||||
p.hset(abase, "mean", mean)
|
||||
p.hset(abase, "total_chans", listlength)
|
||||
p.hset(abase, "big_chans", siglength)
|
||||
p.hset(abase, "small_chans", insiglength)
|
||||
p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
|
||||
p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
|
||||
p.hset(abase, "total_cumul_mem", cumul)
|
||||
p.hset(abase, "big_chan_cumul_mem", sigcumul)
|
||||
p.hset(abase, "small_chan_cumul_mem", insigcumul)
|
||||
p.hset(abase, "relays_for_all_chans", relay)
|
||||
p.hset(abase, "relays_for_big_chans", sigrelay)
|
||||
p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
|
||||
p.hset(abase, "sig_relays_missing", sig_relays_missing)
|
||||
p.hset(abase, "all_relays_missing", all_relays_missing)
|
||||
debug(
|
||||
(
|
||||
f"_initialList() net:{net} num:{num} listlength:{listlength} "
|
||||
f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
|
||||
f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
|
||||
)
|
||||
)
|
||||
|
||||
# Purge existing records before writing
|
||||
# purgeRecords(net)
|
||||
# for i in listinfo:
|
||||
# p.rpush(netbase+"."+i[0], i[1])
|
||||
# p.rpush(netbase+"."+i[0], i[2])
|
||||
# p.sadd(netbase, i[0])
|
||||
|
||||
p.execute()
|
||||
debug("List parsing completed on %s" % net)
|
||||
keepChannels(net, listinfo, mean, sigrelay, relay)
|
||||
|
||||
# return (listinfo, mean, sigrelay, relay)
|
||||
|
||||
|
||||
def convert(data):
|
||||
"""
|
||||
Recursively convert a dictionary.
|
||||
"""
|
||||
if isinstance(data, bytes):
|
||||
return data.decode("ascii")
|
||||
if isinstance(data, dict):
|
||||
return dict(map(convert, data.items()))
|
||||
if isinstance(data, tuple):
|
||||
return map(convert, data)
|
||||
if isinstance(data, list):
|
||||
return list(map(convert, data))
|
||||
return data
|
||||
|
||||
|
||||
def getListInfo(net):
|
||||
abase = f"analytics.list.{net}"
|
||||
info = main.g.hgetall(abase)
|
||||
return convert(info)
|
||||
|
||||
|
||||
def initialList(net, num, listinfo):
|
||||
"""
|
||||
Run _initialList in a thread.
|
||||
See above docstring.
|
||||
"""
|
||||
deferToThread(_initialList, net, num, deepcopy(listinfo))
|
||||
Reference in New Issue
Block a user