XF/monolith
2
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Implement ChanKeep joining functions

Browse Source 
* Low-key channel joining with incrementally increasing delay
* Spin up needed instances to be able to cover a certain channel space
* Fix provisioning functions to prevent race conditions with lots of
relays being created at once
* Tweakable switchover from covering all channels to only covering
channels with more users than the mean of the cumulative user count
This commit is contained in:
Mark Veidemanis
2019-10-11 13:07:57 +01:00
parent c3d0cb04b6
commit 7a6e3338c0
7 changed files with 126 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
modules/chankeep.py
View File

@@ -2,11 +2,61 @@ import main
from utils.logging.log import *
from utils.logging.debug import *
from copy import deepcopy
from math import ceil
from modules.provision import provisionMultipleRelays
from twisted.internet.threads import deferToThread
from numpy import array_split
def provisionInstances(net, relaysNeeded):
#num, alias =
pass
def allRelaysActive(net):
relayNum = len(main.network[net].relays.keys())
existNum = 0
for i in main.network[net].relays.keys():
name = net+str(i)
if name in main.IRCPool.keys():
existNum += 1
if existNum == relayNum:
return True
return False
def populateChans(net, clist, relay):
divided = array_split(clist, relay)
for i in range(0, len(divided)):
if net in main.TempChan.keys():
main.TempChan[net][i] = divided[i]
else:
main.TempChan[net] = {i: divided[i]}
def notifyJoin(net):
for i in main.network[net].relays.keys():
name = net+str(i)
if name in main.IRCPool.keys():
main.IRCPool[name].checkChannels()
def keepChannels(net, listinfo, mean, sigrelay, relay):
#print("list", listinfo)
#print("sigrelay", sigrelay)
#print("cur", len(main.network[net].relays.keys()))
if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
coverAll = True
elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
coverAll = False
if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
return
if coverAll:
needed = relay-len(main.network[net].relays.keys())
flist = [i[0] for i in listinfo]
populateChans(net, flist, relay)
else:
needed = sigrelay-len(main.network[net].relays.keys())
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
populateChans(net, siglist, sigrelay)
notifyJoin(net)
if needed > 0:
provisionMultipleRelays(net, needed)
#print("coverall", coverAll)
#print("needed", needed)
def purgeRecords(net):
base = "list.%s" % net
@@ -51,7 +101,8 @@ def _initialList(net, num, listinfo, chanlimit):
if not net in main.network.keys():
warn("Cannot write list info - no network entry for %s" % net)
return
sigrelay = round(siglength/chanlimit, 2)
sigrelay = ceil(siglength/chanlimit)
relay = ceil(listlength/chanlimit)
netbase = "list.%s" % net
abase = "analytics.list.%s" % net
p = main.g.pipeline()
@@ -64,9 +115,9 @@ def _initialList(net, num, listinfo, chanlimit):
p.hset(abase, "cumul", cumul)
p.hset(abase, "sigcumul", sigcumul)
p.hset(abase, "insigcumul", insigcumul)
p.hset(abase, "relay", round(listlength/chanlimit, 2))
p.hset(abase, "relay", relay)
p.hset(abase, "sigrelay", sigrelay)
p.hset(abase, "insigrelay", round(insiglength/chanlimit, 2))
p.hset(abase, "insigrelay", ceil(insiglength/chanlimit))
# Purge existing records before writing
purgeRecords(net)
for i in listinfo:
@@ -76,6 +127,7 @@ def _initialList(net, num, listinfo, chanlimit):
p.execute()
debug("List parsing completed on %s" % net)
keepChannels(net, listinfo, mean, sigrelay, relay)
def initialList(net, num, listinfo, chanlimit):
deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit)

6
modules/network.py
View File

@@ -17,14 +17,16 @@ class Network:
self.security = security
self.auth = auth
self.last = 0
self.last = 1
self.relays = {}
self.aliases = {}
def add_relay(self, num=None):
if not num:
self.last += 1
num = self.last
self.last += 1
elif num == self.last:
self.last += 1
self.relays[num] = {
"enabled": main.config["ConnectOnCreate"],
"net": self.net,

24
modules/provision.py
View File

@@ -1,8 +1,10 @@
import main
from core.bot import deliverRelayCommands
from utils.logging.log import *
from twisted.internet import reactor
def provisionUserData(num, nick, altnick, ident, realname, unused): # last field is password, which we don't want to inherit here, but still want to use * expansion, so this is a bit of a hack
def provisionUserData(num, nick, altnick, ident, realname, unused): # last field is password,
# which we don't want to inherit here, but still want to use * expansion, so this is a bit of a hack
commands = {}
commands["controlpanel"] = []
commands["controlpanel"].append("AddUser %s %s" % (nick, main.config["Relay"]["Password"]))
@@ -16,7 +18,7 @@ def provisionUserData(num, nick, altnick, ident, realname, unused): # last field
def provisionNetworkData(num, nick, network, host, port, security, auth, password):
commands = {}
stage2commands = {}
stage3commands = {}
stage2commands["status"] = []
commands["controlpanel"] = []
commands["controlpanel"].append("AddNetwork %s %s" % (nick, network))
if security == "ssl":
@@ -25,26 +27,21 @@ def provisionNetworkData(num, nick, network, host, port, security, auth, passwor
elif security == "plain":
commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port))
if auth == "sasl":
stage2commands["status"] = []
stage2commands["sasl"] = []
stage2commands["status"].append("LoadMod sasl")
stage2commands["sasl"].append("Mechanism plain")
stage2commands["sasl"].append("Set %s %s" % (nick, password))
elif auth == "ns":
stage2commands["status"] = []
stage2commands["nickserv"] = []
stage2commands["status"].append("LoadMod nickserv")
stage2commands["nickserv"].append("Set %s" % password)
if not main.config["ConnectOnCreate"]:
stage3commands["status"] = []
stage3commands["status"].append("Disconnect")
stage2commands["status"].append("Disconnect")
if main.config["Toggles"]["CycleChans"]:
stage2commands["status"] = []
stage2commands["status"].append("LoadMod disconkick")
stage2commands["status"].append("LoadMod chansaver")
deliverRelayCommands(num, commands,
stage2=[[nick+"/"+network, stage2commands],
[nick+"/"+network, stage3commands]])
stage2=[[nick+"/"+network, stage2commands]])
return
def provisionRelayForNetwork(num, alias, network):
@@ -60,14 +57,15 @@ def provisionRelay(num, network):
aliasObj = main.alias[num]
alias = aliasObj["nick"]
provisionUserData(num, *aliasObj.values())
provisionRelayForNetwork(num, alias, network)
reactor.callLater(5, provisionRelayForNetwork, num, alias, network)
if main.config["ConnectOnCreate"]:
main.network[network].start_bot(num)
return alias
reactor.callLater(10, main.network[network].start_bot, num)
return
def provisionMultipleRelays(net, relaysNeeded):
for i in range(1, relaysNeeded):
for i in range(relaysNeeded):
num, alias = main.network[net].add_relay()
print(relaysNeeded, "for", net, ":", num, alias)
provisionRelay(num, net)
main.saveConf("network")