Implement ChanKeep joining functions

* Low-key channel joining with incrementally increasing delay
* Spin up needed instances to be able to cover a certain channel space
* Fix provisioning functions to prevent race conditions with lots of
relays being created at once
* Tweakable switchover from covering all channels to only covering
channels with more users than the mean of the cumulative user count

modules/chankeep.py

@@ -2,11 +2,61 @@ import main
 from utils.logging.log import *
 from utils.logging.debug import *
 from copy import deepcopy
+from math import ceil
+from modules.provision import provisionMultipleRelays
 from twisted.internet.threads import deferToThread
+from numpy import array_split
 
-def provisionInstances(net, relaysNeeded):
-    #num, alias =
-    pass
+def allRelaysActive(net):
+    relayNum = len(main.network[net].relays.keys())
+    existNum = 0
+    for i in main.network[net].relays.keys():
+        name = net+str(i)
+        if name in main.IRCPool.keys():
+            existNum += 1
+    if existNum == relayNum:
+        return True
+    return False
+
+def populateChans(net, clist, relay):
+    divided = array_split(clist, relay)
+    for i in range(0, len(divided)):
+        if net in main.TempChan.keys():
+            main.TempChan[net][i] = divided[i]
+        else:
+            main.TempChan[net] = {i: divided[i]}
+
+def notifyJoin(net):
+    for i in main.network[net].relays.keys():
+        name = net+str(i)
+        if name in main.IRCPool.keys():
+            main.IRCPool[name].checkChannels()
+
+def keepChannels(net, listinfo, mean, sigrelay, relay):
+    #print("list", listinfo)
+    #print("sigrelay", sigrelay)
+    #print("cur", len(main.network[net].relays.keys()))
+    if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
+        coverAll = True
+    elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
+        coverAll = False
+        if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
+            error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
+            return
+    if coverAll:
+        needed = relay-len(main.network[net].relays.keys())
+        flist = [i[0] for i in listinfo]
+        populateChans(net, flist, relay)
+    else:
+        needed = sigrelay-len(main.network[net].relays.keys())
+        siglist = [i[0] for i in listinfo if int(i[1]) > mean]
+        populateChans(net, siglist, sigrelay)
+    notifyJoin(net)
+    if needed > 0:
+        provisionMultipleRelays(net, needed)
+
+    #print("coverall", coverAll)
+    #print("needed", needed)
 
 def purgeRecords(net):
     base = "list.%s" % net
@@ -51,7 +101,8 @@ def _initialList(net, num, listinfo, chanlimit):
     if not net in main.network.keys():
         warn("Cannot write list info - no network entry for %s" % net)
         return
-    sigrelay = round(siglength/chanlimit, 2)
+    sigrelay = ceil(siglength/chanlimit)
+    relay = ceil(listlength/chanlimit)
     netbase = "list.%s" % net
     abase = "analytics.list.%s" % net
     p = main.g.pipeline()
@@ -64,9 +115,9 @@ def _initialList(net, num, listinfo, chanlimit):
     p.hset(abase, "cumul", cumul)
     p.hset(abase, "sigcumul", sigcumul)
     p.hset(abase, "insigcumul", insigcumul)
-    p.hset(abase, "relay", round(listlength/chanlimit, 2))
+    p.hset(abase, "relay", relay)
     p.hset(abase, "sigrelay", sigrelay)
-    p.hset(abase, "insigrelay", round(insiglength/chanlimit, 2))
+    p.hset(abase, "insigrelay", ceil(insiglength/chanlimit))
     # Purge existing records before writing
     purgeRecords(net)
     for i in listinfo:
@@ -76,6 +127,7 @@ def _initialList(net, num, listinfo, chanlimit):
 
     p.execute()
     debug("List parsing completed on %s" % net)
+    keepChannels(net, listinfo, mean, sigrelay, relay)
 
 def initialList(net, num, listinfo, chanlimit):
     deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit)