Implement fair channel allocation in ChanKeep

* Allocate channels to relays only if they have free space based on
their chanlimit value
* Minify channels by removing ones that are already covered before
passing them off to be joined

modules/chankeep.py

@@ -13,18 +13,46 @@ def allRelaysActive(net):
     for i in main.network[net].relays.keys():
         name = net+str(i)
         if name in main.IRCPool.keys():
-            existNum += 1
+            if main.IRCPool[name].connected:
+                existNum += 1
     if existNum == relayNum:
         return True
     return False
 
+def getChanFree(net):
+    chanfree = {}
+    for i in main.network[net].relays.keys():
+        name = net+str(i)
+        chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels)
+    return chanfree
+
+def emptyChanAllocate(net, flist, relay):
+    chanfree = getChanFree(net)
+    allocated = {}
+    toalloc = len(flist)
+    if toalloc > sum(chanfree.values()):
+        error("Too many channels to allocate for %s - this is probably a bug" % net)
+        return False
+    for i in chanfree.keys():
+        for x in range(chanfree[i]):
+            if not len(flist):
+                break
+            if i in allocated.keys():
+                allocated[i].append(flist.pop())
+            else:
+                allocated[i] = [flist.pop()]
+    return allocated
+
 def populateChans(net, clist, relay):
-    divided = array_split(clist, relay)
-    for i in range(0, len(divided)):
+    #divided = array_split(clist, relay)
+    allocated = emptyChanAllocate(net, clist, relay)
+    if not allocated:
+        return
+    for i in allocated.keys():
         if net in main.TempChan.keys():
-            main.TempChan[net][i] = divided[i]
+            main.TempChan[net][i] = allocated[i]
         else:
-            main.TempChan[net] = {i: divided[i]}
+            main.TempChan[net] = {i: allocated[i]}
 
 def notifyJoin(net):
     for i in main.network[net].relays.keys():
@@ -32,10 +60,28 @@ def notifyJoin(net):
         if name in main.IRCPool.keys():
             main.IRCPool[name].checkChannels()
 
+def minifyChans(net, listinfo):
+    if not allRelaysActive(net):
+        error("All relays for %s are not active, cannot minify list")
+        return False
+    for i in main.network[net].relays.keys():
+        name = net+str(i)
+        for x in main.IRCPool[name].channels:
+            for y in listinfo:
+                if y[0] == x:
+                    listinfo.remove(y)
+    if not listinfo:
+        log("We're on all the channels we want to be on, dropping LIST")
+        return False
+    return listinfo
+
 def keepChannels(net, listinfo, mean, sigrelay, relay):
     #print("list", listinfo)
     #print("sigrelay", sigrelay)
     #print("cur", len(main.network[net].relays.keys()))
+    listinfo = minifyChans(net, listinfo)
+    if not listinfo:
+        return
     if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
         coverAll = True
     elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
@@ -58,23 +104,24 @@ def keepChannels(net, listinfo, mean, sigrelay, relay):
     #print("coverall", coverAll)
     #print("needed", needed)
 
-def purgeRecords(net):
-    base = "list.%s" % net
-    p = main.g.pipeline()
-    existingChans = main.g.smembers("list."+net)
-    for i in existingChans:
-        p.delete(base+"."+i.decode("utf-8"))
-    p.execute()
-
-def _nukeNetwork(net):
-    purgeRecords(net)
-    p = main.g.pipeline()
-    p.delete("analytics.list."+net)
-    p.delete("list."+net)
-    p.execute()
+#def purgeRecords(net):
+#    base = "list.%s" % net
+#    p = main.g.pipeline()
+#    existingChans = main.g.smembers("list."+net)
+#    for i in existingChans:
+#        p.delete(base+"."+i.decode("utf-8"))
+#    p.execute()
 
 def nukeNetwork(net):
-    deferToThread(_nukeNetwork, net)
+    #purgeRecords(net)
+    #p = main.g.pipeline()
+    main.g.delete("analytics.list."+net)
+    #p.delete("list."+net)
+    #p.execute()
+
+#def nukeNetwork(net):
+#    deferToThread(_nukeNetwork, net)
+
 
 def _initialList(net, num, listinfo, chanlimit):
     #listinfo = sorted(listinfo, key=lambda x: xdd[0])
@@ -98,9 +145,6 @@ def _initialList(net, num, listinfo, chanlimit):
             insiglength += 1
             insigcumul += int(i[1])
 
-    if not net in main.network.keys():
-        warn("Cannot write list info - no network entry for %s" % net)
-        return
     sigrelay = ceil(siglength/chanlimit)
     relay = ceil(listlength/chanlimit)
     netbase = "list.%s" % net
@@ -118,12 +162,13 @@ def _initialList(net, num, listinfo, chanlimit):
     p.hset(abase, "relay", relay)
     p.hset(abase, "sigrelay", sigrelay)
     p.hset(abase, "insigrelay", ceil(insiglength/chanlimit))
+
     # Purge existing records before writing
-    purgeRecords(net)
-    for i in listinfo:
-        p.rpush(netbase+"."+i[0], i[1])
-        p.rpush(netbase+"."+i[0], i[2])
-        p.sadd(netbase, i[0])
+    #purgeRecords(net)
+    #for i in listinfo:
+    #    p.rpush(netbase+"."+i[0], i[1])
+    #    p.rpush(netbase+"."+i[0], i[2])
+    #    p.sadd(netbase, i[0])
 
     p.execute()
     debug("List parsing completed on %s" % net)