Switch to using Redis for LIST storage

modules/chankeep.py

@@ -1,45 +1,82 @@
 import main
 from utils.logging.log import *
+from utils.logging.debug import *
+from copy import deepcopy
+from twisted.internet.threads import deferToThread
 
-# make this into a threaded function
+def provisionInstances(net, relaysNeeded):
-def initialList(net, num, listinfo, chanLimit):
+    #num, alias =
+    pass
+
+def purgeRecords(net):
+    base = "list.%s" % net
+    p = main.g.pipeline()
+    existingChans = main.g.smembers("list."+net)
+    for i in existingChans:
+        p.delete(base+"."+i.decode("utf-8"))
+    p.execute()
+
+def _nukeNetwork(net):
+    purgeRecords(net)
+    p = main.g.pipeline()
+    p.delete("analytics.list."+net)
+    p.delete("list."+net)
+    p.execute()
+
+def nukeNetwork(net):
+    deferToThread(_nukeNetwork, net)
+
+def _initialList(net, num, listinfo, chanlimit):
     #listinfo = sorted(listinfo, key=lambda x: xdd[0])
-    listLength = len(listinfo)
+    listlength = len(listinfo)
     cumul = 0
     try:
         cumul += sum(int(i[1]) for i in listinfo)
     except TypeError:
         warn("Bad LIST data received from %s - %i" % (net, num))
         return
-    mean = cumul/listLength
+    mean = round(cumul/listlength, 2)
-    sigLength = 0
+    siglength = 0
-    insigLength = 0
+    insiglength = 0
-    sigCumul = 0
+    sigcumul = 0
-    insigCumul = 0
+    insigcumul = 0
     for i in listinfo:
         if int(i[1]) > mean:
-            sigLength += 1
+            siglength += 1
-            sigCumul += int(i[1])
+            sigcumul += int(i[1])
         elif int(i[1]) < mean:
-            insigLength += 1
+            insiglength += 1
-            insigCumul += int(i[1])
+            insigcumul += int(i[1])
 
     if not net in main.network.keys():
         warn("Cannot write list info - no network entry for %s" % net)
         return
-    main.network[net].list["mean"] = mean
+    sigrelay = round(siglength/chanlimit, 2)
-    main.network[net].list["total"] = listLength
+    netbase = "list.%s" % net
-    main.network[net].list["sigtotal"] = sigLength
+    abase = "analytics.list.%s" % net
-    main.network[net].list["insigtotal"] = insigLength
+    p = main.g.pipeline()
-    main.network[net].list["sigperc"] = sigLength/listLength*100
+    p.hset(abase, "mean", mean)
-    main.network[net].list["insigperc"] = insigLength/listLength*100
+    p.hset(abase, "total", listlength)
-    main.network[net].list["cumul"] = cumul
+    p.hset(abase, "sigtotal", siglength)
-    main.network[net].list["sigcumul"] = sigCumul
+    p.hset(abase, "insigtotal", insiglength)
-    main.network[net].list["insigcumul"] = insigCumul
+    p.hset(abase, "sigperc", round(siglength/listlength*100, 2))
-    main.network[net].list["relay"] = listLength/chanLimit
+    p.hset(abase, "insigperc", round(insiglength/listlength*100, 2))
-    main.network[net].list["sigrelay"] = sigLength/chanLimit
+    p.hset(abase, "cumul", cumul)
-    main.network[net].list["insigrelay"] = insigLength/chanLimit
+    p.hset(abase, "sigcumul", sigcumul)
+    p.hset(abase, "insigcumul", insigcumul)
+    p.hset(abase, "relay", round(listlength/chanlimit, 2))
+    p.hset(abase, "sigrelay", sigrelay)
+    p.hset(abase, "insigrelay", round(insiglength/chanlimit, 2))
+    # Purge existing records before writing
+    purgeRecords(net)
+    for i in listinfo:
+        p.rpush(netbase+"."+i[0], i[1])
+        p.rpush(netbase+"."+i[0], i[2])
+        p.sadd(netbase, i[0])
+
+    p.execute()
+    debug("List parsing completed on %s" % net)
+
+def initialList(net, num, listinfo, chanlimit):
+    deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit)
 
-    # Rounding
-    main.network[net].list = {x: round(y, 4) for x, y in main.network[net].list.items()}
-    main.saveConf("network")

modules/network.py

@@ -2,6 +2,7 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory
 import json
 
 from modules import alias
+from modules.chankeep import nukeNetwork
 from twisted.internet import reactor
 from core.bot import IRCBot, IRCBotFactory
 import main
@@ -19,7 +20,6 @@ class Network:
         self.last = 0
         self.relays = {}
         self.aliases = {}
-        self.list = {}
 
     def add_relay(self, num=None):
         if not num:
@@ -60,6 +60,7 @@ class Network:
     def seppuku(self):
         # Removes all bots in preperation for deletion
         self.killAliases(self.relays.keys())
+        nukeNetwork(self.net)
 
     def start_bot(self, num):
         # a single name is given to relays in the backend