Defer initialUsers, initialNames and delChannel to threads to improve performance

core/bot.py

@@ -398,7 +398,7 @@ class IRCBot(IRCClient):
             lc.stop()
             del self._getWho[channel]
         userinfo.delChannel(self.net, channel)                    # < we do not need to deduplicate this
-        log("Can no longer cover %s, removing records" % channel) #  as it will only be matched once --
+        #log("Can no longer cover %s, removing records" % channel)#  as it will only be matched once --
                                                                   #  other bots have different nicknames so
     def left(self, user, channel, message):                       #  even if they saw it, they wouldn't react
         self.event(type="part", muser=user, target=channel, message=message)

modules/userinfo.py

@@ -1,6 +1,9 @@
-import main
+from twisted.internet.threads import deferToThread
 from string import digits
+
+import main
 from utils.logging.log import *
+from utils.logging.debug import debug
 
 def getWhoSingle(name, query):
     result = main.r.sscan("live.who."+name, 0, query, count=9999999)
@@ -68,14 +71,19 @@ def getNamespace(name, channel, nick):
     chanspace = "live.chan.%s.%s" % (name, nick)
     return [gnamespace, namespace, chanspace]
 
-def initialUsers(name, channel, users):
+def _initialUsers(name, channel, users):
     gnamespace = "live.who.%s" % name
     p = main.r.pipeline()
     for i in users:
         p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2])
     p.execute()
 
-def initialNames(name, channel, names):
+def initialUsers(name, channel, users):
+    debug("Initialising WHO records for %s on %s" % (channel, name))
+    d = deferToThread(_initialUsers, name, channel, users)
+    #d.addCallback(testCallback)
+
+def _initialNames(name, channel, names):
     namespace = "live.who.%s.%s" % (name, channel)
     p = main.r.pipeline()
     for i in names:
@@ -83,6 +91,11 @@ def initialNames(name, channel, names):
         p.sadd("live.chan."+name+"."+i, channel)
     p.execute()
 
+def initialNames(name, channel, names):
+    debug("Initialising NAMES records for %s on %s" % (channel, name))
+    d = deferToThread(_initialNames, name, channel, names)
+    #d.addCallback(testCallback)
+
 def editUser(name, user):
     gnamespace = "live.who.%s" % name
     main.r.sadd(gnamespace, user)
@@ -158,7 +171,7 @@ def delUserByNetwork(name, nick, user):
     p.delete(chanspace)
     p.execute()
 
-def delChannel(name, channel): # This function is extremely expensive, look to replace
+def _delChannel(name, channel): # This function is extremely expensive, look to replace
     gnamespace = "live.who.%s" % name
     namespace = "live.who.%s.%s" % (name, channel)
     p = main.r.pipeline()
@@ -173,10 +186,16 @@ def delChannel(name, channel): # This function is extremely expensive, look to r
             p.srem("live.chan."+name+"."+i.decode(), channel)
     p.delete(namespace)
     p.execute()
+    return [name, channel]
+
+def delChannel(name, channel):
+    debug("Purging channel %s for %s" % (channel, name))
+    d = deferToThread(_delChannel, name, channel)
+    #d.addCallback(testCallback)
 
 def delNetwork(name, channels):
-    log("Purging channels for %s" % name)
+    debug("Purging channels for %s" % name)
     for i in channels:
         delChannel(name, i)
-    log("Finished purging channels for %s" % name)
+    #log("Finished purging channels for %s" % name)
     return