Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve performance in userinfo 

* Implement a nick -> user mapping, preventing a superfluous SSCAN on
the entire dataset for when networks are disconnected
* Use one thread for all channels when a network instance is
disconnected, instead of one thread per channel
* Made returns comprising of only a list into tuples
This commit is contained in:
Mark Veidemanis 2019-10-17 20:19:35 +01:00
parent a64765121a
commit f34de8940f
3 changed files with 88 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
core/bot.py
View File

@ -21,6 +21,7 @@ import main
from utils.logging.log import * from utils.logging.log import *
from utils.logging.debug import * from utils.logging.debug import *
from utils.logging.send import * from utils.logging.send import *
from utils.parsing import parsen
from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.ssl import DefaultOpenSSLContextFactory
@ -54,20 +55,8 @@ class IRCRelay(IRCClient):
self.stage2 = stage2 self.stage2 = stage2
self.loop = None self.loop = None
def parsen(self, user):
step = user.split("!")
nick = step[0]
if len(step) == 2:
step2 = step[1].split("@")
ident, host = step2
else:
ident = nick
host = nick
return [nick, ident, host]
def privmsg(self, user, channel, msg): def privmsg(self, user, channel, msg):
nick, ident, host = self.parsen(user) nick, ident, host = parsen(user)
for i in main.ZNCErrors: for i in main.ZNCErrors:
if i in msg: if i in msg:
error("ZNC issue:", msg) error("ZNC issue:", msg)
@ -134,22 +123,6 @@ class IRCBot(IRCClient):
self.chanlimit = 0 self.chanlimit = 0
def parsen(self, user):
step = user.split("!")
nick = step[0]
if len(step) == 2:
step2 = step[1].split("@")
if len(step2) == 2:
ident, host = step2
else:
ident = nick
host = nick
else:
ident = nick
host = nick
return [nick, ident, host]
def joinChannels(self, channels): def joinChannels(self, channels):
sleeptime = 0.0 sleeptime = 0.0
increment = 0.8 increment = 0.8
@ -178,7 +151,9 @@ class IRCBot(IRCClient):
if cast[i] == "": # a dictionary that changes length with each iteration if cast[i] == "": # a dictionary that changes length with each iteration
del cast[i] del cast[i]
if "muser" in cast.keys(): if "muser" in cast.keys():
cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
# additional checks here to see if it's a server -- irc.example.net
# discard if it is
#if not cast["type"] in ["nick", "kick", "quit", "part", "join"]: #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]:
# del cast["muser"] # del cast["muser"]
if set(["nick", "ident", "host", "msg"]).issubset(set(cast)): if set(["nick", "ident", "host", "msg"]).issubset(set(cast)):
@ -507,7 +482,7 @@ class IRCBot(IRCClient):
lc = self._getWho[channel] lc = self._getWho[channel]
lc.stop() lc.stop()
del self._getWho[channel] del self._getWho[channel]
userinfo.delChannel(self.net, channel) # < we do not need to deduplicate this userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this
#log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once --
# other bots have different nicknames so # other bots have different nicknames so
def left(self, user, channel, message): # even if they saw it, they wouldn't react def left(self, user, channel, message): # even if they saw it, they wouldn't react
@ -529,7 +504,7 @@ class IRCBot(IRCClient):
self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee) self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee)
def chanlessEvent(self, cast): def chanlessEvent(self, cast):
cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
if dedup(self.name, cast): # Needs to be kept self.name until the dedup if dedup(self.name, cast): # Needs to be kept self.name until the dedup
# function is converted to the new net, num # function is converted to the new net, num
# format # format
@ -590,7 +565,7 @@ class IRCBotFactory(ReconnectingClientFactory):
def clientConnectionLost(self, connector, reason): def clientConnectionLost(self, connector, reason):
if not self.relay: if not self.relay:
userinfo.delNetwork(self.net, self.client.channels) userinfo.delChannels(self.net, self.client.channels)
if not self.client == None: if not self.client == None:
self.client.connected = False self.client.connected = False
self.client.channels = [] self.client.channels = []

98
modules/userinfo.py
View File

@ -4,12 +4,13 @@ from string import digits
import main import main
from utils.logging.log import * from utils.logging.log import *
from utils.logging.debug import debug from utils.logging.debug import debug
from utils.parsing import parsen
def getWhoSingle(name, query): def getWhoSingle(name, query):
result = main.r.sscan("live.who."+name, 0, query, count=9999999) result = main.r.sscan("live.who."+name, 0, query, count=-1)
if result[1] == []: if result[1] == []:
return None return None
return [i.decode() for i in result[1]] return (i.decode() for i in result[1])
def getWho(query): def getWho(query):
result = {} result = {}
@ -24,14 +25,14 @@ def getChansSingle(name, nick):
result = main.r.sinter(*nick) result = main.r.sinter(*nick)
if len(result) == 0: if len(result) == 0:
return None return None
return [i.decode() for i in result] return (i.decode() for i in result)
def getChanList(name, nick): def getChanList(name, nick):
chanspace = "live.chan."+name+"."+nick chanspace = "live.chan."+name+"."+nick
result = main.r.smembers(chanspace) result = main.r.smembers(chanspace)
if len(result) == 0: if len(result) == 0:
return None return None
return [i.decode() for i in result] return (i.decode() for i in result)
def getChans(nick): def getChans(nick):
result = {} result = {}
@ -42,11 +43,11 @@ def getChans(nick):
return result return result
def getUsersSingle(name, nick): def getUsersSingle(name, nick):
nick = ["live.who."+name+"."+i for i in nick] nick = ("live.who."+name+"."+i for i in nick)
result = main.r.sinter(*nick) result = main.r.sinter(*nick)
if len(result) == 0: if len(result) == 0:
return None return None
return [i.decode() for i in result] return (i.decode() for i in result)
def getUsers(nick): def getUsers(nick):
result = {} result = {}
@ -69,13 +70,17 @@ def getNamespace(name, channel, nick):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel) namespace = "live.who.%s.%s" % (name, channel)
chanspace = "live.chan.%s.%s" % (name, nick) chanspace = "live.chan.%s.%s" % (name, nick)
return [gnamespace, namespace, chanspace] mapspace = "live.map.%s" % name
return (gnamespace, namespace, chanspace, mapspace)
def _initialUsers(name, channel, users): def _initialUsers(name, channel, users):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
mapspace = "live.map.%s" % name
p = main.r.pipeline() p = main.r.pipeline()
for i in users: for i in users:
p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2]) user = i[0]+"!"+i[1]+"@"+i[2]
p.hset(mapspace, i[0], user)
p.sadd(gnamespace, user)
p.execute() p.execute()
def initialUsers(name, channel, users): def initialUsers(name, channel, users):
@ -98,29 +103,36 @@ def initialNames(name, channel, names):
def editUser(name, user): def editUser(name, user):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
main.r.sadd(gnamespace, user) mapspace = "live.map.%s" % name
parsed = parsen(user)
p = main.r.pipeline()
p.sadd(gnamespace, user)
p.hset(mapspace, parsed[0], user) # add nick -> user mapping
p.execute()
def addUser(name, channel, nick, user): def addUser(name, channel, nick, user):
gnamespace, namespace, chanspace = getNamespace(name, channel, nick) gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline() p = main.r.pipeline()
p.sadd(gnamespace, user) p.sadd(gnamespace, user)
p.sadd(namespace, nick) p.sadd(namespace, nick)
p.sadd(chanspace, channel) p.sadd(chanspace, channel)
p.hset(mapspace, nick, user)
p.execute() p.execute()
def delUser(name, channel, nick, user): def delUser(name, channel, nick, user):
gnamespace, namespace, chanspace = getNamespace(name, channel, nick) gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline() p = main.r.pipeline()
channels = main.r.smembers(chanspace) channels = main.r.smembers(chanspace)
p.srem(namespace, nick) p.srem(namespace, nick)
if channels == {channel.encode()}: if channels == {channel.encode()}: # can we only see them on this channel?
p.delete(chanspace) p.delete(chanspace) # remove channel tracking entry
p.hdel(mapspace, nick) # remove nick mapping entry
if user: if user:
p.srem(gnamespace, user) p.srem(gnamespace, user) # remove global userinfo entry
else: else:
warn("Attempt to delete nonexistent user: %s" % user) warn("Attempt to delete nonexistent user: %s" % user)
else: else:
p.srem(chanspace, channel) p.srem(chanspace, channel) # keep up - remove the channel from their list
p.execute() p.execute()
def escape(text): def escape(text):
@ -131,7 +143,14 @@ def escape(text):
return text return text
def getUserByNick(name, nick): def getUserByNick(name, nick):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
mapspace = "live.map.%s" % name
if main.r.hexists(mapspace, nick):
return main.r.hget(mapspace, nick)
else:
warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace))
#return Falsedd
# legacy code below - remove when map is reliable
usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1) usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1)
if usermatch[1] == []: if usermatch[1] == []:
return False return False
@ -140,12 +159,13 @@ def getUserByNick(name, nick):
user = usermatch[1][0] user = usermatch[1][0]
return user return user
else: else:
warn("Entry doesn't exist: %s on %s" % (nick, gnamespace)) warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
return False return False
def renameUser(name, oldnick, olduser, newnick, newuser): def renameUser(name, oldnick, olduser, newnick, newuser):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, oldnick) chanspace = "live.chan.%s.%s" % (name, oldnick)
mapspace = "live.map.%s" % name
newchanspace = "live.chan.%s.%s" % (name, newnick) newchanspace = "live.chan.%s.%s" % (name, newnick)
p = main.r.pipeline() p = main.r.pipeline()
p.srem(gnamespace, olduser) p.srem(gnamespace, olduser)
@ -154,51 +174,53 @@ def renameUser(name, oldnick, olduser, newnick, newuser):
i = i.decode() i = i.decode()
p.srem("live.who."+name+"."+i, oldnick) p.srem("live.who."+name+"."+i, oldnick)
p.sadd("live.who."+name+"."+i, newnick) p.sadd("live.who."+name+"."+i, newnick)
p.hdel(mapspace, oldnick)
p.hset(mapspace, newnick, newuser)
if main.r.exists(chanspace): if main.r.exists(chanspace):
p.rename(chanspace, newchanspace) p.rename(chanspace, newchanspace)
else: else:
warn("Key doesn't exist: %s" % chanspace) warn("Key doesn't exist: %s" % chanspace)
p.execute() p.execute()
def delUserByNick(name, channel, nick): def delUserByNick(name, channel, nick): # kick
user = getUserByNick(name, nick) user = getUserByNick(name, nick)
delUser(name, channel, nick, user) delUser(name, channel, nick, user)
def delUserByNetwork(name, nick, user): def delUserByNetwork(name, nick, user): # quit
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, nick) chanspace = "live.chan.%s.%s" % (name, nick)
mapspace = "live.chan.%s" % name
p = main.r.pipeline() p = main.r.pipeline()
p.srem(gnamespace, user) p.srem(gnamespace, user)
for i in main.r.smembers(chanspace): for i in main.r.smembers(chanspace):
p.srem("live.who."+name+"."+i.decode(), nick) p.srem("live.who."+name+"."+i.decode(), nick)
p.delete(chanspace) p.delete(chanspace)
p.hdel(mapspace, nick)
p.execute() p.execute()
def _delChannel(name, channel): # This function is extremely expensive, look to replace def _delChannels(net, channels):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % net
namespace = "live.who.%s.%s" % (name, channel) mapspace = "live.map.%s" % net
p = main.r.pipeline() p = main.r.pipeline()
for channel in channels:
namespace = "live.who.%s.%s" % (net, channel)
for i in main.r.smembers(namespace): for i in main.r.smembers(namespace):
user = getUserByNick(name, i.decode()) nick = i.decode()
if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}: #user = getUserByNick(net, nick) -- far too many function calls
user = main.r.hget(mapspace, nick)
if not user:
warn("User lookup failed: %s on %s" % (nick, net))
if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}:
if user: if user:
p.srem(gnamespace, user) p.srem(gnamespace, user)
p.delete("live.chan."+net+"."+nick)
p.delete("live.chan."+name+"."+i.decode()) p.hdel(mapspace, nick) # remove map entry
else: else:
p.srem("live.chan."+name+"."+i.decode(), channel) p.srem("live.chan."+net+"."+nick, channel)
p.delete(namespace) p.delete(namespace)
p.execute() p.execute()
return [name, channel]
def delChannel(name, channel): def delChannels(net, channels):
debug("Purging channel %s for %s" % (channel, name)) debug("Purging channel %s for %s" % (", ".join(channels), net))
d = deferToThread(_delChannel, name, channel) d = deferToThread(_delChannels, net, channels)
#d.addCallback(testCallback) #d.addCallback(testCallback)
def delNetwork(name, channels):
debug("Purging channels for %s" % name)
for i in channels:
delChannel(name, i)
#log("Finished purging channels for %s" % name)
return

15
utils/parsing.py Normal file
View File

@ -0,0 +1,15 @@
def parsen(user):
step = user.split("!")
nick = step[0]
if len(step) == 2:
step2 = step[1].split("@")
if len(step2) == 2:
ident, host = step2
else:
ident = nick
host = nick
else:
ident = nick
host = nick
return (nick, ident, host)