Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move user metadata info into redis

This commit is contained in:
Mark Veidemanis 2018-08-27 20:42:49 +01:00
parent b31b5d40e8
commit 36105c7e9a
7 changed files with 215 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
commands/stats.py
View File

@ -1,5 +1,6 @@
import main
import modules.counters as count
import modules.userinfo as userinfo
class Stats:
def __init__(self, register):
@ -13,8 +14,7 @@ class Stats:
numWhoEntries = 0
for i in main.IRCPool.keys():
numChannels += len(main.IRCPool[i].channels)
for i in main.wholist.keys():
numWhoEntries += len(main.wholist[i].keys())
numWhoEntries += userinfo.getNumTotalWhoEntries()
stats.append("Servers: %s" % len(main.pool.keys()))
stats.append("Online servers: %s" % len(main.IRCPool.keys()))
stats.append("Channels: %s" % numChannels)
@ -34,20 +34,11 @@ class Stats:
numChannels = 0
numWhoEntries = 0
failures = 0
if spl[1] in main.IRCPool.keys():
numChannels += len(main.IRCPool[spl[1]].channels)
else:
failure("Name does not exist: %s" % spl[1])
failures += 1
if spl[1] in main.wholist.keys():
numWhoEntries += len(main.wholist[spl[1]].keys())
else:
failure("Who entry does not exist: %s" % spl[1])
failures += 1
if failures == 2:
failure("No information found, aborting")
return
numWhoEntries += userinfo.getNumWhoEntries(spl[1])
stats.append("Channels: %s" % numChannels)
stats.append("User records: %s" % numWhoEntries)
counterEvents = count.getEvents(spl[1])

4
commands/who.py
View File

@ -14,8 +14,8 @@ class Who:
rtrn += "Matches from: %s" % i
rtrn += "\n"
for x in result[i]:
x = [y for y in x if not y == None]
rtrn += str((", ".join(x)))
rtrn += (x)
rtrn += "\n"
rtrn += "\n"
info(rtrn)
return

4
conf/help.json
View File

@ -14,8 +14,8 @@
"disable": "disable <name>",
"list": "list",
"stats": "stats [<name>]",
"save": "save <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>",
"load": "load <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>",
"save": "save <config|keyconf|pool|help|counters|masterbuf|monitor|all>",
"load": "load <config|keyconf|pool|help|counters|masterbuf|monitor|all>",
"dist": "dist",
"loadmod": "loadmod <module>",
"msg": "msg <name> <target> <message...>",

107
core/bot.py
View File

@ -2,6 +2,7 @@ from twisted.internet.protocol import ReconnectingClientFactory
from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from string import digits
from random import randint
@ -19,6 +20,8 @@ class IRCBot(IRCClient):
self.connected = False
self.channels = []
self.net = "".join([x for x in name if not x in digits])
if self.net == "":
error("Network with all numbers: %s" % name)
self.buffer = ""
self.name = name
instance = main.pool[name]
@ -37,6 +40,8 @@ class IRCBot(IRCClient):
self._who = {}
self._getWho = {}
self._names = {}
self.authtype = instance["authtype"]
if self.authtype == "ns":
self.authpass = instance["password"]
@ -71,7 +76,7 @@ class IRCBot(IRCClient):
def privmsg(self, user, channel, msg):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "privmsg")
keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name)
@ -79,7 +84,7 @@ class IRCBot(IRCClient):
def noticed(self, user, channel, msg):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "notice")
keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name)
@ -87,7 +92,7 @@ class IRCBot(IRCClient):
def action(self, user, channel, msg):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "action")
keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name)
@ -109,8 +114,10 @@ class IRCBot(IRCClient):
newnick = nickname + "_"
return newnick
def nickChanged(self, nick):
self.nickname = nick
def nickChanged(self, olduser, newnick):
oldnick, ident, host = self.parsen(olduser)
userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host)
self.nickname = newnick
count.event(self.name, "selfnick")
def irc_ERR_NICKNAMEINUSE(self, prefix, params):
@ -122,10 +129,9 @@ class IRCBot(IRCClient):
sendAll("%s: password mismatch" % self.name)
def who(self, channel):
channel = channel
d = Deferred()
if channel not in self._who:
self._who[channel] = ([], {})
self._who[channel] = ([], [])
self._who[channel][0].append(d)
self.sendLine("WHO %s" % channel)
return d
@ -141,7 +147,7 @@ class IRCBot(IRCClient):
if channel not in self._who:
return
n = self._who[channel][1]
n[nick] = [nick, user, host, server, status, realname]
n.append([nick, user, host, server, status, realname])
count.event(self.name, "whoreply")
monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status})
@ -155,7 +161,50 @@ class IRCBot(IRCClient):
del self._who[channel]
def got_who(self, whoinfo):
userinfo.setWho(self.name, whoinfo[1])
userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
def sanit(self, data):
if len(data) >= 1:
if data[0] in ["!", "~", "&", "@", "%", "+"]:
data = data[1:]
return data
return data
else:
return False
def names(self, channel):
d = Deferred()
if channel not in self._names:
self._names[channel] = ([], [])
self._names[channel][0].append(d)
self.sendLine("NAMES %s" % channel)
return d
def irc_RPL_NAMREPLY(self, prefix, params):
channel = params[2]
nicklist = params[3].split(' ')
if channel not in self._names:
return
n = self._names[channel][1]
n.append(nicklist)
def irc_RPL_ENDOFNAMES(self, prefix, params):
channel = params[1]
if channel not in self._names:
return
callbacks, namelist = self._names[channel]
for cb in callbacks:
cb.callback((channel, namelist))
del self._names[channel]
def got_names(self, nicklist):
newNicklist = []
for i in nicklist[1]:
for x in i:
f = self.sanit(x)
if f:
newNicklist.append(f)
userinfo.initialNames(self.net, nicklist[0], newNicklist)
#twisted sucks so i have to do this to actually get the user info
def irc_JOIN(self, prefix, params):
@ -197,7 +246,7 @@ class IRCBot(IRCClient):
"""
nick = prefix.split('!', 1)[0]
if nick == self.nickname:
self.nickChanged(params[0])
self.nickChanged(prefix, params[0])
else:
self.userRenamed(prefix, params[0])
@ -236,61 +285,63 @@ class IRCBot(IRCClient):
self.join(i)
count.event(self.name, "signedon")
def getWho(self, channel):
self.who(channel).addCallback(self.got_who)
def joined(self, channel):
if not channel in self.channels:
self.channels.append(channel)
self.names(channel).addCallback(self.got_names)
self.who(channel).addCallback(self.got_who)
count.event(self.name, "selfjoin")
if self.name == main.config["Master"][0] and channel == main.config["Master"][1]:
for i in range(len(main.masterbuf)):
self.msg(channel, main.masterbuf.pop(0))
main.saveConf("masterbuf")
lc = LoopingCall(self.getWho, channel)
lc = LoopingCall(self.who, channel)
self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
interval = randint(minint, minint+intrange)
lc.start(interval)
def left(self, channel, message):
def botLeft(self, channel):
if channel in self.channels:
self.channels.remove(channel)
if channel in self._getWho.keys():
lc = self._getWho[channel]
lc.stop()
del self._getWho[channel]
userinfo.delChannel(self.net, channel)
def left(self, channel, message):
keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name)
count.event(self.name, "selfpart")
monitor.event(self.net, channel, {"type": "part", "message": message})
lc = self._getWho[channel]
lc.stop()
self.botLeft(channel)
def kickedFrom(self, channel, kicker, message):
nick, ident, host = self.parsen(kicker)
userinfo.setWhoSingle(self.net, nick, ident, host)
if channel in self.channels:
self.channels.remove(channel)
keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message))
count.event(self.name, "selfkick")
monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message})
lc = self._getWho[channel]
lc.stop()
self.botLeft(channel)
def userJoined(self, user, channel):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.addUser(self.net, channel, nick, user)
count.event(self.name, "join")
monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host})
def userLeft(self, user, channel, message):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.delUser(self.net, channel, nick, user)
keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name)
count.event(self.name, "part")
monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message})
def userQuit(self, user, quitMessage):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.delUserByNetwork(self.net, nick, user)
count.event(self.name, "quit")
keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name)
@ -298,7 +349,8 @@ class IRCBot(IRCClient):
def userKicked(self, kickee, channel, kicker, message):
nick, ident, host = self.parsen(kicker)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.editUser(self.net, channel, nick, kicker)
userinfo.delUserByNick(self.net, channel, kickee)
count.event(self.name, "kick")
keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name)
@ -306,14 +358,13 @@ class IRCBot(IRCClient):
def userRenamed(self, oldname, newname):
nick, ident, host = self.parsen(oldname)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.setWhoSingle(self.net, newname, ident, host)
userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host)
count.event(self.name, "nick")
monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname})
def topicUpdated(self, user, channel, newTopic):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "topic")
keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name)
@ -321,7 +372,7 @@ class IRCBot(IRCClient):
def modeChanged(self, user, channel, toset, modes, args):
nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host)
userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "mode")
argList = list(args)
modeList = [i for i in modes]

6
main.py
View File

@ -1,8 +1,10 @@
from json import load, dump, loads
import redis
from utils.loaders.command_loader import loadCommands
from utils.logging.log import *
r = redis.StrictRedis(unix_socket_path='/tmp/redis.sock', db=0)
configPath = "conf/"
certPath = "cert/"
@ -11,7 +13,7 @@ filemap = {
"keyconf": ["keyword.json", "keyword lists"],
"pool": ["pool.json", "pool"],
"help": ["help.json", "command help"],
"wholist": ["wholist.json", "WHO lists"],
# "wholist": ["wholist.json", "WHO lists"],
"counters": ["counters.json", "counters file"],
"masterbuf": ["masterbuf.json", "master buffer"],
"monitor": ["monitor.json", "monitoring database"],

1
modules/keyword.py
View File

@ -29,7 +29,6 @@ def sendMaster(data):
if not hasMonitors:
main.masterbuf.append(data)
main.saveConf("masterbuf")
warn("Master with no IRC instance defined")
for i in main.MonitorPool:
main.connections[i].send(data)

150
modules/userinfo.py
View File

@ -1,31 +1,131 @@
import main
from string import digits
#from utils.logging.log import *
from utils.logging.log import *
def setWho(network, newObjects):
if not network in main.wholist.keys():
main.wholist[network] = {}
for i in newObjects.keys():
main.wholist[network][i] = newObjects[i]
def getWhoSingle(name, query):
result = main.r.sscan("live.who."+name, 0, query)
if result[1] == []:
return None
return [i.decode() for i in result[1]]
return
def setWhoSingle(network, nick, ident, host):
if network in main.wholist.keys():
if nick in main.wholist[network].keys():
main.wholist[network][nick][1] = ident
main.wholist[network][nick][2] = host
else:
main.wholist[network][nick] = [nick, ident, host, None, None, None]
else:
main.wholist[network] = {nick: [nick, ident, host, None, None, None]}
def getWho(nick):
def getWho(query):
result = {}
for i in main.wholist.keys():
for x in main.wholist[i].keys():
if nick.lower() == x.lower():
if not i in result.keys():
result[i] = []
result[i].append(main.wholist[i][x])
for i in main.pool.keys():
f = getWhoSingle(i, query)
if f:
result[i] = f
print(result)
return result
def getNumWhoEntries(name):
return main.r.scard("live.who."+name)
def getNumTotalWhoEntries():
total = 0
for i in main.pool.keys():
total += getNumWhoEntries(i)
return total
def getNamespace(name, channel, nick):
gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel)
chanspace = "live.chan.%s.%s" % (name, nick)
return [gnamespace, namespace, chanspace]
def initialUsers(name, channel, users):
gnamespace = "live.who.%s" % name
p = main.r.pipeline()
for i in users:
p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2])
p.execute()
def initialNames(name, channel, names):
namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline()
for i in names:
p.sadd(namespace, i)
p.sadd("live.chan."+name+"."+i, channel)
p.execute()
def editUser(name, channel, nick, user):
gnamespace = "live.who.%s" % name
main.r.sadd(gnamespace, user)
def addUser(name, channel, nick, user):
gnamespace, namespace, chanspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
p.sadd(gnamespace, user)
p.sadd(namespace, nick)
p.sadd(chanspace, channel)
p.execute()
def delUser(name, channel, nick, user):
gnamespace, namespace, chanspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
channels = main.r.smembers(chanspace)
p.srem(namespace, nick)
if channels == {channel.encode()}:
p.delete(chanspace)
p.srem(gnamespace, user)
else:
p.srem(chanspace, channel)
p.execute()
def getUserByNick(name, nick):
gnamespace = "live.who.%s" % name
usermatch = main.r.sscan(gnamespace, match=nick+"!*")
if usermatch[1] == []:
return False
else:
if len(usermatch[1]) == 1:
user = usermatch[1][0]
return user
else:
return False
def renameUser(name, oldnick, olduser, newnick, newuser):
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, oldnick)
newchanspace = "live.chan.%s.%s" % (name, newnick)
p = main.r.pipeline()
p.srem(gnamespace, olduser)
p.sadd(gnamespace, newuser)
for i in main.r.smembers(chanspace):
i = i.decode()
p.srem("live.who."+name+"."+i, oldnick)
p.sadd("live.who."+name+"."+i, newnick)
if main.r.exists(chanspace):
p.rename(chanspace, newchanspace)
else:
warn("Key doesn't exist: %s" % chanspace)
p.execute()
def delUserByNick(name, channel, nick):
gnamespace = "live.who.%s" % name
user = getUserByNick(name, nick)
delUser(name, channel, nick, user)
def delUserByNetwork(name, nick, user):
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, nick)
p = main.r.pipeline()
p.srem(gnamespace, user)
for i in main.r.smembers(chanspace):
p.srem("live.who."+name+"."+i.decode(), nick)
p.delete(chanspace)
p.execute()
def delChannel(name, channel):
gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline()
for i in main.r.smembers(namespace):
user = getUserByNick(name, i.decode())
if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}:
if user:
p.srem(gnamespace, user)
p.delete("live.chan."+name+"."+i.decode())
else:
p.srem("live.chan."+name+"."+i.decode(), channel)
p.delete(namespace)
p.execute()