Move user metadata info into redis

pull/1/head
Mark Veidemanis 6 years ago
parent b31b5d40e8
commit 36105c7e9a

@ -1,5 +1,6 @@
import main import main
import modules.counters as count import modules.counters as count
import modules.userinfo as userinfo
class Stats: class Stats:
def __init__(self, register): def __init__(self, register):
@ -13,8 +14,7 @@ class Stats:
numWhoEntries = 0 numWhoEntries = 0
for i in main.IRCPool.keys(): for i in main.IRCPool.keys():
numChannels += len(main.IRCPool[i].channels) numChannels += len(main.IRCPool[i].channels)
for i in main.wholist.keys(): numWhoEntries += userinfo.getNumTotalWhoEntries()
numWhoEntries += len(main.wholist[i].keys())
stats.append("Servers: %s" % len(main.pool.keys())) stats.append("Servers: %s" % len(main.pool.keys()))
stats.append("Online servers: %s" % len(main.IRCPool.keys())) stats.append("Online servers: %s" % len(main.IRCPool.keys()))
stats.append("Channels: %s" % numChannels) stats.append("Channels: %s" % numChannels)
@ -34,20 +34,11 @@ class Stats:
numChannels = 0 numChannels = 0
numWhoEntries = 0 numWhoEntries = 0
failures = 0
if spl[1] in main.IRCPool.keys(): if spl[1] in main.IRCPool.keys():
numChannels += len(main.IRCPool[spl[1]].channels) numChannels += len(main.IRCPool[spl[1]].channels)
else: else:
failure("Name does not exist: %s" % spl[1]) failure("Name does not exist: %s" % spl[1])
failures += 1 numWhoEntries += userinfo.getNumWhoEntries(spl[1])
if spl[1] in main.wholist.keys():
numWhoEntries += len(main.wholist[spl[1]].keys())
else:
failure("Who entry does not exist: %s" % spl[1])
failures += 1
if failures == 2:
failure("No information found, aborting")
return
stats.append("Channels: %s" % numChannels) stats.append("Channels: %s" % numChannels)
stats.append("User records: %s" % numWhoEntries) stats.append("User records: %s" % numWhoEntries)
counterEvents = count.getEvents(spl[1]) counterEvents = count.getEvents(spl[1])

@ -14,9 +14,9 @@ class Who:
rtrn += "Matches from: %s" % i rtrn += "Matches from: %s" % i
rtrn += "\n" rtrn += "\n"
for x in result[i]: for x in result[i]:
x = [y for y in x if not y == None] rtrn += (x)
rtrn += str((", ".join(x)))
rtrn += "\n" rtrn += "\n"
rtrn += "\n"
info(rtrn) info(rtrn)
return return
else: else:

@ -14,8 +14,8 @@
"disable": "disable <name>", "disable": "disable <name>",
"list": "list", "list": "list",
"stats": "stats [<name>]", "stats": "stats [<name>]",
"save": "save <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>", "save": "save <config|keyconf|pool|help|counters|masterbuf|monitor|all>",
"load": "load <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>", "load": "load <config|keyconf|pool|help|counters|masterbuf|monitor|all>",
"dist": "dist", "dist": "dist",
"loadmod": "loadmod <module>", "loadmod": "loadmod <module>",
"msg": "msg <name> <target> <message...>", "msg": "msg <name> <target> <message...>",

@ -2,6 +2,7 @@ from twisted.internet.protocol import ReconnectingClientFactory
from twisted.words.protocols.irc import IRCClient from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from string import digits from string import digits
from random import randint from random import randint
@ -19,6 +20,8 @@ class IRCBot(IRCClient):
self.connected = False self.connected = False
self.channels = [] self.channels = []
self.net = "".join([x for x in name if not x in digits]) self.net = "".join([x for x in name if not x in digits])
if self.net == "":
error("Network with all numbers: %s" % name)
self.buffer = "" self.buffer = ""
self.name = name self.name = name
instance = main.pool[name] instance = main.pool[name]
@ -37,6 +40,8 @@ class IRCBot(IRCClient):
self._who = {} self._who = {}
self._getWho = {} self._getWho = {}
self._names = {}
self.authtype = instance["authtype"] self.authtype = instance["authtype"]
if self.authtype == "ns": if self.authtype == "ns":
self.authpass = instance["password"] self.authpass = instance["password"]
@ -71,7 +76,7 @@ class IRCBot(IRCClient):
def privmsg(self, user, channel, msg): def privmsg(self, user, channel, msg):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "privmsg") count.event(self.name, "privmsg")
keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name)
@ -79,7 +84,7 @@ class IRCBot(IRCClient):
def noticed(self, user, channel, msg): def noticed(self, user, channel, msg):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "notice") count.event(self.name, "notice")
keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name)
@ -87,7 +92,7 @@ class IRCBot(IRCClient):
def action(self, user, channel, msg): def action(self, user, channel, msg):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "action") count.event(self.name, "action")
keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name)
@ -109,8 +114,10 @@ class IRCBot(IRCClient):
newnick = nickname + "_" newnick = nickname + "_"
return newnick return newnick
def nickChanged(self, nick): def nickChanged(self, olduser, newnick):
self.nickname = nick oldnick, ident, host = self.parsen(olduser)
userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host)
self.nickname = newnick
count.event(self.name, "selfnick") count.event(self.name, "selfnick")
def irc_ERR_NICKNAMEINUSE(self, prefix, params): def irc_ERR_NICKNAMEINUSE(self, prefix, params):
@ -122,10 +129,9 @@ class IRCBot(IRCClient):
sendAll("%s: password mismatch" % self.name) sendAll("%s: password mismatch" % self.name)
def who(self, channel): def who(self, channel):
channel = channel
d = Deferred() d = Deferred()
if channel not in self._who: if channel not in self._who:
self._who[channel] = ([], {}) self._who[channel] = ([], [])
self._who[channel][0].append(d) self._who[channel][0].append(d)
self.sendLine("WHO %s" % channel) self.sendLine("WHO %s" % channel)
return d return d
@ -141,7 +147,7 @@ class IRCBot(IRCClient):
if channel not in self._who: if channel not in self._who:
return return
n = self._who[channel][1] n = self._who[channel][1]
n[nick] = [nick, user, host, server, status, realname] n.append([nick, user, host, server, status, realname])
count.event(self.name, "whoreply") count.event(self.name, "whoreply")
monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status})
@ -155,7 +161,50 @@ class IRCBot(IRCClient):
del self._who[channel] del self._who[channel]
def got_who(self, whoinfo): def got_who(self, whoinfo):
userinfo.setWho(self.name, whoinfo[1]) userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
def sanit(self, data):
if len(data) >= 1:
if data[0] in ["!", "~", "&", "@", "%", "+"]:
data = data[1:]
return data
return data
else:
return False
def names(self, channel):
d = Deferred()
if channel not in self._names:
self._names[channel] = ([], [])
self._names[channel][0].append(d)
self.sendLine("NAMES %s" % channel)
return d
def irc_RPL_NAMREPLY(self, prefix, params):
channel = params[2]
nicklist = params[3].split(' ')
if channel not in self._names:
return
n = self._names[channel][1]
n.append(nicklist)
def irc_RPL_ENDOFNAMES(self, prefix, params):
channel = params[1]
if channel not in self._names:
return
callbacks, namelist = self._names[channel]
for cb in callbacks:
cb.callback((channel, namelist))
del self._names[channel]
def got_names(self, nicklist):
newNicklist = []
for i in nicklist[1]:
for x in i:
f = self.sanit(x)
if f:
newNicklist.append(f)
userinfo.initialNames(self.net, nicklist[0], newNicklist)
#twisted sucks so i have to do this to actually get the user info #twisted sucks so i have to do this to actually get the user info
def irc_JOIN(self, prefix, params): def irc_JOIN(self, prefix, params):
@ -197,7 +246,7 @@ class IRCBot(IRCClient):
""" """
nick = prefix.split('!', 1)[0] nick = prefix.split('!', 1)[0]
if nick == self.nickname: if nick == self.nickname:
self.nickChanged(params[0]) self.nickChanged(prefix, params[0])
else: else:
self.userRenamed(prefix, params[0]) self.userRenamed(prefix, params[0])
@ -236,61 +285,63 @@ class IRCBot(IRCClient):
self.join(i) self.join(i)
count.event(self.name, "signedon") count.event(self.name, "signedon")
def getWho(self, channel):
self.who(channel).addCallback(self.got_who)
def joined(self, channel): def joined(self, channel):
if not channel in self.channels: if not channel in self.channels:
self.channels.append(channel) self.channels.append(channel)
self.names(channel).addCallback(self.got_names)
self.who(channel).addCallback(self.got_who) self.who(channel).addCallback(self.got_who)
count.event(self.name, "selfjoin") count.event(self.name, "selfjoin")
if self.name == main.config["Master"][0] and channel == main.config["Master"][1]: if self.name == main.config["Master"][0] and channel == main.config["Master"][1]:
for i in range(len(main.masterbuf)): for i in range(len(main.masterbuf)):
self.msg(channel, main.masterbuf.pop(0)) self.msg(channel, main.masterbuf.pop(0))
main.saveConf("masterbuf") main.saveConf("masterbuf")
lc = LoopingCall(self.getWho, channel) lc = LoopingCall(self.who, channel)
self._getWho[channel] = lc self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"] intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
minint = main.config["Tweaks"]["Delays"]["WhoLoop"] minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
interval = randint(minint, minint+intrange) interval = randint(minint, minint+intrange)
lc.start(interval) lc.start(interval)
def left(self, channel, message): def botLeft(self, channel):
if channel in self.channels: if channel in self.channels:
self.channels.remove(channel) self.channels.remove(channel)
if channel in self._getWho.keys():
lc = self._getWho[channel]
lc.stop()
del self._getWho[channel]
userinfo.delChannel(self.net, channel)
def left(self, channel, message):
keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name)
count.event(self.name, "selfpart") count.event(self.name, "selfpart")
monitor.event(self.net, channel, {"type": "part", "message": message}) monitor.event(self.net, channel, {"type": "part", "message": message})
lc = self._getWho[channel] self.botLeft(channel)
lc.stop()
def kickedFrom(self, channel, kicker, message): def kickedFrom(self, channel, kicker, message):
nick, ident, host = self.parsen(kicker) nick, ident, host = self.parsen(kicker)
userinfo.setWhoSingle(self.net, nick, ident, host)
if channel in self.channels: if channel in self.channels:
self.channels.remove(channel) self.channels.remove(channel)
keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message))
count.event(self.name, "selfkick") count.event(self.name, "selfkick")
monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message})
lc = self._getWho[channel] self.botLeft(channel)
lc.stop()
def userJoined(self, user, channel): def userJoined(self, user, channel):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.addUser(self.net, channel, nick, user)
count.event(self.name, "join") count.event(self.name, "join")
monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host})
def userLeft(self, user, channel, message): def userLeft(self, user, channel, message):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.delUser(self.net, channel, nick, user)
keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name)
count.event(self.name, "part") count.event(self.name, "part")
monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message})
def userQuit(self, user, quitMessage): def userQuit(self, user, quitMessage):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.delUserByNetwork(self.net, nick, user)
count.event(self.name, "quit") count.event(self.name, "quit")
keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name)
@ -298,7 +349,8 @@ class IRCBot(IRCClient):
def userKicked(self, kickee, channel, kicker, message): def userKicked(self, kickee, channel, kicker, message):
nick, ident, host = self.parsen(kicker) nick, ident, host = self.parsen(kicker)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.editUser(self.net, channel, nick, kicker)
userinfo.delUserByNick(self.net, channel, kickee)
count.event(self.name, "kick") count.event(self.name, "kick")
keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name)
@ -306,14 +358,13 @@ class IRCBot(IRCClient):
def userRenamed(self, oldname, newname): def userRenamed(self, oldname, newname):
nick, ident, host = self.parsen(oldname) nick, ident, host = self.parsen(oldname)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host)
userinfo.setWhoSingle(self.net, newname, ident, host)
count.event(self.name, "nick") count.event(self.name, "nick")
monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname})
def topicUpdated(self, user, channel, newTopic): def topicUpdated(self, user, channel, newTopic):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "topic") count.event(self.name, "topic")
keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name)
@ -321,7 +372,7 @@ class IRCBot(IRCClient):
def modeChanged(self, user, channel, toset, modes, args): def modeChanged(self, user, channel, toset, modes, args):
nick, ident, host = self.parsen(user) nick, ident, host = self.parsen(user)
userinfo.setWhoSingle(self.net, nick, ident, host) userinfo.editUser(self.net, channel, nick, user)
count.event(self.name, "mode") count.event(self.name, "mode")
argList = list(args) argList = list(args)
modeList = [i for i in modes] modeList = [i for i in modes]

@ -1,8 +1,10 @@
from json import load, dump, loads from json import load, dump, loads
import redis
from utils.loaders.command_loader import loadCommands from utils.loaders.command_loader import loadCommands
from utils.logging.log import * from utils.logging.log import *
r = redis.StrictRedis(unix_socket_path='/tmp/redis.sock', db=0)
configPath = "conf/" configPath = "conf/"
certPath = "cert/" certPath = "cert/"
@ -11,7 +13,7 @@ filemap = {
"keyconf": ["keyword.json", "keyword lists"], "keyconf": ["keyword.json", "keyword lists"],
"pool": ["pool.json", "pool"], "pool": ["pool.json", "pool"],
"help": ["help.json", "command help"], "help": ["help.json", "command help"],
"wholist": ["wholist.json", "WHO lists"], # "wholist": ["wholist.json", "WHO lists"],
"counters": ["counters.json", "counters file"], "counters": ["counters.json", "counters file"],
"masterbuf": ["masterbuf.json", "master buffer"], "masterbuf": ["masterbuf.json", "master buffer"],
"monitor": ["monitor.json", "monitoring database"], "monitor": ["monitor.json", "monitoring database"],

@ -29,7 +29,6 @@ def sendMaster(data):
if not hasMonitors: if not hasMonitors:
main.masterbuf.append(data) main.masterbuf.append(data)
main.saveConf("masterbuf") main.saveConf("masterbuf")
warn("Master with no IRC instance defined")
for i in main.MonitorPool: for i in main.MonitorPool:
main.connections[i].send(data) main.connections[i].send(data)

@ -1,31 +1,131 @@
import main import main
from string import digits from string import digits
#from utils.logging.log import * from utils.logging.log import *
def setWho(network, newObjects): def getWhoSingle(name, query):
if not network in main.wholist.keys(): result = main.r.sscan("live.who."+name, 0, query)
main.wholist[network] = {} if result[1] == []:
for i in newObjects.keys(): return None
main.wholist[network][i] = newObjects[i] return [i.decode() for i in result[1]]
return def getWho(query):
result = {}
for i in main.pool.keys():
f = getWhoSingle(i, query)
if f:
result[i] = f
print(result)
return result
def getNumWhoEntries(name):
return main.r.scard("live.who."+name)
def getNumTotalWhoEntries():
total = 0
for i in main.pool.keys():
total += getNumWhoEntries(i)
return total
def getNamespace(name, channel, nick):
gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel)
chanspace = "live.chan.%s.%s" % (name, nick)
return [gnamespace, namespace, chanspace]
def initialUsers(name, channel, users):
gnamespace = "live.who.%s" % name
p = main.r.pipeline()
for i in users:
p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2])
p.execute()
def initialNames(name, channel, names):
namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline()
for i in names:
p.sadd(namespace, i)
p.sadd("live.chan."+name+"."+i, channel)
p.execute()
def editUser(name, channel, nick, user):
gnamespace = "live.who.%s" % name
main.r.sadd(gnamespace, user)
def addUser(name, channel, nick, user):
gnamespace, namespace, chanspace = getNamespace(name, channel, nick)
p = main.r.pipeline()
p.sadd(gnamespace, user)
p.sadd(namespace, nick)
p.sadd(chanspace, channel)
p.execute()
def setWhoSingle(network, nick, ident, host): def delUser(name, channel, nick, user):
if network in main.wholist.keys(): gnamespace, namespace, chanspace = getNamespace(name, channel, nick)
if nick in main.wholist[network].keys(): p = main.r.pipeline()
main.wholist[network][nick][1] = ident channels = main.r.smembers(chanspace)
main.wholist[network][nick][2] = host p.srem(namespace, nick)
if channels == {channel.encode()}:
p.delete(chanspace)
p.srem(gnamespace, user)
else:
p.srem(chanspace, channel)
p.execute()
def getUserByNick(name, nick):
gnamespace = "live.who.%s" % name
usermatch = main.r.sscan(gnamespace, match=nick+"!*")
if usermatch[1] == []:
return False
else:
if len(usermatch[1]) == 1:
user = usermatch[1][0]
return user
else: else:
main.wholist[network][nick] = [nick, ident, host, None, None, None] return False
def renameUser(name, oldnick, olduser, newnick, newuser):
gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, oldnick)
newchanspace = "live.chan.%s.%s" % (name, newnick)
p = main.r.pipeline()
p.srem(gnamespace, olduser)
p.sadd(gnamespace, newuser)
for i in main.r.smembers(chanspace):
i = i.decode()
p.srem("live.who."+name+"."+i, oldnick)
p.sadd("live.who."+name+"."+i, newnick)
if main.r.exists(chanspace):
p.rename(chanspace, newchanspace)
else: else:
main.wholist[network] = {nick: [nick, ident, host, None, None, None]} warn("Key doesn't exist: %s" % chanspace)
p.execute()
def getWho(nick): def delUserByNick(name, channel, nick):
result = {} gnamespace = "live.who.%s" % name
for i in main.wholist.keys(): user = getUserByNick(name, nick)
for x in main.wholist[i].keys(): delUser(name, channel, nick, user)
if nick.lower() == x.lower():
if not i in result.keys(): def delUserByNetwork(name, nick, user):
result[i] = [] gnamespace = "live.who.%s" % name
result[i].append(main.wholist[i][x]) chanspace = "live.chan.%s.%s" % (name, nick)
return result p = main.r.pipeline()
p.srem(gnamespace, user)
for i in main.r.smembers(chanspace):
p.srem("live.who."+name+"."+i.decode(), nick)
p.delete(chanspace)
p.execute()
def delChannel(name, channel):
gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline()
for i in main.r.smembers(namespace):
user = getUserByNick(name, i.decode())
if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}:
if user:
p.srem(gnamespace, user)
p.delete("live.chan."+name+"."+i.decode())
else:
p.srem("live.chan."+name+"."+i.decode(), channel)
p.delete(namespace)
p.execute()

Loading…
Cancel
Save