From 45371a88b7b184ecbd5257fefa163166bfdf3aec Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 18 Nov 2017 20:36:51 +0000 Subject: [PATCH 001/394] Create gitignore file --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b137e6e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.pyc +*.db +*.pem +env/ From 6adb59c170c5f6025b5e88f3692b7e469fc9d3a6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 19 Nov 2017 14:46:42 +0000 Subject: [PATCH 002/394] Implement config parsing and make a simple listener --- .gitignore | 1 + threshold.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 threshold.py diff --git a/.gitignore b/.gitignore index b137e6e..6081d82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc *.db *.pem +*.json env/ diff --git a/threshold.py b/threshold.py new file mode 100644 index 0000000..1c38729 --- /dev/null +++ b/threshold.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +from twisted.internet import reactor +from twisted.internet.ssl import DefaultOpenSSLContextFactory +from twisted.internet.protocol import Protocol, Factory +from json import load, dump +from sys import exit + +listener = None +connections = {} + +def log(data): + print("[LOG]", data) + +def debug(data): + print("[DEBUG]", data) + +class Base(Protocol): + def __init__(self, addr): + self.addr = addr + self.authed = False + + def send(self, data): + data += "\r\n" + data = data.encode("utf-8", "replace") + self.transport.write(data) + + def dataReceived(self, data): + log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, data)) + + def connectionMade(self): + log("Connection from %s:%s" % (self.addr.host, self.addr.port)) + self.send("Hello.") + + def connectionLost(self, reason): + global connections + log("Connection lost from %s: %s" % (self.addr.host, reason.getErrorMessage())) + if not listener == None: + if self.addr in connections.keys(): + del connections[self.addr] + else: + debug("Tried to remove a non-existant connection.") + else: + debug("Tried to remove a connection from a listener that wasn't running.") + +class BaseFactory(Factory): + def buildProtocol(self, addr): + global connections + entry = Base(addr) + connections[addr] = entry + return entry + + def send(self, addr, data): + global connections + if addr in connections.keys(): + connection = connections[addr] + connection.send(data) + else: + return + +def getconfig(): + with open("config.json", "r") as f: + config = load(f) + if set(["port", "bind", "usessl"]).issubset(set(config.keys())): + if config["usessl"] == True: + if not set(["cert", "key"]).issubset(set(config.keys())): + debug("SSL is on but certificate or key is not defined") + exit(1) + return config + else: + debug("Mandatory values missing from config") + exit(1) + +if __name__ == "__main__": + config = getconfig() + + listener = BaseFactory() + if config["usessl"] == True: + reactor.listenSSL(config["port"], listener, DefaultOpenSSLContextFactory(config["key"], config["cert"]), interface=config["bind"]) + log("Threshold running with SSL on %s:%s" % (config["bind"], config["port"])) + else: + reactor.listen(config["port"], listener, interface=config["bind"]) + log("Threshold running on %s:%s" % (config["bind"], config["port"])) + + reactor.run() From 013b9b7cfe5036f57f8f62f2d22e880f81b2e59a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 19 Nov 2017 14:50:29 +0000 Subject: [PATCH 003/394] Print config parsing failures instead of sending them to the debug function --- threshold.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/threshold.py b/threshold.py index 1c38729..0cf5c8d 100644 --- a/threshold.py +++ b/threshold.py @@ -63,11 +63,11 @@ def getconfig(): if set(["port", "bind", "usessl"]).issubset(set(config.keys())): if config["usessl"] == True: if not set(["cert", "key"]).issubset(set(config.keys())): - debug("SSL is on but certificate or key is not defined") + print("SSL is on but certificate or key is not defined") exit(1) return config else: - debug("Mandatory values missing from config") + print("Mandatory values missing from config") exit(1) if __name__ == "__main__": From 021a7851cc47cef0a2f0e759bd545d7733667f6f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 19 Nov 2017 14:51:55 +0000 Subject: [PATCH 004/394] Add an example configuration file --- .gitignore | 2 +- example.json | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 example.json diff --git a/.gitignore b/.gitignore index 6081d82..1cc6c46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ *.pyc *.db *.pem -*.json +config.json env/ diff --git a/example.json b/example.json new file mode 100644 index 0000000..baf3a03 --- /dev/null +++ b/example.json @@ -0,0 +1,7 @@ +{ + "port":13867, + "bind":"127.0.0.1", + "usessl":true, + "key":"key.pem", + "cert":"cert.pem" +} From 1fbcec393106de5656e9659c7f01e49fe18b69fc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 20 Nov 2017 19:15:58 +0000 Subject: [PATCH 005/394] Add error and warning functions and use camelcase for getConfig --- threshold.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) mode change 100644 => 100755 threshold.py diff --git a/threshold.py b/threshold.py old mode 100644 new mode 100755 index 0cf5c8d..4572740 --- a/threshold.py +++ b/threshold.py @@ -14,6 +14,13 @@ def log(data): def debug(data): print("[DEBUG]", data) +def warn(data): + print("[WARNING]", data) + +def error(data): + print("[ERROR]", data) + exit(1) + class Base(Protocol): def __init__(self, addr): self.addr = addr @@ -38,9 +45,9 @@ class Base(Protocol): if self.addr in connections.keys(): del connections[self.addr] else: - debug("Tried to remove a non-existant connection.") + warn("Tried to remove a non-existant connection.") else: - debug("Tried to remove a connection from a listener that wasn't running.") + warn("Tried to remove a connection from a listener that wasn't running.") class BaseFactory(Factory): def buildProtocol(self, addr): @@ -57,21 +64,19 @@ class BaseFactory(Factory): else: return -def getconfig(): +def getConfig(): with open("config.json", "r") as f: config = load(f) if set(["port", "bind", "usessl"]).issubset(set(config.keys())): if config["usessl"] == True: if not set(["cert", "key"]).issubset(set(config.keys())): - print("SSL is on but certificate or key is not defined") - exit(1) + error("SSL is on but certificate or key is not defined") return config else: - print("Mandatory values missing from config") - exit(1) + error("Mandatory values missing from config") if __name__ == "__main__": - config = getconfig() + config = getConfig() listener = BaseFactory() if config["usessl"] == True: From 92b9692304363e71bb2a78f2ec474980ea24dc87 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 20 Nov 2017 19:53:25 +0000 Subject: [PATCH 006/394] Move getConfig to a helper class --- threshold.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/threshold.py b/threshold.py index 4572740..3951b1d 100755 --- a/threshold.py +++ b/threshold.py @@ -64,19 +64,21 @@ class BaseFactory(Factory): else: return -def getConfig(): - with open("config.json", "r") as f: - config = load(f) - if set(["port", "bind", "usessl"]).issubset(set(config.keys())): - if config["usessl"] == True: - if not set(["cert", "key"]).issubset(set(config.keys())): - error("SSL is on but certificate or key is not defined") - return config - else: - error("Mandatory values missing from config") +class Helper(object): + def getConfig(self): + with open("config.json", "r") as f: + config = load(f) + if set(["port", "bind", "usessl"]).issubset(set(config.keys())): + if config["usessl"] == True: + if not set(["cert", "key"]).issubset(set(config.keys())): + error("SSL is on but certificate or key is not defined") + return config + else: + error("Mandatory values missing from config") if __name__ == "__main__": - config = getConfig() + helper = Helper() + config = helper.getConfig() listener = BaseFactory() if config["usessl"] == True: From a1051dfa687d3d8f6287a4903cd66863c08accf5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 20 Nov 2017 20:04:03 +0000 Subject: [PATCH 007/394] Strip newlines from received lines and change file extension --- threshold.py => threshold | 1 + 1 file changed, 1 insertion(+) rename threshold.py => threshold (98%) diff --git a/threshold.py b/threshold similarity index 98% rename from threshold.py rename to threshold index 3951b1d..ec43bb3 100755 --- a/threshold.py +++ b/threshold @@ -32,6 +32,7 @@ class Base(Protocol): self.transport.write(data) def dataReceived(self, data): + data = data.rstrip() log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, data)) def connectionMade(self): From f235c5cb44e62c417d044ad2c0cc387e3f014273 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 20 Nov 2017 21:40:04 +0000 Subject: [PATCH 008/394] Implement command parsing and fix plaintext listener and encodings --- example.json | 4 ++- threshold | 71 ++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/example.json b/example.json index baf3a03..c363f4e 100644 --- a/example.json +++ b/example.json @@ -3,5 +3,7 @@ "bind":"127.0.0.1", "usessl":true, "key":"key.pem", - "cert":"cert.pem" + "cert":"cert.pem", + "usepassword":true, + "password":"example" } diff --git a/threshold b/threshold index ec43bb3..574b704 100755 --- a/threshold +++ b/threshold @@ -21,6 +21,18 @@ def error(data): print("[ERROR]", data) exit(1) +def sendData(addr, data): + connections[addr].send(data) + +def sendSuccess(addr, data): + sendData(addr, "[y] " + data) + +def sendFailure(addr, data): + sendData(addr, "[n] " + data) + +def sendInfo(addr, data): + sendData(addr, "[i] " + data) + class Base(Protocol): def __init__(self, addr): self.addr = addr @@ -32,8 +44,9 @@ class Base(Protocol): self.transport.write(data) def dataReceived(self, data): - data = data.rstrip() - log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, data)) + data = data.decode("utf-8", "replace") + log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) + helper.parseCommand(self.addr, self.authed, data) def connectionMade(self): log("Connection from %s:%s" % (self.addr.host, self.addr.port)) @@ -41,7 +54,7 @@ class Base(Protocol): def connectionLost(self, reason): global connections - log("Connection lost from %s: %s" % (self.addr.host, reason.getErrorMessage())) + log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) if not listener == None: if self.addr in connections.keys(): del connections[self.addr] @@ -69,14 +82,62 @@ class Helper(object): def getConfig(self): with open("config.json", "r") as f: config = load(f) - if set(["port", "bind", "usessl"]).issubset(set(config.keys())): + if set(["port", "bind", "usessl", "usepassword"]).issubset(set(config.keys())): if config["usessl"] == True: if not set(["cert", "key"]).issubset(set(config.keys())): error("SSL is on but certificate or key is not defined") + if config["usepassword"] == True: + if not "password" in config.keys(): + error("Password authentication is on but password is not defined") return config else: error("Mandatory values missing from config") + def parseCommand(self, addr, authed, data): + data = data.strip() + spl = data.split() + obj = connections[addr] + + success = lambda data: sendSuccess(addr, data) + failure = lambda data: sendFailure(addr, data) + info = lambda data: sendInfo(addr, data) + + incUsage = lambda: sendFailure(addr, "Incorrect usage") + length = len(spl) + if len(spl) > 0: + cmd = spl[0] + else: + send("No text was sent") + return + if authed == True: + if cmd == "pass": + info("You are already authenticated") + return + elif cmd == "logout": + obj.authed = False + success("Logged out") + return + else: + incUsage() + return + else: + if cmd == "pass": + if length == 2: + if spl[1] == config["password"]: + success("Authenticated successfully") + obj.authed = True + return + else: + failure("Password incorrect") + obj.transport.loseConnection() + return + else: + incUsage() + return + else: + incUsage() + return + if __name__ == "__main__": helper = Helper() config = helper.getConfig() @@ -86,7 +147,7 @@ if __name__ == "__main__": reactor.listenSSL(config["port"], listener, DefaultOpenSSLContextFactory(config["key"], config["cert"]), interface=config["bind"]) log("Threshold running with SSL on %s:%s" % (config["bind"], config["port"])) else: - reactor.listen(config["port"], listener, interface=config["bind"]) + reactor.listenTCP(config["port"], listener, interface=config["bind"]) log("Threshold running on %s:%s" % (config["bind"], config["port"])) reactor.run() From 21ed41071788c2b3ed820025a17f6e9c773af681 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 21 Nov 2017 18:41:18 +0000 Subject: [PATCH 009/394] Use the failure() call instead of send() --- threshold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threshold b/threshold index 574b704..1a0142d 100755 --- a/threshold +++ b/threshold @@ -107,7 +107,7 @@ class Helper(object): if len(spl) > 0: cmd = spl[0] else: - send("No text was sent") + failure("No text was sent") return if authed == True: if cmd == "pass": From acad766dea4f4bdfe2ed1765f56b2aedbd3798e1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 21 Nov 2017 20:16:14 +0000 Subject: [PATCH 010/394] Implement connection information storage and parsing --- example.json | 14 +++---- help.json | 6 +++ threshold | 102 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 99 insertions(+), 23 deletions(-) create mode 100644 help.json diff --git a/example.json b/example.json index c363f4e..07d4b3e 100644 --- a/example.json +++ b/example.json @@ -1,9 +1,9 @@ { - "port":13867, - "bind":"127.0.0.1", - "usessl":true, - "key":"key.pem", - "cert":"cert.pem", - "usepassword":true, - "password":"example" + "port": 13867, + "bind": "127.0.0.1", + "usessl": true, + "key": "key.pem", + "cert": "cert.pem", + "usepassword": true, + "password": "example" } diff --git a/help.json b/help.json new file mode 100644 index 0000000..984b16b --- /dev/null +++ b/help.json @@ -0,0 +1,6 @@ +{ + "pass": "pass [password]", + "logout": "logout", + "connect": "connect [name] [address] [port] [ssl|plain] [nickname]", + "list": "list" +} diff --git a/threshold b/threshold index 1a0142d..076c053 100755 --- a/threshold +++ b/threshold @@ -2,7 +2,7 @@ from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.protocol import Protocol, Factory -from json import load, dump +from json import load, dump, loads from sys import exit listener = None @@ -54,6 +54,7 @@ class Base(Protocol): def connectionLost(self, reason): global connections + self.authed = False log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) if not listener == None: if self.addr in connections.keys(): @@ -93,7 +94,36 @@ class Helper(object): else: error("Mandatory values missing from config") + def getPool(self): + with open("pool.json", "r") as f: + data = f.read() + if not data == "": + pool = loads(data.strip()) + return pool + else: + return {} + + def savePool(self): + global pool + with open("pool.json", "w") as f: + dump(pool, f, indent=4) + return + + def getHelp(self): + with open("help.json", "r") as f: + help = load(f) + return help + + def incorrectUsage(self, addr, mode): + if mode == None: + sendFailure(addr, "Incorrect usage") + return + if mode in help.keys(): + sendFailure(addr, "Usage: " + help[mode]) + return + def parseCommand(self, addr, authed, data): + global pool data = data.strip() spl = data.split() obj = connections[addr] @@ -102,45 +132,85 @@ class Helper(object): failure = lambda data: sendFailure(addr, data) info = lambda data: sendInfo(addr, data) - incUsage = lambda: sendFailure(addr, "Incorrect usage") + incUsage = lambda mode: self.incorrectUsage(addr, mode) length = len(spl) if len(spl) > 0: cmd = spl[0] else: failure("No text was sent") return + if authed == True: - if cmd == "pass": + if cmd == "help": + helpMap = [] + for i in help.keys(): + helpMap.append("%s: %s" % (i, help[i])) + info("\n".join(helpMap)) + return + + elif cmd == "pass": info("You are already authenticated") return + elif cmd == "logout": obj.authed = False success("Logged out") return + + elif cmd == "list": + poolMap = [] + for i in pool.keys(): + poolMap.append("%s: %s" % (i, pool[i])) + info("\n".join(poolMap)) + return + + elif cmd == "connect": + if length == 6: + + if spl[1] in pool.keys(): + failure("Name already exists: %s" % spl[1]) + return + + protocol = spl[4].lower() + + if not protocol in ["ssl", "plain"]: + incUsage("connect") + return + + pool[spl[1]] = { "address": spl[2], + "port": spl[3], + "protocol": protocol, + "nickname": spl[5], + } + success("Successfully created bot") + self.savePool() + return + else: + incUsage("connect") + return + else: - incUsage() + incUsage(None) return else: - if cmd == "pass": - if length == 2: - if spl[1] == config["password"]: - success("Authenticated successfully") - obj.authed = True - return - else: - failure("Password incorrect") - obj.transport.loseConnection() - return + if cmd == "pass" and length == 2: + if spl[1] == config["password"]: + success("Authenticated successfully") + obj.authed = True + return else: - incUsage() + failure("Password incorrect") + obj.transport.loseConnection() return else: - incUsage() + incUsage(None) return if __name__ == "__main__": helper = Helper() config = helper.getConfig() + pool = helper.getPool() + help = helper.getHelp() listener = BaseFactory() if config["usessl"] == True: From 2355412a2e73138f426059ed7d1092058636ff8f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 21 Nov 2017 20:19:40 +0000 Subject: [PATCH 011/394] Add pool file to gitignore and create an empty pool file --- .gitignore | 1 + pool.json | 0 2 files changed, 1 insertion(+) create mode 100644 pool.json diff --git a/.gitignore b/.gitignore index 1cc6c46..1c8959f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.db *.pem config.json +pool.json env/ diff --git a/pool.json b/pool.json new file mode 100644 index 0000000..e69de29 From 79d1e2a86cd358549dcacc1dd04c9ede41cff089 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 23 Nov 2017 18:32:30 +0000 Subject: [PATCH 012/394] Capitalise configuration keys and force-authenticate when UsePassword is disabled --- example.json | 14 +++++++------- threshold | 24 +++++++++++++----------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/example.json b/example.json index 07d4b3e..78f64ce 100644 --- a/example.json +++ b/example.json @@ -1,9 +1,9 @@ { - "port": 13867, - "bind": "127.0.0.1", - "usessl": true, - "key": "key.pem", - "cert": "cert.pem", - "usepassword": true, - "password": "example" + "Port": 13867, + "BindAddress": "127.0.0.1", + "UseSSL": true, + "ListenerKey": "key.pem", + "ListenerCertificate": "cert.pem", + "UsePassword": true, + "Password": "s" } diff --git a/threshold b/threshold index 076c053..b8569af 100755 --- a/threshold +++ b/threshold @@ -37,6 +37,8 @@ class Base(Protocol): def __init__(self, addr): self.addr = addr self.authed = False + if config["UsePassword"] == False: + self.authed = True def send(self, data): data += "\r\n" @@ -83,12 +85,12 @@ class Helper(object): def getConfig(self): with open("config.json", "r") as f: config = load(f) - if set(["port", "bind", "usessl", "usepassword"]).issubset(set(config.keys())): - if config["usessl"] == True: - if not set(["cert", "key"]).issubset(set(config.keys())): + if set(["Port", "BindAddress", "UseSSL", "UsePassword"]).issubset(set(config.keys())): + if config["UseSSL"] == True: + if not set(["ListenerKey", "ListenerCertificate"]).issubset(set(config.keys())): error("SSL is on but certificate or key is not defined") - if config["usepassword"] == True: - if not "password" in config.keys(): + if config["UsePassword"] == True: + if not "Password" in config.keys(): error("Password authentication is on but password is not defined") return config else: @@ -194,7 +196,7 @@ class Helper(object): return else: if cmd == "pass" and length == 2: - if spl[1] == config["password"]: + if spl[1] == config["Password"]: success("Authenticated successfully") obj.authed = True return @@ -213,11 +215,11 @@ if __name__ == "__main__": help = helper.getHelp() listener = BaseFactory() - if config["usessl"] == True: - reactor.listenSSL(config["port"], listener, DefaultOpenSSLContextFactory(config["key"], config["cert"]), interface=config["bind"]) - log("Threshold running with SSL on %s:%s" % (config["bind"], config["port"])) + if config["UseSSL"] == True: + reactor.listenSSL(config["Port"], listener, DefaultOpenSSLContextFactory(config["ListenerKey"], config["ListenerCertificate"]), interface=config["BindAddress"]) + log("Threshold running with SSL on %s:%s" % (config["BindAddress"], config["Port"])) else: - reactor.listenTCP(config["port"], listener, interface=config["bind"]) - log("Threshold running on %s:%s" % (config["bind"], config["port"])) + reactor.listenTCP(config["Port"], listener, interface=config["BindAddress"]) + log("Threshold running on %s:%s" % (config["BindAddress"], config["Port"])) reactor.run() From 587c3bb87c5de8c9e85a873ae77ac17ad8129940 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 23 Nov 2017 18:39:08 +0000 Subject: [PATCH 013/394] Implement reloading the configuration at runtime --- help.json | 3 ++- threshold | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/help.json b/help.json index 984b16b..a470e49 100644 --- a/help.json +++ b/help.json @@ -2,5 +2,6 @@ "pass": "pass [password]", "logout": "logout", "connect": "connect [name] [address] [port] [ssl|plain] [nickname]", - "list": "list" + "list": "list", + "rehash": "rehash" } diff --git a/threshold b/threshold index b8569af..19c24ef 100755 --- a/threshold +++ b/threshold @@ -150,6 +150,11 @@ class Helper(object): info("\n".join(helpMap)) return + elif cmd == "rehash": + global config + config = helper.getConfig() + success("Configuration rehashed successfully") + elif cmd == "pass": info("You are already authenticated") return From 12be21fb4efbe141554e6f202dda036418f3c443 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 23 Nov 2017 18:45:18 +0000 Subject: [PATCH 014/394] Implement removing servers and rename some commands --- help.json | 3 ++- pool.json | 1 + threshold | 18 ++++++++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/help.json b/help.json index a470e49..ef6f7a5 100644 --- a/help.json +++ b/help.json @@ -1,7 +1,8 @@ { "pass": "pass [password]", "logout": "logout", - "connect": "connect [name] [address] [port] [ssl|plain] [nickname]", + "add": "add [name] [address] [port] [ssl|plain] [nickname]", + "del": "del [name", "list": "list", "rehash": "rehash" } diff --git a/pool.json b/pool.json index e69de29..9e26dfe 100644 --- a/pool.json +++ b/pool.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/threshold b/threshold index 19c24ef..08e83a4 100755 --- a/threshold +++ b/threshold @@ -171,7 +171,7 @@ class Helper(object): info("\n".join(poolMap)) return - elif cmd == "connect": + elif cmd == "add": if length == 6: if spl[1] in pool.keys(): @@ -193,7 +193,21 @@ class Helper(object): self.savePool() return else: - incUsage("connect") + incUsage("add") + return + + elif cmd == "del": + if length == 2: + + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + del pool[spl[1]] + success("Successfully removed bot") + self.savePool() + return + else: + incUsage("del") return else: From 4f70cf69b3527bd04eb6c5b98329d6fd6c81d9e3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 23 Nov 2017 19:11:29 +0000 Subject: [PATCH 015/394] Implement option viewing and setting --- help.json | 7 ++++--- threshold | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/help.json b/help.json index ef6f7a5..e320efa 100644 --- a/help.json +++ b/help.json @@ -1,8 +1,9 @@ { - "pass": "pass [password]", + "pass": "pass ", "logout": "logout", - "add": "add [name] [address] [port] [ssl|plain] [nickname]", - "del": "del [name", + "add": "add
", + "del": "del ", + "mod": "mod [] []", "list": "list", "rehash": "rehash" } diff --git a/threshold b/threshold index 08e83a4..7835282 100755 --- a/threshold +++ b/threshold @@ -167,7 +167,10 @@ class Helper(object): elif cmd == "list": poolMap = [] for i in pool.keys(): - poolMap.append("%s: %s" % (i, pool[i])) + poolMap.append("Server: %s" % i) + for x in pool[i].keys(): + poolMap.append("%s: %s" % (x, pool[i][x])) + poolMap.append("\n") info("\n".join(poolMap)) return @@ -188,6 +191,17 @@ class Helper(object): "port": spl[3], "protocol": protocol, "nickname": spl[5], + "username": None, + "realname": None, + "userinfo": None, + "finger": None, + "version": None, + "source": None, + "authtype": None, + "password": None, + "authentity": None, + "key": None, + "certificate": None, } success("Successfully created bot") self.savePool() @@ -210,6 +224,44 @@ class Helper(object): incUsage("del") return + elif cmd == "mod": + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + optionMap = ["Viewing options for %s" % spl[1]] + for i in pool[spl[1]].keys(): + optionMap.append("%s: %s" % (i, pool[spl[1]][i])) + info("\n".join(optionMap)) + return + + elif length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[2] in pool[spl[1]].keys(): + failure("No such key: %s" % spl[2]) + return + info("%s: %s" % (spl[2], pool[spl[1]][spl[2]])) + return + + elif length == 4: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[2] in pool[spl[1]].keys(): + failure("No such key: %s" % spl[2]) + return + if spl[3] == pool[spl[1]][spl[2]]: + failure("Value already exists: %s" % spl[3]) + return + pool[spl[1]][spl[2]] = spl[3] + success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) + return + + else: + incUsage("mod") + else: incUsage(None) return From c3aa3715c6a7fcacc8a9659db59878e416f94c68 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 23 Nov 2017 19:19:48 +0000 Subject: [PATCH 016/394] Add port and protocol validation --- pool.json | 1 - threshold | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) delete mode 100644 pool.json diff --git a/pool.json b/pool.json deleted file mode 100644 index 9e26dfe..0000000 --- a/pool.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/threshold b/threshold index 7835282..5f1790c 100755 --- a/threshold +++ b/threshold @@ -187,6 +187,12 @@ class Helper(object): incUsage("connect") return + try: + int(spl[3]) + except: + failure("Port must be an integer, not %s" % spl[3]) + return + pool[spl[1]] = { "address": spl[2], "port": spl[3], "protocol": protocol, @@ -252,6 +258,16 @@ class Helper(object): if not spl[2] in pool[spl[1]].keys(): failure("No such key: %s" % spl[2]) return + if spl[2] == "port": + try: + int(spl[3]) + except: + failure("Port must be an integer, not %s" % spl[3]) + return + if spl[2] == "protocol": + if not spl[3] in ["ssl", "plain"]: + failure("Protocol must be ssl or plain, not %s" % spl[3]) + return if spl[3] == pool[spl[1]][spl[2]]: failure("Value already exists: %s" % spl[3]) return From 0912ac71cb7e64dfd0e725edcc1c512732080f0c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 23 Nov 2017 20:37:00 +0000 Subject: [PATCH 017/394] Implement connecting to IRC --- help.json | 2 + threshold | 235 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 230 insertions(+), 7 deletions(-) diff --git a/help.json b/help.json index e320efa..e69d776 100644 --- a/help.json +++ b/help.json @@ -4,6 +4,8 @@ "add": "add
", "del": "del ", "mod": "mod [] []", + "enable": "enable ", "list": "list", "rehash": "rehash" } diff --git a/threshold b/threshold index 5f1790c..0ae261b 100755 --- a/threshold +++ b/threshold @@ -2,11 +2,15 @@ from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.protocol import Protocol, Factory +from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint, connectProtocol +from twisted.words.protocols.irc import IRCClient + from json import load, dump, loads from sys import exit listener = None connections = {} +IRCPool = {} def log(data): print("[LOG]", data) @@ -33,6 +37,136 @@ def sendFailure(addr, data): def sendInfo(addr, data): sendData(addr, "[i] " + data) +class IRCBot(IRCClient): + def __init__(self, name): + self.name = name + instance = pool[name] + + self.nickname = instance["nickname"] + self.realname = instance["realname"] + self.username = instance["username"] + self.userinfo = instance["userinfo"] + self.fingerReply = instance["finger"] + self.versionName = instance["version"] + self.versionNum = None + self.versionEnv = None + self.sourceURL = instance["source"] + + self._who = {} + self._getWho = {} + self.wholist = {} + + self.authtype = instance["authtype"] + if self.authtype == "ns": + self.authpass = instance["password"] + self.authentity = instance["authentity"] + else: + self.password = instance["password"] + + def refresh(self): + instance = pool[name] + if not instance["nickname"] == self.nickname: + self.nickname = instance["nickname"] + self.setNick(self.nickname) + + self.userinfo = instance["userinfo"] + self.fingerReply = instance["finger"] + self.versionName = instance["version"] + self.versionNum = None + self.versionEnv = None + self.sourceURL = instance["source"] + + def sanitize(self, user): + user = user.replace("!", "") + user = user.replace("~", "") + user = user.replace("&", "") + user = user.replace("@", "") + user = user.replace("%", "") + user = user.replace("+", "") + return user + + def setNick(self, nickname): + self._attemptedNick = nickname + self.sendLine("NICK %s" % nickname) + self.nickname = nickname + + def alterCollidedNick(self, nickname): + newnick = nickname + "_" + return newnick + + def irc_ERR_NICKNAMEINUSE(self, prefix, params): + self._attemptedNick = self.alterCollidedNick(self._attemptedNick) + self.setNick(self._attemptedNick) + + def irc_ERR_PASSWDMISMATCH(self, prefix, params): + ircLog(self.ident, "Password mismatch") + manager.handleWrongPassword(self.ident, prefix, params) + + def irc_RPL_NAMREPLY(self, prefix, params): + channel = params[2] + nicklist = params[3].split(' ') + + if channel not in self._names: + return + n = self._names[channel][1] + n += nicklist + + def irc_RPL_ENDOFNAMES(self, prefix, params): + channel = params[1] + if channel not in self._names: + return + callbacks, namelist = self._names[channel] + for cb in callbacks: + cb.callback((channel, namelist)) + del self._names[channel] + + def got_names(self, nicklist): + ncklist = [] + for i in nicklist[1]: + ncklist.append(self.sanitize(i)) + self.nameslist[nicklist[0]] = ncklist + + def who(self, channel): + channel = channel + d = defer.Deferred() + if channel not in self._who: + self._who[channel] = ([], {}) + self._who[channel][0].append(d) + self.sendLine("WHO %s" % channel) + return d + + def irc_RPL_WHOREPLY(self, prefix, params): + channel = params[1] + user = params[2] + host = params[3] + server = params[4] + nick = params[5] + status = params[6] + realname = params[7] + if channel not in self._who: + return + n = self._who[channel][1] + n[nick] = [nick, user, host, server, status, realname] + + def irc_RPL_ENDOFWHO(self, prefix, params): + channel = params[1] + if channel not in self._who: + return + callbacks, info = self._who[channel] + for cb in callbacks: + cb.callback((channel, info)) + del self._who[channel] + + def got_who(self, whoinfo): + self.wholist[whoinfo[0]] = whoinfo[1] + + def signedOn(self): + if self.authtype == "ns": + self.msg(self.authentity, "IDENTIFY %s" % self.nspass) + + def joined(self, channel): + self.who(channel).addCallback(self.got_who) + class Base(Protocol): def __init__(self, addr): self.addr = addr @@ -124,6 +258,37 @@ class Helper(object): sendFailure(addr, "Usage: " + help[mode]) return + def addBot(self, name): + global IRCPool + instance = pool[name] + if instance["protocol"] == "plain": + if instance["bind"] == None: + point = TCP4ClientEndpoint(reactor, instance["host"], int(instance["port"]), timeout=int(instance["timeout"])) + bot = IRCBot(name) + IRCPool[name] = bot + d = connectProtocol(point, bot) + return + else: + point = TCP4ClientEndpoint(reactor, instance["host"], int(instance["port"]), timeout=int(instance["timeout"]), bindAddress=instance["bind"]) + bot = IRCBot(name) + IRCPool[name] = bot + d = connectProtocol(point, bot) + return + elif instance["protocol"] == "ssl": + contextFactory = DefaultOpenSSLContextFactory(instance["key"].encode("utf-8", "replace"), instance["certificate"].encode("utf-8", "replace")) + if instance["bind"] == None: + point = SSL4ClientEndpoint(reactor, instance["host"], int(instance["port"]), contextFactory, timeout=int(instance["timeout"])) + bot = IRCBot(name) + IRCPool[name] = bot + d = connectProtocol(point, bot) + return + else: + point = SSL4ClientEndpoint(reactor, instance["host"], int(instance["port"]), contextFactory, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) + bot = IRCBot(name) + IRCPool[name] = bot + d = connectProtocol(point, bot) + return + def parseCommand(self, addr, authed, data): global pool data = data.strip() @@ -174,6 +339,33 @@ class Helper(object): info("\n".join(poolMap)) return + elif cmd == "enable": + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + pool[spl[1]]["enabled"] = True + self.addBot(spl[1]) + success("Successfully enabled bot %s" % spl[1]) + return + else: + incUsage("enable") + return + + elif cmd == "disable": + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + pool[spl[1]]["enabled"] = False + if spl[1] in IRCPool.keys(): + IRCPool[spl[1]].transport.loseConnection() + success("Successfully disabled bot %s" % spl[1]) + return + else: + incUsage("disable") + return + elif cmd == "add": if length == 6: @@ -193,9 +385,11 @@ class Helper(object): failure("Port must be an integer, not %s" % spl[3]) return - pool[spl[1]] = { "address": spl[2], + pool[spl[1]] = { "host": spl[2], "port": spl[3], "protocol": protocol, + "bind": None, + "timeout": 30, "nickname": spl[5], "username": None, "realname": None, @@ -205,10 +399,13 @@ class Helper(object): "source": None, "authtype": None, "password": None, - "authentity": None, - "key": None, - "certificate": None, + "authentity": "NickServ", + "key": config["ListenerKey"], + "certificate": config["ListenerCertificate"], + "enabled": config["ConnectOnCreate"], } + if config["ConnectOnCreate"] == True: + self.addBot(spl[1]) success("Successfully created bot") self.savePool() return @@ -223,6 +420,9 @@ class Helper(object): failure("Name does not exist: %s" % spl[1]) return del pool[spl[1]] + if spl[1] in IRCPool.keys(): + IRCPool[spl[1]].transport.loseConnection() + del IRCPool[spl[1]] success("Successfully removed bot") self.savePool() return @@ -231,6 +431,7 @@ class Helper(object): return elif cmd == "mod": + toUnset = False if length == 2: if not spl[1] in pool.keys(): failure("Name does not exist: %s" % spl[1]) @@ -258,11 +459,11 @@ class Helper(object): if not spl[2] in pool[spl[1]].keys(): failure("No such key: %s" % spl[2]) return - if spl[2] == "port": + if spl[2] in ["port", "timeout"]: try: int(spl[3]) except: - failure("Port must be an integer, not %s" % spl[3]) + failure("Value must be an integer, not %s" % spl[3]) return if spl[2] == "protocol": if not spl[3] in ["ssl", "plain"]: @@ -271,8 +472,25 @@ class Helper(object): if spl[3] == pool[spl[1]][spl[2]]: failure("Value already exists: %s" % spl[3]) return + + if spl[3].lower() in ["none", "nil"]: + spl[3] = None + toUnset = True + + if spl[2] == "authtype": + if not spl[3] in ["sp", "ns"]: + failure("Authtype must be sp or ns, not %s" % spl[3]) + return + if spl[2] == "enabled": + failure("Use the enable and disable commands to manage this") + return + pool[spl[1]][spl[2]] = spl[3] - success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) + self.savePool() + if toUnset: + success("Successfully unset key %s on %s" % (spl[2], spl[3], spl[1])) + else: + success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return else: @@ -300,6 +518,9 @@ if __name__ == "__main__": config = helper.getConfig() pool = helper.getPool() help = helper.getHelp() + for i in pool.keys(): + if pool[i]["enabled"] == True: + helper.addBot(i) listener = BaseFactory() if config["UseSSL"] == True: From 73e16aebae212030c05b61fb1c168bca0ef4f290 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 24 Nov 2017 19:16:08 +0000 Subject: [PATCH 018/394] Implement join/part, channel tracking and retrieving variables from IRCBot classes at runtime --- help.json | 2 + threshold | 136 ++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 108 insertions(+), 30 deletions(-) diff --git a/help.json b/help.json index e69d776..98cd1b4 100644 --- a/help.json +++ b/help.json @@ -4,6 +4,8 @@ "add": "add
", "del": "del ", "mod": "mod [] []", + "get": "get ", + "join": "join []", "enable": "enable ", "list": "list", diff --git a/threshold b/threshold index 0ae261b..8253a77 100755 --- a/threshold +++ b/threshold @@ -1,5 +1,6 @@ #!/usr/bin/env python from twisted.internet import reactor +from twisted.internet.defer import Deferred from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.protocol import Protocol, Factory from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint, connectProtocol @@ -39,6 +40,9 @@ def sendInfo(addr, data): class IRCBot(IRCClient): def __init__(self, name): + self.connected = False + self.channels = [] + self.name = name instance = pool[name] @@ -64,7 +68,7 @@ class IRCBot(IRCClient): self.password = instance["password"] def refresh(self): - instance = pool[name] + instance = pool[self.name] if not instance["nickname"] == self.nickname: self.nickname = instance["nickname"] self.setNick(self.nickname) @@ -76,6 +80,13 @@ class IRCBot(IRCClient): self.versionEnv = None self.sourceURL = instance["source"] + def get(self, var): + try: + result = getattr(self, var) + except AttributeError: + result = None + return result + def sanitize(self, user): user = user.replace("!", "") user = user.replace("~", "") @@ -99,36 +110,12 @@ class IRCBot(IRCClient): self.setNick(self._attemptedNick) def irc_ERR_PASSWDMISMATCH(self, prefix, params): - ircLog(self.ident, "Password mismatch") - manager.handleWrongPassword(self.ident, prefix, params) - - def irc_RPL_NAMREPLY(self, prefix, params): - channel = params[2] - nicklist = params[3].split(' ') - - if channel not in self._names: - return - n = self._names[channel][1] - n += nicklist - - def irc_RPL_ENDOFNAMES(self, prefix, params): - channel = params[1] - if channel not in self._names: - return - callbacks, namelist = self._names[channel] - for cb in callbacks: - cb.callback((channel, namelist)) - del self._names[channel] - - def got_names(self, nicklist): - ncklist = [] - for i in nicklist[1]: - ncklist.append(self.sanitize(i)) - self.nameslist[nicklist[0]] = ncklist + log("%s: password mismatch" % self.name) + helper.sendAll("%s: password mismatch" % self.name) def who(self, channel): channel = channel - d = defer.Deferred() + d = Deferred() if channel not in self._who: self._who[channel] = ([], {}) self._who[channel][0].append(d) @@ -161,12 +148,37 @@ class IRCBot(IRCClient): self.wholist[whoinfo[0]] = whoinfo[1] def signedOn(self): + self.connected = True if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) def joined(self, channel): + if not channel in self.channels: + self.channels.append(channel) self.who(channel).addCallback(self.got_who) + def left(self, channel): + if channel in self.channels: + self.channels.remove(channel) + + def kickedFrom(self, channel, kicker, message): + if channel in self.channels: + self.channels.remove(channel) + + def connectionLost(self, reason): + self.connected = False + self.channels = [] + error = reason.getErrorMessage() + log("%s: connection lost: %s" % (self.name, error)) + helper.sendAll("%s: connection lost: %s" % (self.name, error)) + + def connectionFailed(self, reason): + self.connected = False + self.channels = [] + error = reason.getErrorMessage() + log("%s: connection failed: %s" % (self.name, error)) + helper.sendAll("%s: connection failed: %s" % (self.name, error)) + class Base(Protocol): def __init__(self, addr): self.addr = addr @@ -258,9 +270,16 @@ class Helper(object): sendFailure(addr, "Usage: " + help[mode]) return + def sendAll(self, data): + global connections + for i in connections: + connections[i].send(data) + return + def addBot(self, name): global IRCPool instance = pool[name] + log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) if instance["protocol"] == "plain": if instance["bind"] == None: point = TCP4ClientEndpoint(reactor, instance["host"], int(instance["port"]), timeout=int(instance["timeout"])) @@ -345,6 +364,7 @@ class Helper(object): failure("Name does not exist: %s" % spl[1]) return pool[spl[1]]["enabled"] = True + helper.savePool() self.addBot(spl[1]) success("Successfully enabled bot %s" % spl[1]) return @@ -358,14 +378,67 @@ class Helper(object): failure("Name does not exist: %s" % spl[1]) return pool[spl[1]]["enabled"] = False + helper.savePool() if spl[1] in IRCPool.keys(): - IRCPool[spl[1]].transport.loseConnection() + if IRCPool[spl[1]].connected == True: + IRCPool[spl[1]].transport.loseConnection() success("Successfully disabled bot %s" % spl[1]) return else: incUsage("disable") return + elif cmd == "join": + if length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + IRCPool[spl[1]].join(spl[2]) + success("Joined %s" % spl[2]) + return + elif length == 4: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + IRCPool[spl[1]].join(spl[2], spl[3]) + success("Joined %s with key %s" % (spl[2], spl[3])) + return + else: + incUsage("join") + + elif cmd == "part": + if length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + IRCPool[spl[1]].part(spl[2]) + success("Left %s" % spl[2]) + return + else: + incUsage("part") + + elif cmd == "get": + if length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + info(str(IRCPool[spl[1]].get(spl[2]))) + return + else: + incUsage("get") + elif cmd == "add": if length == 6: @@ -421,7 +494,8 @@ class Helper(object): return del pool[spl[1]] if spl[1] in IRCPool.keys(): - IRCPool[spl[1]].transport.loseConnection() + if IRCPool[spl[1]].connected == True: + IRCPool[spl[1]].transport.loseConnection() del IRCPool[spl[1]] success("Successfully removed bot") self.savePool() @@ -486,6 +560,8 @@ class Helper(object): return pool[spl[1]][spl[2]] = spl[3] + if spl[1] in IRCPool.keys(): + IRCPool[spl[1]].refresh() self.savePool() if toUnset: success("Successfully unset key %s on %s" % (spl[2], spl[3], spl[1])) From b8e9efa41ead46fc0be8d72d5d6e749b5e7c8700 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 00:38:06 +0000 Subject: [PATCH 019/394] Implement adding and removing keywords, and setting and viewing the keyword master (reporting target) --- help.json | 1 + threshold | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/help.json b/help.json index 98cd1b4..612f2fe 100644 --- a/help.json +++ b/help.json @@ -5,6 +5,7 @@ "del": "del ", "mod": "mod [] []", "get": "get ", + "key": "key {[] []} []", "join": "join []", "enable": "enable ", diff --git a/threshold b/threshold index 8253a77..66af64d 100755 --- a/threshold +++ b/threshold @@ -242,6 +242,12 @@ class Helper(object): else: error("Mandatory values missing from config") + def saveConfig(self): + global config + with open("config.json", "w") as f: + dump(config, f, indent=4) + return + def getPool(self): with open("pool.json", "r") as f: data = f.read() @@ -308,9 +314,27 @@ class Helper(object): d = connectProtocol(point, bot) return + def addKeyword(self, keyword): + if keyword in config["Keywords"]: + return "EXISTS" + else: + for i in config["Keywords"]: + if i in keyword or keyword in i: + return "ISIN" + config["Keywords"].append(keyword) + helper.saveConfig() + return True + + def delKeyword(self, keyword): + if not keyword in config["Keywords"]: + return "NOKEY" + config["Keywords"].remove(keyword) + helper.saveConfig() + return True + def parseCommand(self, addr, authed, data): global pool - data = data.strip() + data = data.strip("\n") spl = data.split() obj = connections[addr] @@ -411,6 +435,7 @@ class Helper(object): return else: incUsage("join") + return elif cmd == "part": if length == 3: @@ -425,6 +450,7 @@ class Helper(object): return else: incUsage("part") + return elif cmd == "get": if length == 3: @@ -438,6 +464,66 @@ class Helper(object): return else: incUsage("get") + return + + elif cmd == "key": + if data.startswith("key add"): + if not data == "key add " and not data == "key add": + keywordsToAdd = data[8:] + keywords = keywordsToAdd.split(",") + for keyword in keywords: + rtrn = self.addKeyword(keyword) + if rtrn == "EXISTS": + failure("Keyword already exists: %s" % keyword) + elif rtrn == "ISIN": + failure("Keyword already matched: %s" % keyword) + elif rtrn == True: + success("Keyword added: %s" % keyword) + return + else: + incUsage("key") + return + elif data.startswith("key del"): + if not data == "key del " and not data == "key del": + keywordsToDel = data[8:] + keywords = keywordsToDel.split(",") + for keyword in keywords: + rtrn = self.delKeyword(keyword) + if rtrn == "NOKEY": + failure("Keyword does not exist: %s" % keyword) + elif rtrn == True: + success("Keyword deleted: %s" % keyword) + return + else: + incUsage("key") + return + + elif spl[1] == "show": + info(", ".join(config["Keywords"])) + return + + elif spl[1] == "master": + if length == 4: + if not spl[2] in pool.keys(): + failure("Name does not exist: %s" % spl[2]) + return + if spl[2] in IRCPool.keys(): + if not spl[3] in IRCPool[spl[2]].channels: + info("Bot not on channel: %s" % spl[3]) + config["Master"] = [spl[2], spl[3]] + helper.saveConfig() + success("Master set to %s on %s" % (spl[3], spl[2])) + return + elif length == 2: + info(" - ".join(config["Master"])) + return + else: + incUsage("key") + return + + else: + incUsage("key") + return elif cmd == "add": if length == 6: @@ -571,6 +657,7 @@ class Helper(object): else: incUsage("mod") + return else: incUsage(None) From 742d9c0c91cfaddf344a9748c583fa5dad2d8582 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 18:42:32 +0000 Subject: [PATCH 020/394] Implement delivery of keyword match notifications --- example.json | 5 ++++- threshold | 42 +++++++++++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/example.json b/example.json index 78f64ce..685e4fd 100644 --- a/example.json +++ b/example.json @@ -5,5 +5,8 @@ "ListenerKey": "key.pem", "ListenerCertificate": "cert.pem", "UsePassword": true, - "Password": "s" + "ConnectOnCreate": false, + "Password": "s", + "Keywords": [], + "Master": [] } diff --git a/threshold b/threshold index 66af64d..c1795aa 100755 --- a/threshold +++ b/threshold @@ -67,6 +67,21 @@ class IRCBot(IRCClient): else: self.password = instance["password"] + def privmsg(self, user, channel, msg): + toSend = helper.isKeyword(msg) + if toSend: + helper.sendMaster("MATCH PRV %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + + def noticed(self, user, channel, msg): + toSend = helper.isKeyword(msg) + if toSend: + helper.sendMaster("MATCH NOT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + + def action(self, user, channel, msg): + toSend = helper.isKeyword(msg) + if toSend: + helper.sendMaster("MATCH ACT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + def refresh(self): instance = pool[self.name] if not instance["nickname"] == self.nickname: @@ -87,15 +102,6 @@ class IRCBot(IRCClient): result = None return result - def sanitize(self, user): - user = user.replace("!", "") - user = user.replace("~", "") - user = user.replace("&", "") - user = user.replace("@", "") - user = user.replace("%", "") - user = user.replace("+", "") - return user - def setNick(self, nickname): self._attemptedNick = nickname self.sendLine("NICK %s" % nickname) @@ -282,6 +288,24 @@ class Helper(object): connections[i].send(data) return + def sendMaster(self, data): + if config["Master"][0] in IRCPool.keys(): + IRCPool[config["Master"][0]].msg(config["Master"][1], data) + + def isKeyword(self, msg): + message = msg.lower() + uniqueNum = 0 + totalNum = 0 + for i in config["Keywords"]: + if i in message: + totalNum += message.count(i) + message = message.replace(i, "{"+i+"}") + uniqueNum += 1 + if message == msg: + return False + else: + return [message, uniqueNum, totalNum] + def addBot(self, name): global IRCPool instance = pool[name] From 2ef3782c6de2d9b897ec994c2836e8b25a0e389d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:02:13 +0000 Subject: [PATCH 021/394] Don't send keyword notifications if the message source is the reporting channel --- threshold | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/threshold b/threshold index c1795aa..e2daafe 100755 --- a/threshold +++ b/threshold @@ -67,21 +67,6 @@ class IRCBot(IRCClient): else: self.password = instance["password"] - def privmsg(self, user, channel, msg): - toSend = helper.isKeyword(msg) - if toSend: - helper.sendMaster("MATCH PRV %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) - - def noticed(self, user, channel, msg): - toSend = helper.isKeyword(msg) - if toSend: - helper.sendMaster("MATCH NOT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) - - def action(self, user, channel, msg): - toSend = helper.isKeyword(msg) - if toSend: - helper.sendMaster("MATCH ACT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) - def refresh(self): instance = pool[self.name] if not instance["nickname"] == self.nickname: @@ -95,6 +80,30 @@ class IRCBot(IRCClient): self.versionEnv = None self.sourceURL = instance["source"] + def privmsg(self, user, channel, msg): + toSend = helper.isKeyword(msg) + if toSend: + if self.name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + helper.sendMaster("MATCH PRV %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + + def noticed(self, user, channel, msg): + toSend = helper.isKeyword(msg) + if toSend: + if self.name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + helper.sendMaster("MATCH NOT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + + def action(self, user, channel, msg): + toSend = helper.isKeyword(msg) + if toSend: + if self.name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + helper.sendMaster("MATCH ACT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + def get(self, var): try: result = getattr(self, var) From d1fbbe0e2a74f71693f25d06a57551063447b87f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:20:23 +0000 Subject: [PATCH 022/394] Implement defaults --- example.json | 5 +++++ threshold | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/example.json b/example.json index 685e4fd..e7fe2ee 100644 --- a/example.json +++ b/example.json @@ -7,6 +7,11 @@ "UsePassword": true, "ConnectOnCreate": false, "Password": "s", + "Default": { + "password": null, + "username": null, + "authtype": null, + }, "Keywords": [], "Master": [] } diff --git a/threshold b/threshold index e2daafe..e26c8cd 100755 --- a/threshold +++ b/threshold @@ -583,14 +583,14 @@ class Helper(object): "bind": None, "timeout": 30, "nickname": spl[5], - "username": None, + "username": config["Default"]["username"], "realname": None, "userinfo": None, "finger": None, "version": None, "source": None, - "authtype": None, - "password": None, + "authtype": config["Default"]["authtype"], + "password": config["Default"]["password"], "authentity": "NickServ", "key": config["ListenerKey"], "certificate": config["ListenerCertificate"], From 1a17d04947a5f68ca1467e7dec1fb44a1c24ab8a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:23:18 +0000 Subject: [PATCH 023/394] Put comma in the right place --- example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example.json b/example.json index e7fe2ee..83675a9 100644 --- a/example.json +++ b/example.json @@ -10,7 +10,7 @@ "Default": { "password": null, "username": null, - "authtype": null, + "authtype": null }, "Keywords": [], "Master": [] From 1145541155f4fdfca8d86ed14a4df51cc43b98ee Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:26:13 +0000 Subject: [PATCH 024/394] Fix format string on unsetting variables --- threshold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threshold b/threshold index e26c8cd..f38e9e7 100755 --- a/threshold +++ b/threshold @@ -683,7 +683,7 @@ class Helper(object): IRCPool[spl[1]].refresh() self.savePool() if toUnset: - success("Successfully unset key %s on %s" % (spl[2], spl[3], spl[1])) + success("Successfully unset key %s on %s" % (spl[2], spl[1])) else: success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return From ab7ac6bade7f5afb5846aeea995268045afda7db Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:29:48 +0000 Subject: [PATCH 025/394] Fix bug with unsetting keys --- threshold | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/threshold b/threshold index f38e9e7..244423c 100755 --- a/threshold +++ b/threshold @@ -671,9 +671,10 @@ class Helper(object): toUnset = True if spl[2] == "authtype": - if not spl[3] in ["sp", "ns"]: - failure("Authtype must be sp or ns, not %s" % spl[3]) - return + if not toUnset: + if not spl[3] in ["sp", "ns"]: + failure("Authtype must be sp or ns, not %s" % spl[3]) + return if spl[2] == "enabled": failure("Use the enable and disable commands to manage this") return From a673a2288e81d3a10acabd57cbfe1272627d5990 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:44:03 +0000 Subject: [PATCH 026/394] Don't create duplicate bot instances --- threshold | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/threshold b/threshold index 244423c..8ec836a 100755 --- a/threshold +++ b/threshold @@ -422,7 +422,8 @@ class Helper(object): return pool[spl[1]]["enabled"] = True helper.savePool() - self.addBot(spl[1]) + if not spl[1] in IRCPool.keys(): + self.addBot(spl[1]) success("Successfully enabled bot %s" % spl[1]) return else: From ac798016bad889bc1556c5f62a60193ff35d44f8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:48:20 +0000 Subject: [PATCH 027/394] Implement autojoin --- threshold | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/threshold b/threshold index 8ec836a..aee8e65 100755 --- a/threshold +++ b/threshold @@ -55,6 +55,7 @@ class IRCBot(IRCClient): self.versionNum = None self.versionEnv = None self.sourceURL = instance["source"] + self.autojoin = instance["autojoin"] self._who = {} self._getWho = {} @@ -166,6 +167,8 @@ class IRCBot(IRCClient): self.connected = True if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) + for i in self.autojoin: + self.join(i) def joined(self, channel): if not channel in self.channels: @@ -590,6 +593,7 @@ class Helper(object): "finger": None, "version": None, "source": None, + "autojoin": [], "authtype": config["Default"]["authtype"], "password": config["Default"]["password"], "authentity": "NickServ", @@ -679,6 +683,8 @@ class Helper(object): if spl[2] == "enabled": failure("Use the enable and disable commands to manage this") return + if spl[2] == "autojoin": + spl[3] = spl[3].split(",") pool[spl[1]][spl[2]] = spl[3] if spl[1] in IRCPool.keys(): From ce8da87ea756160dcacf9489537e220f2710610c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 19:50:57 +0000 Subject: [PATCH 028/394] Use a better metric for determining keyword matches --- threshold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threshold b/threshold index aee8e65..5a0234e 100755 --- a/threshold +++ b/threshold @@ -313,7 +313,7 @@ class Helper(object): totalNum += message.count(i) message = message.replace(i, "{"+i+"}") uniqueNum += 1 - if message == msg: + if totalNum == 0: return False else: return [message, uniqueNum, totalNum] From 9950a527c17037e281c1573e3ae8e6449a3e4b9e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 20:26:56 +0000 Subject: [PATCH 029/394] Remove the bot instance when disabling --- threshold | 1 + 1 file changed, 1 insertion(+) diff --git a/threshold b/threshold index 5a0234e..882e78c 100755 --- a/threshold +++ b/threshold @@ -443,6 +443,7 @@ class Helper(object): if spl[1] in IRCPool.keys(): if IRCPool[spl[1]].connected == True: IRCPool[spl[1]].transport.loseConnection() + del IRCPool[spl[1]] success("Successfully disabled bot %s" % spl[1]) return else: From 4d4b4b77242c9fb5a34b3199946008fd37de5143 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 25 Nov 2017 20:28:49 +0000 Subject: [PATCH 030/394] Refresh and reconnect when enabling an existing instance --- threshold | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/threshold b/threshold index 882e78c..b561997 100755 --- a/threshold +++ b/threshold @@ -197,6 +197,9 @@ class IRCBot(IRCClient): log("%s: connection failed: %s" % (self.name, error)) helper.sendAll("%s: connection failed: %s" % (self.name, error)) + def reconnect(self): + connector.connect() + class Base(Protocol): def __init__(self, addr): self.addr = addr @@ -427,6 +430,9 @@ class Helper(object): helper.savePool() if not spl[1] in IRCPool.keys(): self.addBot(spl[1]) + else: + IRCPool[spl[1]].refresh() + IRCPool[spl[1]].reconnect() success("Successfully enabled bot %s" % spl[1]) return else: From d9e142758ca22ab067d751eb7b85c5c3434d7022 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 26 Nov 2017 11:14:15 +0000 Subject: [PATCH 031/394] No trailing space when listing keywords --- threshold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threshold b/threshold index b561997..f1ba60e 100755 --- a/threshold +++ b/threshold @@ -543,7 +543,7 @@ class Helper(object): return elif spl[1] == "show": - info(", ".join(config["Keywords"])) + info(",".join(config["Keywords"])) return elif spl[1] == "master": From 2e552f8d2e6f9a02eec75b102b40371139e248bf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 27 Nov 2017 19:54:35 +0000 Subject: [PATCH 032/394] Implement keyword exceptions --- example.json | 1 + help.json | 2 +- threshold | 82 ++++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/example.json b/example.json index 83675a9..f7e7250 100644 --- a/example.json +++ b/example.json @@ -13,5 +13,6 @@ "authtype": null }, "Keywords": [], + "KeywordsExcept": {}, "Master": [] } diff --git a/help.json b/help.json index 612f2fe..64e2207 100644 --- a/help.json +++ b/help.json @@ -5,7 +5,7 @@ "del": "del ", "mod": "mod [] []", "get": "get ", - "key": "key {[] []} []", + "key": "key [] [] []", "join": "join []", "enable": "enable ", diff --git a/threshold b/threshold index f1ba60e..1a206a7 100755 --- a/threshold +++ b/threshold @@ -214,7 +214,7 @@ class Base(Protocol): def dataReceived(self, data): data = data.decode("utf-8", "replace") - log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) + #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) helper.parseCommand(self.addr, self.authed, data) def connectionMade(self): @@ -309,13 +309,22 @@ class Helper(object): def isKeyword(self, msg): message = msg.lower() + toUndo = False uniqueNum = 0 totalNum = 0 for i in config["Keywords"]: if i in message: + if i in config["KeywordsExcept"].keys(): + for x in config["KeywordsExcept"][i]: + if x in message: + toUndo = True + + if toUndo == False: totalNum += message.count(i) message = message.replace(i, "{"+i+"}") uniqueNum += 1 + toUndo = False + if totalNum == 0: return False else: @@ -512,7 +521,7 @@ class Helper(object): elif cmd == "key": if data.startswith("key add"): - if not data == "key add " and not data == "key add": + if not data in ["key add ", "key add"] and data[3] == " ": keywordsToAdd = data[8:] keywords = keywordsToAdd.split(",") for keyword in keywords: @@ -528,7 +537,7 @@ class Helper(object): incUsage("key") return elif data.startswith("key del"): - if not data == "key del " and not data == "key del": + if not data in ["key del ", "key del"] and data[3] == " ": keywordsToDel = data[8:] keywords = keywordsToDel.split(",") for keyword in keywords: @@ -541,13 +550,67 @@ class Helper(object): else: incUsage("key") return + if length == 4: + if spl[1] == "except": + if not spl[2] in config["Keywords"]: + failure("No such keyword: %s" % spl[2]) + return + if spl[2] in config["KeywordsExcept"].keys(): + if spl[3] in config["KeywordsExcept"][spl[2]]: + failure("Exception exists: %s" % spl[3]) + return + else: + if not spl[2] in spl[3]: + failure("Keyword %s not in exception %s. This won't work" % (spl[2], spl[3])) + return + config["KeywordsExcept"][spl[2]] = [] - elif spl[1] == "show": - info(",".join(config["Keywords"])) - return + config["KeywordsExcept"][spl[2]].append(spl[3]) + helper.saveConfig() + success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) + return - elif spl[1] == "master": - if length == 4: + elif spl[1] == "unexcept": + if not spl[2] in config["KeywordsExcept"].keys(): + failure("No such exception: %s" % spl[2]) + return + if not spl[3] in config["KeywordsExcept"][spl[2]]: + failure("Exception %s has no attribute %s" % (spl[2], spl[3])) + return + config["KeywordsExcept"][spl[2]].remove(spl[3]) + if config["KeywordsExcept"][spl[2]] == []: + del config["KeywordsExcept"][spl[2]] + helper.saveConfig() + success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) + return + else: + incUsage("key") + return + if length == 3: + if spl[1] == "unexcept": + if not spl[2] in config["KeywordsExcept"].keys(): + failure("No such exception: %s" % spl[2]) + return + del config["KeywordsExcept"][spl[2]] + helper.saveConfig() + success("Successfully removed exception list of %s" % spl[2]) + return + if length == 2: + if spl[1] == "show": + info(",".join(config["Keywords"])) + return + + elif spl[1] == "showexcept": + exceptMap = [] + for i in config["KeywordsExcept"].keys(): + exceptMap.append("Key: %s" % i) + exceptMap.append("%s: %s" % (i, ",".join(config["KeywordsExcept"][i]))) + exceptMap.append("\n") + info("\n".join(exceptMap)) + return + + if length == 4: + if spl[1] == "master": if not spl[2] in pool.keys(): failure("Name does not exist: %s" % spl[2]) return @@ -558,7 +621,8 @@ class Helper(object): helper.saveConfig() success("Master set to %s on %s" % (spl[3], spl[2])) return - elif length == 2: + elif length == 2: + if spl[1] == "master": info(" - ".join(config["Master"])) return else: From 656931426721c6f9d9d3f3d97a10a36f3ff8ec49 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 27 Nov 2017 20:18:39 +0000 Subject: [PATCH 033/394] Don't try to reconnect when running the enable command on an existing bot instance --- threshold | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/threshold b/threshold index 1a206a7..d9c1e81 100755 --- a/threshold +++ b/threshold @@ -197,9 +197,6 @@ class IRCBot(IRCClient): log("%s: connection failed: %s" % (self.name, error)) helper.sendAll("%s: connection failed: %s" % (self.name, error)) - def reconnect(self): - connector.connect() - class Base(Protocol): def __init__(self, addr): self.addr = addr @@ -440,8 +437,7 @@ class Helper(object): if not spl[1] in IRCPool.keys(): self.addBot(spl[1]) else: - IRCPool[spl[1]].refresh() - IRCPool[spl[1]].reconnect() + pass success("Successfully enabled bot %s" % spl[1]) return else: From 1b0217e0a67133b70d5fb2fa8455086401f0627f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 28 Nov 2017 19:19:27 +0000 Subject: [PATCH 034/394] Only match eligible exclude statements and make sure legitimate keywords get through --- threshold | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/threshold b/threshold index d9c1e81..301849b 100755 --- a/threshold +++ b/threshold @@ -306,6 +306,7 @@ class Helper(object): def isKeyword(self, msg): message = msg.lower() + messageDuplicate = message toUndo = False uniqueNum = 0 totalNum = 0 @@ -315,11 +316,19 @@ class Helper(object): for x in config["KeywordsExcept"][i]: if x in message: toUndo = True + messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") + for y in config["Keywords"]: + if i in messageDuplicate: + totalNum += messageDuplicate.count(i) + message = messageDuplicate.replace(i, "{"+i+"}") + message = message.replace("\0\r\n\n\0", x) + uniqueNum += 1 + + if toUndo == False: + totalNum += message.count(i) + message = message.replace(i, "{"+i+"}") + uniqueNum += 1 - if toUndo == False: - totalNum += message.count(i) - message = message.replace(i, "{"+i+"}") - uniqueNum += 1 toUndo = False if totalNum == 0: @@ -789,6 +798,7 @@ if __name__ == "__main__": config = helper.getConfig() pool = helper.getPool() help = helper.getHelp() + for i in pool.keys(): if pool[i]["enabled"] == True: helper.addBot(i) From e6367af7f259efbe9c92ad880e8a7f1c47de3773 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 28 Nov 2017 19:31:02 +0000 Subject: [PATCH 035/394] Send kick notifications --- threshold | 1 + 1 file changed, 1 insertion(+) diff --git a/threshold b/threshold index 301849b..ed4dca7 100755 --- a/threshold +++ b/threshold @@ -182,6 +182,7 @@ class IRCBot(IRCClient): def kickedFrom(self, channel, kicker, message): if channel in self.channels: self.channels.remove(channel) + helper.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) def connectionLost(self, reason): self.connected = False From cc75535bcf3022ad83ea23fac9930620b1daaa5c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 30 Nov 2017 18:54:08 +0000 Subject: [PATCH 036/394] Implement notifications when the bot's nickname is said --- example.json | 1 + threshold | 39 +++++++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/example.json b/example.json index f7e7250..aa7543c 100644 --- a/example.json +++ b/example.json @@ -6,6 +6,7 @@ "ListenerCertificate": "cert.pem", "UsePassword": true, "ConnectOnCreate": false, + "HighlightNotifications": true, "Password": "s", "Default": { "password": null, diff --git a/threshold b/threshold index ed4dca7..c32a25c 100755 --- a/threshold +++ b/threshold @@ -83,26 +83,41 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): toSend = helper.isKeyword(msg) - if toSend: - if self.name == config["Master"][0] and channel == config["Master"][1]: - pass - else: + if self.name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + if config["HighlightNotifications"]: + msgLower = msg.lower() + if self.nickname.lower() in msgLower: + msgLower = msgLower.replace(self.nickname, "{"+self.nickname+"}") + helper.sendMaster("NICK PRV %s (T:%s): (%s/%s) %s" % (self.name, msg.count(self.nickname), user, channel, msgLower)) + if toSend: helper.sendMaster("MATCH PRV %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) def noticed(self, user, channel, msg): toSend = helper.isKeyword(msg) - if toSend: - if self.name == config["Master"][0] and channel == config["Master"][1]: - pass - else: + if self.name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + if config["HighlightNotifications"]: + msgLower = msg.lower() + if self.nickname.lower() in msgLower: + msgLower = msgLower.replace(self.nickname, "{"+self.nickname+"}") + helper.sendMaster("NICK NOT %s (T:%s): (%s/%s) %s" % (self.name, msg.count(self.nickname), user, channel, msgLower)) + if toSend: helper.sendMaster("MATCH NOT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) def action(self, user, channel, msg): toSend = helper.isKeyword(msg) - if toSend: - if self.name == config["Master"][0] and channel == config["Master"][1]: - pass - else: + if self.name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + if config["HighlightNotifications"]: + msgLower = msg.lower() + if self.nickname.lower() in msgLower: + msgLower = msgLower.replace(self.nickname, "{"+self.nickname+"}") + helper.sendMaster("NICK ACT %s (T:%s): (%s/%s) %s" % (self.name, msg.count(self.nickname), user, channel, msgLower)) + if toSend: helper.sendMaster("MATCH ACT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) def get(self, var): From 9506d3d1f40d65fc38695f1954dcb4855126271c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 2 Dec 2017 19:49:00 +0000 Subject: [PATCH 037/394] Fix counting of nickname highlights --- threshold | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/threshold b/threshold index c32a25c..99a4659 100755 --- a/threshold +++ b/threshold @@ -88,9 +88,10 @@ class IRCBot(IRCClient): else: if config["HighlightNotifications"]: msgLower = msg.lower() - if self.nickname.lower() in msgLower: - msgLower = msgLower.replace(self.nickname, "{"+self.nickname+"}") - helper.sendMaster("NICK PRV %s (T:%s): (%s/%s) %s" % (self.name, msg.count(self.nickname), user, channel, msgLower)) + nickLower = self.nickname.lower() + if nickLower in msgLower: + msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + helper.sendMaster("NICK PRV %s (T:%s): (%s/%s) %s" % (self.name, msgLower.count(nickLower), user, channel, msgLower)) if toSend: helper.sendMaster("MATCH PRV %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) @@ -101,9 +102,10 @@ class IRCBot(IRCClient): else: if config["HighlightNotifications"]: msgLower = msg.lower() - if self.nickname.lower() in msgLower: - msgLower = msgLower.replace(self.nickname, "{"+self.nickname+"}") - helper.sendMaster("NICK NOT %s (T:%s): (%s/%s) %s" % (self.name, msg.count(self.nickname), user, channel, msgLower)) + nickLower = self.nickname.lower() + if nickLower in msgLower: + msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + helper.sendMaster("NICK NOT %s (T:%s): (%s/%s) %s" % (self.name, msgLower.count(nickLower), user, channel, msgLower)) if toSend: helper.sendMaster("MATCH NOT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) @@ -114,9 +116,10 @@ class IRCBot(IRCClient): else: if config["HighlightNotifications"]: msgLower = msg.lower() - if self.nickname.lower() in msgLower: - msgLower = msgLower.replace(self.nickname, "{"+self.nickname+"}") - helper.sendMaster("NICK ACT %s (T:%s): (%s/%s) %s" % (self.name, msg.count(self.nickname), user, channel, msgLower)) + nickLower = self.nickname.lower() + if nickLower in msgLower: + msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + helper.sendMaster("NICK ACT %s (T:%s): (%s/%s) %s" % (self.name, msgLower.count(nickLower), user, channel, msgLower)) if toSend: helper.sendMaster("MATCH ACT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) From 77928dc52c0485568e13cb767cf65765b3395711 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 3 Dec 2017 13:10:51 +0000 Subject: [PATCH 038/394] Move keywords into a seperate file --- keyword.json | 6 ++++ threshold | 94 +++++++++++++++++++++++++++++++++++----------------- 2 files changed, 69 insertions(+), 31 deletions(-) create mode 100644 keyword.json diff --git a/keyword.json b/keyword.json new file mode 100644 index 0000000..cc4da5b --- /dev/null +++ b/keyword.json @@ -0,0 +1,6 @@ +{ + "Keywords": [ + "kek" + ], + "KeywordsExcept": {} +} \ No newline at end of file diff --git a/threshold b/threshold index 99a4659..e53a25a 100755 --- a/threshold +++ b/threshold @@ -231,6 +231,15 @@ class Base(Protocol): def dataReceived(self, data): data = data.decode("utf-8", "replace") #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) + if "\n" in data: + splitData = [x for x in data.split("\n") if x] + if "\n" in data: + #timePlus = 0.0 + for i in splitData: + helper.parseCommand(self.addr, self.authed, i) + #reactor.callLater(timePlus, lambda: helper.parseCommand(self.addr, self.authed, i)) + #timePlus += 0.01 + return helper.parseCommand(self.addr, self.authed, data) def connectionMade(self): @@ -279,12 +288,24 @@ class Helper(object): else: error("Mandatory values missing from config") + def saveConfig(self): global config with open("config.json", "w") as f: dump(config, f, indent=4) return + def getKeywordConfig(self): + with open("keyword.json", "r") as f: + keyconf = load(f) + return keyconf + + def saveKeywordConfig(self): + global keyconf + with open("keyword.json", "w") as f: + dump(keyconf, f, indent=4) + return + def getPool(self): with open("pool.json", "r") as f: data = f.read() @@ -329,14 +350,14 @@ class Helper(object): toUndo = False uniqueNum = 0 totalNum = 0 - for i in config["Keywords"]: + for i in keyconf["Keywords"]: if i in message: - if i in config["KeywordsExcept"].keys(): - for x in config["KeywordsExcept"][i]: + if i in keyconf["KeywordsExcept"].keys(): + for x in keyconf["KeywordsExcept"][i]: if x in message: toUndo = True messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") - for y in config["Keywords"]: + for y in keyconf["Keywords"]: if i in messageDuplicate: totalNum += messageDuplicate.count(i) message = messageDuplicate.replace(i, "{"+i+"}") @@ -388,28 +409,31 @@ class Helper(object): return def addKeyword(self, keyword): - if keyword in config["Keywords"]: + if keyword in keyconf["Keywords"]: return "EXISTS" else: - for i in config["Keywords"]: + for i in keyconf["Keywords"]: if i in keyword or keyword in i: return "ISIN" - config["Keywords"].append(keyword) - helper.saveConfig() + keyconf["Keywords"].append(keyword) + helper.saveKeywordConfig() return True def delKeyword(self, keyword): - if not keyword in config["Keywords"]: + if not keyword in keyconf["Keywords"]: return "NOKEY" - config["Keywords"].remove(keyword) - helper.saveConfig() + keyconf["Keywords"].remove(keyword) + helper.saveKeywordConfig() return True def parseCommand(self, addr, authed, data): global pool - data = data.strip("\n") spl = data.split() - obj = connections[addr] + if addr in connections.keys(): + obj = connections[addr] + else: + warning("Got connection object with no instance in the address pool") + return success = lambda data: sendSuccess(addr, data) failure = lambda data: sendFailure(addr, data) @@ -434,8 +458,15 @@ class Helper(object): elif cmd == "rehash": global config config = helper.getConfig() + log("Configuration rehashed") success("Configuration rehashed successfully") + elif cmd == "rekey": + global keyconf + keyconf = helper.getKeywordConfig() + log("Keyword configuration rehashed") + success("Keyword configuration rehashed successfully") + elif cmd == "pass": info("You are already authenticated") return @@ -576,35 +607,35 @@ class Helper(object): return if length == 4: if spl[1] == "except": - if not spl[2] in config["Keywords"]: + if not spl[2] in keyconf["Keywords"]: failure("No such keyword: %s" % spl[2]) return - if spl[2] in config["KeywordsExcept"].keys(): - if spl[3] in config["KeywordsExcept"][spl[2]]: + if spl[2] in keyconf["KeywordsExcept"].keys(): + if spl[3] in keyconf["KeywordsExcept"][spl[2]]: failure("Exception exists: %s" % spl[3]) return else: if not spl[2] in spl[3]: failure("Keyword %s not in exception %s. This won't work" % (spl[2], spl[3])) return - config["KeywordsExcept"][spl[2]] = [] + keyconf["KeywordsExcept"][spl[2]] = [] - config["KeywordsExcept"][spl[2]].append(spl[3]) - helper.saveConfig() + keyconf["KeywordsExcept"][spl[2]].append(spl[3]) + helper.saveKeywordConfig() success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) return elif spl[1] == "unexcept": - if not spl[2] in config["KeywordsExcept"].keys(): + if not spl[2] in keyconf["KeywordsExcept"].keys(): failure("No such exception: %s" % spl[2]) return - if not spl[3] in config["KeywordsExcept"][spl[2]]: + if not spl[3] in keyconf["KeywordsExcept"][spl[2]]: failure("Exception %s has no attribute %s" % (spl[2], spl[3])) return - config["KeywordsExcept"][spl[2]].remove(spl[3]) - if config["KeywordsExcept"][spl[2]] == []: - del config["KeywordsExcept"][spl[2]] - helper.saveConfig() + keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) + if keyconf["KeywordsExcept"][spl[2]] == []: + del keyconf["KeywordsExcept"][spl[2]] + helper.saveKeywordConfig() success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) return else: @@ -612,23 +643,23 @@ class Helper(object): return if length == 3: if spl[1] == "unexcept": - if not spl[2] in config["KeywordsExcept"].keys(): + if not spl[2] in keyconf["KeywordsExcept"].keys(): failure("No such exception: %s" % spl[2]) return - del config["KeywordsExcept"][spl[2]] - helper.saveConfig() + del keyconf["KeywordsExcept"][spl[2]] + helper.saveKeywordConfig() success("Successfully removed exception list of %s" % spl[2]) return if length == 2: if spl[1] == "show": - info(",".join(config["Keywords"])) + info(",".join(keyconf["Keywords"])) return elif spl[1] == "showexcept": exceptMap = [] - for i in config["KeywordsExcept"].keys(): + for i in keyconf["KeywordsExcept"].keys(): exceptMap.append("Key: %s" % i) - exceptMap.append("%s: %s" % (i, ",".join(config["KeywordsExcept"][i]))) + exceptMap.append("%s: %s" % (i, ",".join(keyconf["KeywordsExcept"][i]))) exceptMap.append("\n") info("\n".join(exceptMap)) return @@ -815,6 +846,7 @@ class Helper(object): if __name__ == "__main__": helper = Helper() config = helper.getConfig() + keyconf = helper.getKeywordConfig() pool = helper.getPool() help = helper.getHelp() From 63396d54e3893f42c8df5c6fda068be3ab02ef9d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 3 Dec 2017 18:34:56 +0000 Subject: [PATCH 039/394] Implement a dist command --- example.json | 3 +-- help.json | 4 +++- threshold | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/example.json b/example.json index aa7543c..f7ba8a9 100644 --- a/example.json +++ b/example.json @@ -7,13 +7,12 @@ "UsePassword": true, "ConnectOnCreate": false, "HighlightNotifications": true, + "DistEnabled": true, "Password": "s", "Default": { "password": null, "username": null, "authtype": null }, - "Keywords": [], - "KeywordsExcept": {}, "Master": [] } diff --git a/help.json b/help.json index 64e2207..1e77914 100644 --- a/help.json +++ b/help.json @@ -10,5 +10,7 @@ "enable": "enable ", "list": "list", - "rehash": "rehash" + "rehash": "rehash", + "rekey": "rekey", + "dist": "dist" } diff --git a/threshold b/threshold index e53a25a..4764701 100755 --- a/threshold +++ b/threshold @@ -8,6 +8,7 @@ from twisted.words.protocols.irc import IRCClient from json import load, dump, loads from sys import exit +from subprocess import run, PIPE listener = None connections = {} @@ -467,6 +468,14 @@ class Helper(object): log("Keyword configuration rehashed") success("Keyword configuration rehashed successfully") + elif cmd == "dist": + if config["DistEnabled"]: + rtrn = run(["./dist.sh"], shell=True, stdout=PIPE) + info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) + else: + failure("The dist command is not enabled") + return + elif cmd == "pass": info("You are already authenticated") return From e2682615aec11ccbaef9b484870894ae3e73335f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 3 Dec 2017 19:11:06 +0000 Subject: [PATCH 040/394] Reshuffle key section --- threshold | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/threshold b/threshold index 4764701..000b039 100755 --- a/threshold +++ b/threshold @@ -633,7 +633,17 @@ class Helper(object): helper.saveKeywordConfig() success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) return - + elif spl[1] == "master": + if not spl[2] in pool.keys(): + failure("Name does not exist: %s" % spl[2]) + return + if spl[2] in IRCPool.keys(): + if not spl[3] in IRCPool[spl[2]].channels: + info("Bot not on channel: %s" % spl[3]) + config["Master"] = [spl[2], spl[3]] + helper.saveConfig() + success("Master set to %s on %s" % (spl[3], spl[2])) + return elif spl[1] == "unexcept": if not spl[2] in keyconf["KeywordsExcept"].keys(): failure("No such exception: %s" % spl[2]) @@ -650,7 +660,7 @@ class Helper(object): else: incUsage("key") return - if length == 3: + elif length == 3: if spl[1] == "unexcept": if not spl[2] in keyconf["KeywordsExcept"].keys(): failure("No such exception: %s" % spl[2]) @@ -659,7 +669,9 @@ class Helper(object): helper.saveKeywordConfig() success("Successfully removed exception list of %s" % spl[2]) return - if length == 2: + else: + incUsage("key") + elif length == 2: if spl[1] == "show": info(",".join(keyconf["Keywords"])) return @@ -672,31 +684,13 @@ class Helper(object): exceptMap.append("\n") info("\n".join(exceptMap)) return - - if length == 4: - if spl[1] == "master": - if not spl[2] in pool.keys(): - failure("Name does not exist: %s" % spl[2]) - return - if spl[2] in IRCPool.keys(): - if not spl[3] in IRCPool[spl[2]].channels: - info("Bot not on channel: %s" % spl[3]) - config["Master"] = [spl[2], spl[3]] - helper.saveConfig() - success("Master set to %s on %s" % (spl[3], spl[2])) - return - elif length == 2: - if spl[1] == "master": + elif spl[1] == "master": info(" - ".join(config["Master"])) return else: incUsage("key") return - else: - incUsage("key") - return - elif cmd == "add": if length == 6: From 96dd71bbdd4515ce33bd39c213ad9d96d453c76b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 3 Dec 2017 19:23:29 +0000 Subject: [PATCH 041/394] Make sending the output of the dist command configurable --- .gitignore | 1 + example.json | 1 + threshold | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1c8959f..7361a72 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ *.pem config.json pool.json +dist.sh env/ diff --git a/example.json b/example.json index f7ba8a9..fcc89a3 100644 --- a/example.json +++ b/example.json @@ -8,6 +8,7 @@ "ConnectOnCreate": false, "HighlightNotifications": true, "DistEnabled": true, + "SendDistOutput": false, "Password": "s", "Default": { "password": null, diff --git a/threshold b/threshold index 000b039..ad9faa8 100755 --- a/threshold +++ b/threshold @@ -471,7 +471,10 @@ class Helper(object): elif cmd == "dist": if config["DistEnabled"]: rtrn = run(["./dist.sh"], shell=True, stdout=PIPE) - info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) + if config["SendDistOutput"]: + info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) + else: + info("Exit code: %s" % rtrn.returncode) else: failure("The dist command is not enabled") return From fc8a115e179223de69abd6e068f3823d11ccd9c8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 24 Dec 2017 21:04:48 +0000 Subject: [PATCH 042/394] Implement alternative keyword notifications and move the per-instance wholist into the global namespace --- example.json | 1 + help.json | 2 +- threshold | 70 ++++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/example.json b/example.json index fcc89a3..d22a183 100644 --- a/example.json +++ b/example.json @@ -7,6 +7,7 @@ "UsePassword": true, "ConnectOnCreate": false, "HighlightNotifications": true, + "Debugger": false, "DistEnabled": true, "SendDistOutput": false, "Password": "s", diff --git a/help.json b/help.json index 1e77914..d87f86f 100644 --- a/help.json +++ b/help.json @@ -5,7 +5,7 @@ "del": "del ", "mod": "mod [] []", "get": "get ", - "key": "key [] [] []", + "key": "key [] [] [] []", "join": "join []", "enable": "enable ", diff --git a/threshold b/threshold index ad9faa8..7371efb 100755 --- a/threshold +++ b/threshold @@ -6,14 +6,24 @@ from twisted.internet.protocol import Protocol, Factory from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint, connectProtocol from twisted.words.protocols.irc import IRCClient +#from twisted.python import log +#from sys import stdout +#log.startLogging(stdout) + from json import load, dump, loads from sys import exit from subprocess import run, PIPE +numbers = "0123456789" + listener = None connections = {} IRCPool = {} +MonitorPool = [] + +wholist = {} + def log(data): print("[LOG]", data) @@ -60,7 +70,6 @@ class IRCBot(IRCClient): self._who = {} self._getWho = {} - self.wholist = {} self.authtype = instance["authtype"] if self.authtype == "ns": @@ -180,7 +189,8 @@ class IRCBot(IRCClient): del self._who[channel] def got_who(self, whoinfo): - self.wholist[whoinfo[0]] = whoinfo[1] + global wholist + helper.setWho(self.name, whoinfo[1]) def signedOn(self): self.connected = True @@ -235,11 +245,8 @@ class Base(Protocol): if "\n" in data: splitData = [x for x in data.split("\n") if x] if "\n" in data: - #timePlus = 0.0 for i in splitData: helper.parseCommand(self.addr, self.authed, i) - #reactor.callLater(timePlus, lambda: helper.parseCommand(self.addr, self.authed, i)) - #timePlus += 0.01 return helper.parseCommand(self.addr, self.authed, data) @@ -248,7 +255,7 @@ class Base(Protocol): self.send("Hello.") def connectionLost(self, reason): - global connections + global connections, MonitorPool self.authed = False log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) if not listener == None: @@ -258,6 +265,8 @@ class Base(Protocol): warn("Tried to remove a non-existant connection.") else: warn("Tried to remove a connection from a listener that wasn't running.") + if self.addr in MonitorPool: + MonitorPool.remove(self.addr) class BaseFactory(Factory): def buildProtocol(self, addr): @@ -289,6 +298,23 @@ class Helper(object): else: error("Mandatory values missing from config") + def setWho(self, network, newObjects): + global wholist + network = "".join([x for x in network if not x in numbers]) + if not network in wholist.keys(): + wholist[network] = {} + for i in newObjects.keys(): + wholist[network][i] = newObjects[i] + return + + def setWhoSingle(self, network, nick, ident, host): + global wholist + network = "".join([x for x in network if not x in numbers]) + + if network in wholist.keys(): + if nick in wholist[network].keys(): + wholist[network][nick][1] = ident + wholist[network][nick][2] = host def saveConfig(self): global config @@ -344,6 +370,8 @@ class Helper(object): def sendMaster(self, data): if config["Master"][0] in IRCPool.keys(): IRCPool[config["Master"][0]].msg(config["Master"][1], data) + for i in MonitorPool: + connections[i].send(data) def isKeyword(self, msg): message = msg.lower() @@ -428,7 +456,7 @@ class Helper(object): return True def parseCommand(self, addr, authed, data): - global pool + global pool, MonitorPool spl = data.split() if addr in connections.keys(): obj = connections[addr] @@ -672,8 +700,29 @@ class Helper(object): helper.saveKeywordConfig() success("Successfully removed exception list of %s" % spl[2]) return + elif spl[1] == "monitor": + if spl[2] == "on": + if not obj.addr in MonitorPool: + MonitorPool.append(obj.addr) + success("Keyword monitoring enabled") + return + else: + failure("Keyword monitoring is already enabled") + return + elif spl[2] == "off": + if obj.addr in MonitorPool: + MonitorPool.remove(obj.addr) + success("Keyword monitoring disabled") + return + else: + failure("Keyword monitoring is already disabled") + return + else: + incUsage("key") + return else: incUsage("key") + return elif length == 2: if spl[1] == "show": info(",".join(keyconf["Keywords"])) @@ -690,6 +739,13 @@ class Helper(object): elif spl[1] == "master": info(" - ".join(config["Master"])) return + elif spl[1] == "monitor": + if obj.addr in MonitorPool: + info("Keyword monitoring is enabled on this connection") + return + else: + info("Keyword monitoring is disabled on this connection") + return else: incUsage("key") return From 7e1e0ee0d24c7ec0b2be1d26d4d9ed89e74e2c0e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 24 Dec 2017 21:09:18 +0000 Subject: [PATCH 043/394] Some additional error checking for when no master is defined --- example.json | 2 +- threshold | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/example.json b/example.json index d22a183..5faaaaf 100644 --- a/example.json +++ b/example.json @@ -16,5 +16,5 @@ "username": null, "authtype": null }, - "Master": [] + "Master": [null, null] } diff --git a/threshold b/threshold index 7371efb..c630333 100755 --- a/threshold +++ b/threshold @@ -370,6 +370,8 @@ class Helper(object): def sendMaster(self, data): if config["Master"][0] in IRCPool.keys(): IRCPool[config["Master"][0]].msg(config["Master"][1], data) + else: + warning("Master with no IRC instance defined") for i in MonitorPool: connections[i].send(data) From 5b6bf59d7aeaf9766c491d1ba2a910cba243e013 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 25 Dec 2017 19:02:17 +0000 Subject: [PATCH 044/394] Implement searching through the wholist --- help.json | 1 + threshold | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/help.json b/help.json index d87f86f..7322907 100644 --- a/help.json +++ b/help.json @@ -6,6 +6,7 @@ "mod": "mod [] []", "get": "get ", "key": "key [] [] [] []", + "who": "who ", "join": "join []", "enable": "enable ", diff --git a/threshold b/threshold index c630333..cd5d205 100755 --- a/threshold +++ b/threshold @@ -194,6 +194,7 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True + log("signed on: %s" % self.name) if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) for i in self.autojoin: @@ -315,6 +316,18 @@ class Helper(object): if nick in wholist[network].keys(): wholist[network][nick][1] = ident wholist[network][nick][2] = host + else: + wholist[network][nick] = [nick, ident, host, None, None, None] + + def getWho(self, nick): + result = {} + for i in wholist.keys(): + for x in wholist[i].keys(): + if nick.lower() == x.lower(): + if not i in result.keys(): + result[i] = [] + result[i].append(wholist[i][x]) + return result def saveConfig(self): global config @@ -616,6 +629,23 @@ class Helper(object): incUsage("get") return + elif cmd == "who": + global wholist + if length == 2: + result = self.getWho(spl[1]) + rtrn = "" + for i in result.keys(): + rtrn += "Matches from: %s" % i + rtrn += "\n" + for x in result[i]: + rtrn += (", ".join(x)) + rtrn += "\n" + info(rtrn) + return + else: + incUsage("who") + return + elif cmd == "key": if data.startswith("key add"): if not data in ["key add ", "key add"] and data[3] == " ": @@ -752,6 +782,10 @@ class Helper(object): incUsage("key") return + else: + incUsage("key") + return + elif cmd == "add": if length == 6: From bef1875186052e3333f71ea7f86a4e3316760b83 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 25 Dec 2017 22:09:38 +0000 Subject: [PATCH 045/394] Implement statistics and shorten some functions --- help.json | 3 +- threshold | 148 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 111 insertions(+), 40 deletions(-) diff --git a/help.json b/help.json index 7322907..99ff659 100644 --- a/help.json +++ b/help.json @@ -8,9 +8,10 @@ "key": "key [] [] [] []", "who": "who ", "join": "join []", - "enable": "enable ", "disable": "disable ", "list": "list", + "stats": "stats", "rehash": "rehash", "rekey": "rekey", "dist": "dist" diff --git a/threshold b/threshold index cd5d205..574c5ec 100755 --- a/threshold +++ b/threshold @@ -91,47 +91,35 @@ class IRCBot(IRCClient): self.versionEnv = None self.sourceURL = instance["source"] - def privmsg(self, user, channel, msg): - toSend = helper.isKeyword(msg) - if self.name == config["Master"][0] and channel == config["Master"][1]: - pass + def parsen(self, user): + step = user.split("!") + nick = step[0] + if len(step) == 2: + step2 = step[1].split("@") + ident, host = step2 else: - if config["HighlightNotifications"]: - msgLower = msg.lower() - nickLower = self.nickname.lower() - if nickLower in msgLower: - msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") - helper.sendMaster("NICK PRV %s (T:%s): (%s/%s) %s" % (self.name, msgLower.count(nickLower), user, channel, msgLower)) - if toSend: - helper.sendMaster("MATCH PRV %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + ident = nick + host = nick + + return [nick, ident, host] + + def privmsg(self, user, channel, msg): + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + helper.actKeyword(user, channel, msg, self.nickname, "PRV", self.name) def noticed(self, user, channel, msg): - toSend = helper.isKeyword(msg) - if self.name == config["Master"][0] and channel == config["Master"][1]: - pass - else: - if config["HighlightNotifications"]: - msgLower = msg.lower() - nickLower = self.nickname.lower() - if nickLower in msgLower: - msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") - helper.sendMaster("NICK NOT %s (T:%s): (%s/%s) %s" % (self.name, msgLower.count(nickLower), user, channel, msgLower)) - if toSend: - helper.sendMaster("MATCH NOT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + helper.actKeyword(user, channel, msg, self.nickname, "NOT", self.name) def action(self, user, channel, msg): - toSend = helper.isKeyword(msg) - if self.name == config["Master"][0] and channel == config["Master"][1]: - pass - else: - if config["HighlightNotifications"]: - msgLower = msg.lower() - nickLower = self.nickname.lower() - if nickLower in msgLower: - msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") - helper.sendMaster("NICK ACT %s (T:%s): (%s/%s) %s" % (self.name, msgLower.count(nickLower), user, channel, msgLower)) - if toSend: - helper.sendMaster("MATCH ACT %s (U:%s T:%s): (%s/%s) %s" % (self.name, toSend[1], toSend[2], user, channel, toSend[0])) + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + helper.actKeyword(user, channel, msg, self.nickname, "ACT", self.name) def get(self, var): try: @@ -214,6 +202,33 @@ class IRCBot(IRCClient): self.channels.remove(channel) helper.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) + def userJoined(self, user, channel): + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + def userLeft(self, user, channel): + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + def userQuit(self, user, quitMessage): + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + helper.actKeyword(user, None, quitMessage, self.nickname, "QUT", self.name) + + def userKicked(self, kickee, channel, kicker, message): + helper.actKeyword(kicker, channel, message, self.nickname, "KCK", self.name) + + def topicUpdated(self, user, channel, newTopic): + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + + helper.actKeyword(user, channel, newTopic, self.nickname, "TOP", self.name) + + def modeChanged(self, user, channel, toset, modes, args): + nick, ident, host = self.parsen(user) + helper.setWhoSingle(self.name, nick, ident, host) + def connectionLost(self, reason): self.connected = False self.channels = [] @@ -420,6 +435,20 @@ class Helper(object): else: return [message, uniqueNum, totalNum] + def actKeyword(self, user, channel, message, nickname, actType, name): + toSend = self.isKeyword(message) + if name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + if config["HighlightNotifications"]: + msgLower = message.lower() + nickLower = nickname.lower() + if nickLower in msgLower: + msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + self.sendMaster("NICK %s %s (T:%s): (%s/%s) %s" % (actType, name, msgLower.count(nickLower), user, channel, msgLower)) + if toSend: + helper.sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) + def addBot(self, name): global IRCPool instance = pool[name] @@ -471,7 +500,7 @@ class Helper(object): return True def parseCommand(self, addr, authed, data): - global pool, MonitorPool + global pool, MonitorPool, wholist spl = data.split() if addr in connections.keys(): obj = connections[addr] @@ -541,6 +570,47 @@ class Helper(object): info("\n".join(poolMap)) return + elif cmd == "stats": + if length == 1: + stats = [] + numChannels = 0 + numWhoEntries = 0 + for i in IRCPool.keys(): + numChannels += len(IRCPool[i].channels) + for i in wholist.keys(): + numWhoEntries += len(wholist[i].keys()) + stats.append("Servers: %s" % len(IRCPool.keys())) + stats.append("Channels: %s" % numChannels) + stats.append("User records: %s" % numWhoEntries) + info("\n".join(stats)) + return + + elif length == 2: + stats = [] + numChannels = 0 + numWhoEntries = 0 + + failures = 0 + if spl[1] in IRCPool.keys(): + numChannels += len(IRCPool[spl[1]].channels) + else: + failure("Name does not exist: %s" % spl[1]) + failures += 1 + if spl[1] in wholist.keys(): + numWhoEntries += len(wholist[spl[1]].keys()) + else: + failure("Who entry does not exist: %s" % spl[1]) + failures += 1 + if failures == 2: + failure("No information found, aborting") + return + stats.append("Channels: %s" % numChannels) + stats.append("User records: %s" % numWhoEntries) + info("\n".join(stats)) + return + else: + incUsage("stats") + elif cmd == "enable": if length == 2: if not spl[1] in pool.keys(): @@ -630,7 +700,6 @@ class Helper(object): return elif cmd == "who": - global wholist if length == 2: result = self.getWho(spl[1]) rtrn = "" @@ -638,7 +707,8 @@ class Helper(object): rtrn += "Matches from: %s" % i rtrn += "\n" for x in result[i]: - rtrn += (", ".join(x)) + x = [y for y in x if not y == None] + rtrn += str((", ".join(x))) rtrn += "\n" info(rtrn) return From 3eb4e2dcb2f317f2ac905dc37314f9a7c0d0f250 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 25 Dec 2017 22:27:09 +0000 Subject: [PATCH 046/394] Implement writing and loading the wholist from a file --- threshold | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/threshold b/threshold index 574c5ec..f36ac69 100755 --- a/threshold +++ b/threshold @@ -321,6 +321,7 @@ class Helper(object): wholist[network] = {} for i in newObjects.keys(): wholist[network][i] = newObjects[i] + return def setWhoSingle(self, network, nick, ident, host): @@ -344,22 +345,13 @@ class Helper(object): result[i].append(wholist[i][x]) return result - def saveConfig(self): - global config - with open("config.json", "w") as f: - dump(config, f, indent=4) - return - def getKeywordConfig(self): with open("keyword.json", "r") as f: - keyconf = load(f) - return keyconf + return load(f) - def saveKeywordConfig(self): - global keyconf - with open("keyword.json", "w") as f: - dump(keyconf, f, indent=4) - return + def getWholist(self): + with open("wholist.json", "r") as f: + return load(f) def getPool(self): with open("pool.json", "r") as f: @@ -370,6 +362,24 @@ class Helper(object): else: return {} + def saveConfig(self): + global config + with open("config.json", "w") as f: + dump(config, f, indent=4) + return + + def saveKeywordConfig(self): + global keyconf + with open("keyword.json", "w") as f: + dump(keyconf, f, indent=4) + return + + def saveWholist(self): + global wholist + with open("wholist.json", "w") as f: + dump(wholist, f, indent=4) + return + def savePool(self): global pool with open("pool.json", "w") as f: @@ -540,6 +550,11 @@ class Helper(object): log("Keyword configuration rehashed") success("Keyword configuration rehashed successfully") + elif cmd == "savewho": + self.saveWholist() + success("Saved WHO info to file") + return + elif cmd == "dist": if config["DistEnabled"]: rtrn = run(["./dist.sh"], shell=True, stdout=PIPE) @@ -1018,6 +1033,8 @@ if __name__ == "__main__": pool = helper.getPool() help = helper.getHelp() + wholist = helper.getWholist() + for i in pool.keys(): if pool[i]["enabled"] == True: helper.addBot(i) From d144b864525464609cc450b67c34c6b5e8ca6583 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 25 Dec 2017 22:27:41 +0000 Subject: [PATCH 047/394] Add the wholist to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7361a72..91e5dd0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ *.pem config.json pool.json +wholist.json dist.sh env/ From 740fc0034ceb541e20c893ae2837677f0d6f6f3d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 26 Dec 2017 13:57:29 +0000 Subject: [PATCH 048/394] Make the add command much more flexible and properly implement the defaults system --- example.json | 20 +++++++- help.json | 2 +- threshold | 133 +++++++++++++++++++++++++++++++++++---------------- 3 files changed, 110 insertions(+), 45 deletions(-) diff --git a/example.json b/example.json index 5faaaaf..8244110 100644 --- a/example.json +++ b/example.json @@ -12,9 +12,25 @@ "SendDistOutput": false, "Password": "s", "Default": { - "password": null, + "host": null, + "port": null, + "protocol": null, + "bind": null, + "timeout": 30, + "nickname": null, "username": null, - "authtype": null + "realname": null, + "userinfo": null, + "finger": null, + "version": null, + "source": null, + "autojoin": [], + "authtype": null, + "password": null, + "authentity": "NickServ", + "key": "key.pem", + "certificate": "cert.pem", + "enabled": true }, "Master": [null, null] } diff --git a/help.json b/help.json index 99ff659..4473d75 100644 --- a/help.json +++ b/help.json @@ -1,7 +1,7 @@ { "pass": "pass ", "logout": "logout", - "add": "add
", + "add": "add [
] [] [] []", "del": "del ", "mod": "mod [] []", "get": "get ", diff --git a/threshold b/threshold index f36ac69..728b038 100755 --- a/threshold +++ b/threshold @@ -872,52 +872,101 @@ class Helper(object): return elif cmd == "add": - if length == 6: - - if spl[1] in pool.keys(): - failure("Name already exists: %s" % spl[1]) - return - - protocol = spl[4].lower() - - if not protocol in ["ssl", "plain"]: - incUsage("connect") - return - - try: - int(spl[3]) - except: - failure("Port must be an integer, not %s" % spl[3]) - return - - pool[spl[1]] = { "host": spl[2], - "port": spl[3], - "protocol": protocol, - "bind": None, - "timeout": 30, - "nickname": spl[5], - "username": config["Default"]["username"], - "realname": None, - "userinfo": None, - "finger": None, - "version": None, - "source": None, - "autojoin": [], - "authtype": config["Default"]["authtype"], - "password": config["Default"]["password"], - "authentity": "NickServ", - "key": config["ListenerKey"], - "certificate": config["ListenerCertificate"], - "enabled": config["ConnectOnCreate"], - } - if config["ConnectOnCreate"] == True: - self.addBot(spl[1]) - success("Successfully created bot") - self.savePool() + if length > 6: + failure("Too many arguments") return + + if length > 1: + name = spl[1] else: incUsage("add") return + if length > 2: + host = spl[2] + if length > 3: + port = spl[3] + if length > 4: + protocol = spl[4] + if length > 5: + nickname = spl[5] + + toFail = False + if length < 6: + if config["Default"]["nickname"] == None: + failure("Choose a nickname, or configure one in defaults") + toFail = True + else: + nickname = config["Default"]["nickname"] + + if length < 5: + if config["Default"]["protocol"] == None: + failure("Choose a protocol, or configure one in defaults") + toFail = True + else: + protocol = config["Default"]["protocol"] + + if length < 4: + if config["Default"]["port"] == None: + failure("Choose a port, or configure one in defaults") + toFail = True + else: + port = config["Default"]["port"] + + if length < 3: + if config["Default"]["host"] == None: + failure("Choose a host, or configure one in defaults") + toFail = True + else: + host = config["Default"]["host"] + if toFail: + failure("Stopping due to previous error(s)") + return + + if length < 2: + incUsage("add") + return + + if name in pool.keys(): + failure("Name already exists: %s" % name) + return + + protocol = protocol.lower() + + if not protocol in ["ssl", "plain"]: + failure("Protocol must be ssl or plain, not %s" % protocol) + return + + try: + int(port) + except: + failure("Port must be an integer, not %s" % port) + return + + pool[name] = { "host": host, + "port": port, + "protocol": protocol, + "bind": config["Default"]["bind"], + "timeout": config["Default"]["timeout"], + "nickname": nickname, + "username": config["Default"]["username"], + "realname": config["Default"]["realname"], + "userinfo": config["Default"]["userinfo"], + "finger": config["Default"]["finger"], + "version": config["Default"]["version"], + "source": config["Default"]["source"], + "autojoin": config["Default"]["autojoin"], + "authtype": config["Default"]["authtype"], + "password": config["Default"]["password"], + "authentity": config["Default"]["authentity"], + "key": config["ListenerKey"], + "certificate": config["ListenerCertificate"], + "enabled": config["ConnectOnCreate"], + } + if config["ConnectOnCreate"] == True: + self.addBot(name) + success("Successfully created bot") + self.savePool() + return elif cmd == "del": if length == 2: From 34d54686720799a279a170f8893be655950a57ca Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 26 Dec 2017 14:25:36 +0000 Subject: [PATCH 049/394] Implement command to manage defaults --- example.json | 1 - help.json | 1 + threshold | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/example.json b/example.json index 8244110..fa6e81d 100644 --- a/example.json +++ b/example.json @@ -7,7 +7,6 @@ "UsePassword": true, "ConnectOnCreate": false, "HighlightNotifications": true, - "Debugger": false, "DistEnabled": true, "SendDistOutput": false, "Password": "s", diff --git a/help.json b/help.json index 4473d75..eb73280 100644 --- a/help.json +++ b/help.json @@ -4,6 +4,7 @@ "add": "add [
] [] [] []", "del": "del ", "mod": "mod [] []", + "default": "default [] []", "get": "get ", "key": "key [] [] [] []", "who": "who ", diff --git a/threshold b/threshold index 728b038..6dd3720 100755 --- a/threshold +++ b/threshold @@ -985,6 +985,70 @@ class Helper(object): else: incUsage("del") return + elif cmd == "default": + toUnset = False + if length == 1: + optionMap = ["Viewing defaults"] + for i in config["Default"].keys(): + optionMap.append("%s: %s" % (i, config["Default"][i])) + info("\n".join(optionMap)) + return + elif length == 2: + if not spl[1] in config["Default"].keys(): + failure("No such key: %s" % spl[1]) + return + info("%s: %s" % (spl[1], config["Default"][spl[1]])) + return + elif length == 3: + if not spl[1] in config["Default"].keys(): + failure("No such key: %s" % spl[1]) + return + + if spl[2].lower() in ["none", "nil"]: + spl[2] = None + toUnset = True + + if spl[1] in ["port", "timeout"]: + try: + int(spl[2]) + except: + failure("Value must be an integer, not %s" % spl[2]) + return + + if spl[1] == "protocol": + if not toUnset: + if not spl[2] in ["ssl", "plain"]: + failure("Protocol must be ssl or plain, not %s" % spl[2]) + return + + if spl[2] == config["Default"][spl[1]]: + failure("Value already exists: %s" % spl[2]) + return + + if spl[1] == "authtype": + if not toUnset: + if not spl[2] in ["sp", "ns"]: + failure("Authtype must be sp or ns, not %s" % spl[2]) + return + if spl[1] == "enabled": + failure("Use the ConnectOnCreate config parameter to set this") + return + if spl[1] == "autojoin": + if not toUnset: + spl[2] = spl[2].split(",") + else: + spl[2] = [] + + config["Default"][spl[1]] = spl[2] + self.saveConfig() + if toUnset: + success("Successfully unset key %s" % spl[1]) + else: + success("Successfully set key %s to %s" % (spl[1], spl[2])) + return + else: + incUsage("default") + return elif cmd == "mod": toUnset = False From 97cc80f3b8d753ad67bf91632c0408a834299aab Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Dec 2017 18:46:17 +0000 Subject: [PATCH 050/394] Get more information from kick and nick commands --- threshold | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/threshold b/threshold index 6dd3720..9dc202a 100755 --- a/threshold +++ b/threshold @@ -217,8 +217,16 @@ class IRCBot(IRCClient): helper.actKeyword(user, None, quitMessage, self.nickname, "QUT", self.name) def userKicked(self, kickee, channel, kicker, message): + nick, ident, host = self.parsen(kicker) + helper.setWhoSingle(self.name, nick, ident, host) + helper.actKeyword(kicker, channel, message, self.nickname, "KCK", self.name) + def userRenamed(self, oldname, newname): + nick, ident, host = self.parsen(oldname) + helper.setWhoSingle(self.name, nick, ident, host) + helper.setWhoSingle(self.name, newname, ident, host) + def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) helper.setWhoSingle(self.name, nick, ident, host) From 12b6fe2acb454b51f4e78d1ef6d3d64b09d80e58 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Jan 2018 19:10:57 +0000 Subject: [PATCH 051/394] Replace numerous functions which loaded and saved/loaded from/to JSON files with a single handler --- help.json | 4 +- threshold | 128 ++++++++++++++++++++++-------------------------------- 2 files changed, 53 insertions(+), 79 deletions(-) diff --git a/help.json b/help.json index eb73280..b17ae32 100644 --- a/help.json +++ b/help.json @@ -13,7 +13,7 @@ "disable": "disable ", "list": "list", "stats": "stats", - "rehash": "rehash", - "rekey": "rekey", + "save": "save ", + "load": "load ", "dist": "dist" } diff --git a/threshold b/threshold index 9dc202a..9f7650c 100755 --- a/threshold +++ b/threshold @@ -22,7 +22,13 @@ IRCPool = {} MonitorPool = [] -wholist = {} +filemap = { + "config": ["config.json", "configuration"], + "keyconf": ["keyword.json", "keyword lists"], + "pool": ["pool.json", "pool"], + "help": ["help.json", "command help"], + "wholist": ["wholist.json", "WHO lists"], +} def log(data): print("[LOG]", data) @@ -353,51 +359,14 @@ class Helper(object): result[i].append(wholist[i][x]) return result - def getKeywordConfig(self): - with open("keyword.json", "r") as f: - return load(f) - - def getWholist(self): - with open("wholist.json", "r") as f: - return load(f) - - def getPool(self): - with open("pool.json", "r") as f: - data = f.read() - if not data == "": - pool = loads(data.strip()) - return pool - else: - return {} - - def saveConfig(self): - global config - with open("config.json", "w") as f: - dump(config, f, indent=4) + def save(self, var): + with open(filemap[var][0], "w") as f: + dump(globals()[var], f, indent=4) return - def saveKeywordConfig(self): - global keyconf - with open("keyword.json", "w") as f: - dump(keyconf, f, indent=4) - return - - def saveWholist(self): - global wholist - with open("wholist.json", "w") as f: - dump(wholist, f, indent=4) - return - - def savePool(self): - global pool - with open("pool.json", "w") as f: - dump(pool, f, indent=4) - return - - def getHelp(self): - with open("help.json", "r") as f: - help = load(f) - return help + def load(self, var): + with open(filemap[var][0], "r") as f: + globals()[var] = load(f) def incorrectUsage(self, addr, mode): if mode == None: @@ -507,14 +476,14 @@ class Helper(object): if i in keyword or keyword in i: return "ISIN" keyconf["Keywords"].append(keyword) - helper.saveKeywordConfig() + self.save("keyconf")() return True def delKeyword(self, keyword): if not keyword in keyconf["Keywords"]: return "NOKEY" keyconf["Keywords"].remove(keyword) - helper.saveKeywordConfig() + self.save("keyconf")() return True def parseCommand(self, addr, authed, data): @@ -546,22 +515,31 @@ class Helper(object): info("\n".join(helpMap)) return - elif cmd == "rehash": - global config - config = helper.getConfig() - log("Configuration rehashed") - success("Configuration rehashed successfully") + elif cmd == "save": + if length == 2: + if spl[1] in filemap.keys(): + self.save(spl[1]) + success("Saved %s to %s" % (spl[1], filemap[spl[1]][0])) + return + else: + incUsage("save") + return + else: + incUsage("save") + return - elif cmd == "rekey": - global keyconf - keyconf = helper.getKeywordConfig() - log("Keyword configuration rehashed") - success("Keyword configuration rehashed successfully") - - elif cmd == "savewho": - self.saveWholist() - success("Saved WHO info to file") - return + elif cmd == "load": + if length == 2: + if spl[1] in filemap.keys(): + self.load(spl[1]) + success("Loaded %s from %s" % (spl[1], filemap[spl[1]][0])) + return + else: + incUsage("load") + return + else: + incUsage("load") + return elif cmd == "dist": if config["DistEnabled"]: @@ -640,7 +618,7 @@ class Helper(object): failure("Name does not exist: %s" % spl[1]) return pool[spl[1]]["enabled"] = True - helper.savePool() + self.save("pool") if not spl[1] in IRCPool.keys(): self.addBot(spl[1]) else: @@ -657,7 +635,7 @@ class Helper(object): failure("Name does not exist: %s" % spl[1]) return pool[spl[1]]["enabled"] = False - helper.savePool() + self.save("pool") if spl[1] in IRCPool.keys(): if IRCPool[spl[1]].connected == True: IRCPool[spl[1]].transport.loseConnection() @@ -786,7 +764,7 @@ class Helper(object): keyconf["KeywordsExcept"][spl[2]] = [] keyconf["KeywordsExcept"][spl[2]].append(spl[3]) - helper.saveKeywordConfig() + self.save("keyconf")() success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) return elif spl[1] == "master": @@ -797,7 +775,7 @@ class Helper(object): if not spl[3] in IRCPool[spl[2]].channels: info("Bot not on channel: %s" % spl[3]) config["Master"] = [spl[2], spl[3]] - helper.saveConfig() + self.save("config") success("Master set to %s on %s" % (spl[3], spl[2])) return elif spl[1] == "unexcept": @@ -810,7 +788,7 @@ class Helper(object): keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) if keyconf["KeywordsExcept"][spl[2]] == []: del keyconf["KeywordsExcept"][spl[2]] - helper.saveKeywordConfig() + self.save("keyconf")() success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) return else: @@ -822,7 +800,7 @@ class Helper(object): failure("No such exception: %s" % spl[2]) return del keyconf["KeywordsExcept"][spl[2]] - helper.saveKeywordConfig() + self.save("keyconf")() success("Successfully removed exception list of %s" % spl[2]) return elif spl[1] == "monitor": @@ -973,7 +951,7 @@ class Helper(object): if config["ConnectOnCreate"] == True: self.addBot(name) success("Successfully created bot") - self.savePool() + self.save("pool") return elif cmd == "del": @@ -988,7 +966,7 @@ class Helper(object): IRCPool[spl[1]].transport.loseConnection() del IRCPool[spl[1]] success("Successfully removed bot") - self.savePool() + self.save("pool") return else: incUsage("del") @@ -1048,7 +1026,7 @@ class Helper(object): spl[2] = [] config["Default"][spl[1]] = spl[2] - self.saveConfig() + self.save("config") if toUnset: success("Successfully unset key %s" % spl[1]) else: @@ -1119,7 +1097,7 @@ class Helper(object): pool[spl[1]][spl[2]] = spl[3] if spl[1] in IRCPool.keys(): IRCPool[spl[1]].refresh() - self.savePool() + self.save("pool") if toUnset: success("Successfully unset key %s on %s" % (spl[2], spl[1])) else: @@ -1149,12 +1127,8 @@ class Helper(object): if __name__ == "__main__": helper = Helper() - config = helper.getConfig() - keyconf = helper.getKeywordConfig() - pool = helper.getPool() - help = helper.getHelp() - - wholist = helper.getWholist() + for i in filemap.keys(): + helper.load(i) for i in pool.keys(): if pool[i]["enabled"] == True: From a7599dddfeee0c2a23d56b947b7e3d8d71f125e0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 17 Jan 2018 19:02:41 +0000 Subject: [PATCH 052/394] Separate out into different directories and reshuffle configuration directives --- conf/example.json | 40 ++++++++++++++++++++++++++++++++++++++++ conf/help.json | 19 +++++++++++++++++++ conf/keyword.json | 6 ++++++ threshold | 43 +++++++++++++++++-------------------------- 4 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 conf/example.json create mode 100644 conf/help.json create mode 100644 conf/keyword.json diff --git a/conf/example.json b/conf/example.json new file mode 100644 index 0000000..c4e6afe --- /dev/null +++ b/conf/example.json @@ -0,0 +1,40 @@ +{ + "Listener": { + "Port": 13867, + "Address": "127.0.0.1", + "UseSSL": true, + "Key": "key.pem", + "Certificate": "cert.pem" + }, + "UsePassword": true, + "ConnectOnCreate": false, + "HighlightNotifications": true, + "Dist": { + "Enabled": true, + "SendOutput": false, + "File": "conf/dist.sh" + }, + "Password": "s", + "Default": { + "host": null, + "port": null, + "protocol": null, + "bind": null, + "timeout": 30, + "nickname": null, + "username": null, + "realname": null, + "userinfo": null, + "finger": null, + "version": null, + "source": null, + "autojoin": [], + "authtype": null, + "password": null, + "authentity": "NickServ", + "key": "key.pem", + "certificate": "cert.pem", + "enabled": true + }, + "Master": [null, null] +} diff --git a/conf/help.json b/conf/help.json new file mode 100644 index 0000000..b17ae32 --- /dev/null +++ b/conf/help.json @@ -0,0 +1,19 @@ +{ + "pass": "pass ", + "logout": "logout", + "add": "add [
] [] [] []", + "del": "del ", + "mod": "mod [] []", + "default": "default [] []", + "get": "get ", + "key": "key [] [] [] []", + "who": "who ", + "join": "join []", + "enable": "enable ", + "disable": "disable ", + "list": "list", + "stats": "stats", + "save": "save ", + "load": "load ", + "dist": "dist" +} diff --git a/conf/keyword.json b/conf/keyword.json new file mode 100644 index 0000000..cc4da5b --- /dev/null +++ b/conf/keyword.json @@ -0,0 +1,6 @@ +{ + "Keywords": [ + "kek" + ], + "KeywordsExcept": {} +} \ No newline at end of file diff --git a/threshold b/threshold index 9f7650c..c38d8fb 100755 --- a/threshold +++ b/threshold @@ -22,6 +22,9 @@ IRCPool = {} MonitorPool = [] +configPath = "conf/" +certPath = "cert/" + filemap = { "config": ["config.json", "configuration"], "keyconf": ["keyword.json", "keyword lists"], @@ -314,20 +317,6 @@ class BaseFactory(Factory): return class Helper(object): - def getConfig(self): - with open("config.json", "r") as f: - config = load(f) - if set(["Port", "BindAddress", "UseSSL", "UsePassword"]).issubset(set(config.keys())): - if config["UseSSL"] == True: - if not set(["ListenerKey", "ListenerCertificate"]).issubset(set(config.keys())): - error("SSL is on but certificate or key is not defined") - if config["UsePassword"] == True: - if not "Password" in config.keys(): - error("Password authentication is on but password is not defined") - return config - else: - error("Mandatory values missing from config") - def setWho(self, network, newObjects): global wholist network = "".join([x for x in network if not x in numbers]) @@ -360,12 +349,12 @@ class Helper(object): return result def save(self, var): - with open(filemap[var][0], "w") as f: + with open(configPath+filemap[var][0], "w") as f: dump(globals()[var], f, indent=4) return def load(self, var): - with open(filemap[var][0], "r") as f: + with open(configPath+filemap[var][0], "r") as f: globals()[var] = load(f) def incorrectUsage(self, addr, mode): @@ -437,7 +426,7 @@ class Helper(object): helper.sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) def addBot(self, name): - global IRCPool + global IRCPool, certPath instance = pool[name] log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) if instance["protocol"] == "plain": @@ -454,7 +443,9 @@ class Helper(object): d = connectProtocol(point, bot) return elif instance["protocol"] == "ssl": - contextFactory = DefaultOpenSSLContextFactory(instance["key"].encode("utf-8", "replace"), instance["certificate"].encode("utf-8", "replace")) + keyFN = certPath+instance["key"] + certFN = certPath+instance["certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) if instance["bind"] == None: point = SSL4ClientEndpoint(reactor, instance["host"], int(instance["port"]), contextFactory, timeout=int(instance["timeout"])) bot = IRCBot(name) @@ -542,9 +533,9 @@ class Helper(object): return elif cmd == "dist": - if config["DistEnabled"]: - rtrn = run(["./dist.sh"], shell=True, stdout=PIPE) - if config["SendDistOutput"]: + if config["Dist"]["Enabled"]: + rtrn = run([config["Dist"]["File"]], shell=True, stdout=PIPE) + if config["Dist"]["SendOutput"]: info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) else: info("Exit code: %s" % rtrn.returncode) @@ -1135,11 +1126,11 @@ if __name__ == "__main__": helper.addBot(i) listener = BaseFactory() - if config["UseSSL"] == True: - reactor.listenSSL(config["Port"], listener, DefaultOpenSSLContextFactory(config["ListenerKey"], config["ListenerCertificate"]), interface=config["BindAddress"]) - log("Threshold running with SSL on %s:%s" % (config["BindAddress"], config["Port"])) + if config["Listener"]["UseSSL"] == True: + reactor.listenSSL(config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(certPath+config["Listener"]["Key"], certPath+config["Listener"]["Certificate"]), interface=config["Listener"]["Address"]) + log("Threshold running with SSL on %s:%s" % (config["Listener"]["Address"], config["Listener"]["Port"])) else: - reactor.listenTCP(config["Port"], listener, interface=config["BindAddress"]) - log("Threshold running on %s:%s" % (config["BindAddress"], config["Port"])) + reactor.listenTCP(config["Listener"]["Port"], listener, interface=config["Listener"]["Address"]) + log("Threshold running on %s:%s" % (config["Listener"]["Address"], config["Listener"]["Port"])) reactor.run() From 6776cec2d4fd63824e19ee7ac64f368cab5689da Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 17 Jan 2018 19:04:58 +0000 Subject: [PATCH 053/394] Remove old config files --- conf/keyword.json | 4 ++-- example.json | 35 ----------------------------------- help.json | 19 ------------------- keyword.json | 6 ------ 4 files changed, 2 insertions(+), 62 deletions(-) delete mode 100644 example.json delete mode 100644 help.json delete mode 100644 keyword.json diff --git a/conf/keyword.json b/conf/keyword.json index cc4da5b..cd5374c 100644 --- a/conf/keyword.json +++ b/conf/keyword.json @@ -1,6 +1,6 @@ { "Keywords": [ - "kek" + "example" ], "KeywordsExcept": {} -} \ No newline at end of file +} diff --git a/example.json b/example.json deleted file mode 100644 index fa6e81d..0000000 --- a/example.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "Port": 13867, - "BindAddress": "127.0.0.1", - "UseSSL": true, - "ListenerKey": "key.pem", - "ListenerCertificate": "cert.pem", - "UsePassword": true, - "ConnectOnCreate": false, - "HighlightNotifications": true, - "DistEnabled": true, - "SendDistOutput": false, - "Password": "s", - "Default": { - "host": null, - "port": null, - "protocol": null, - "bind": null, - "timeout": 30, - "nickname": null, - "username": null, - "realname": null, - "userinfo": null, - "finger": null, - "version": null, - "source": null, - "autojoin": [], - "authtype": null, - "password": null, - "authentity": "NickServ", - "key": "key.pem", - "certificate": "cert.pem", - "enabled": true - }, - "Master": [null, null] -} diff --git a/help.json b/help.json deleted file mode 100644 index b17ae32..0000000 --- a/help.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "pass": "pass ", - "logout": "logout", - "add": "add [
] [] [] []", - "del": "del ", - "mod": "mod [] []", - "default": "default [] []", - "get": "get ", - "key": "key [] [] [] []", - "who": "who ", - "join": "join []", - "enable": "enable ", - "disable": "disable ", - "list": "list", - "stats": "stats", - "save": "save ", - "load": "load ", - "dist": "dist" -} diff --git a/keyword.json b/keyword.json deleted file mode 100644 index cc4da5b..0000000 --- a/keyword.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "Keywords": [ - "kek" - ], - "KeywordsExcept": {} -} \ No newline at end of file From ff66c21f6345dfa7fa260684891a65d671920220 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 20 Jan 2018 21:26:35 +0000 Subject: [PATCH 054/394] Use the correct warning call --- threshold | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/threshold b/threshold index c38d8fb..f93351e 100755 --- a/threshold +++ b/threshold @@ -375,7 +375,7 @@ class Helper(object): if config["Master"][0] in IRCPool.keys(): IRCPool[config["Master"][0]].msg(config["Master"][1], data) else: - warning("Master with no IRC instance defined") + warn("Master with no IRC instance defined") for i in MonitorPool: connections[i].send(data) @@ -483,7 +483,7 @@ class Helper(object): if addr in connections.keys(): obj = connections[addr] else: - warning("Got connection object with no instance in the address pool") + warn("Got connection object with no instance in the address pool") return success = lambda data: sendSuccess(addr, data) From 9119a7b511fe52f5416e2330dcd72574ce0a8563 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 20 Jan 2018 21:39:43 +0000 Subject: [PATCH 055/394] Use the proper values for the default certificate and key --- threshold | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/threshold b/threshold index f93351e..98e9f1a 100755 --- a/threshold +++ b/threshold @@ -935,8 +935,8 @@ class Helper(object): "authtype": config["Default"]["authtype"], "password": config["Default"]["password"], "authentity": config["Default"]["authentity"], - "key": config["ListenerKey"], - "certificate": config["ListenerCertificate"], + "key": config["Default"]["key"], + "certificate": config["Default"]["certificate"], "enabled": config["ConnectOnCreate"], } if config["ConnectOnCreate"] == True: From b1020589e2e1e13f4d8aa10e1c12464755928d88 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 22 Jan 2018 08:01:23 +0000 Subject: [PATCH 056/394] Fix save calls --- threshold | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/threshold b/threshold index 98e9f1a..162e107 100755 --- a/threshold +++ b/threshold @@ -467,14 +467,14 @@ class Helper(object): if i in keyword or keyword in i: return "ISIN" keyconf["Keywords"].append(keyword) - self.save("keyconf")() + self.save("keyconf") return True def delKeyword(self, keyword): if not keyword in keyconf["Keywords"]: return "NOKEY" keyconf["Keywords"].remove(keyword) - self.save("keyconf")() + self.save("keyconf") return True def parseCommand(self, addr, authed, data): @@ -755,7 +755,7 @@ class Helper(object): keyconf["KeywordsExcept"][spl[2]] = [] keyconf["KeywordsExcept"][spl[2]].append(spl[3]) - self.save("keyconf")() + self.save("keyconf") success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) return elif spl[1] == "master": @@ -779,7 +779,7 @@ class Helper(object): keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) if keyconf["KeywordsExcept"][spl[2]] == []: del keyconf["KeywordsExcept"][spl[2]] - self.save("keyconf")() + self.save("keyconf") success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) return else: @@ -791,7 +791,7 @@ class Helper(object): failure("No such exception: %s" % spl[2]) return del keyconf["KeywordsExcept"][spl[2]] - self.save("keyconf")() + self.save("keyconf") success("Successfully removed exception list of %s" % spl[2]) return elif spl[1] == "monitor": From 5c7669e09c8bb8d89cbcda5bdc39b0e99d5fc2ba Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 2 Feb 2018 18:04:56 +0000 Subject: [PATCH 057/394] Fix the help for the stats command --- conf/help.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/help.json b/conf/help.json index b17ae32..0d8ac61 100644 --- a/conf/help.json +++ b/conf/help.json @@ -12,7 +12,7 @@ "enable": "enable ", "disable": "disable ", "list": "list", - "stats": "stats", + "stats": "stats []", "save": "save ", "load": "load ", "dist": "dist" From 89ac1c1ce372885940dd0fb2ab6a84709700f80d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 2 Feb 2018 18:56:50 +0000 Subject: [PATCH 058/394] Implement loading/saving all configuration files at once --- conf/help.json | 2 +- conf/keyword.json | 2 +- threshold | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/conf/help.json b/conf/help.json index 0d8ac61..b505b6c 100644 --- a/conf/help.json +++ b/conf/help.json @@ -16,4 +16,4 @@ "save": "save ", "load": "load ", "dist": "dist" -} +} \ No newline at end of file diff --git a/conf/keyword.json b/conf/keyword.json index cd5374c..3980412 100644 --- a/conf/keyword.json +++ b/conf/keyword.json @@ -3,4 +3,4 @@ "example" ], "KeywordsExcept": {} -} +} \ No newline at end of file diff --git a/threshold b/threshold index 162e107..f7c4b97 100755 --- a/threshold +++ b/threshold @@ -512,6 +512,11 @@ class Helper(object): self.save(spl[1]) success("Saved %s to %s" % (spl[1], filemap[spl[1]][0])) return + elif spl[1] == "all": + for i in filemap.keys(): + helper.save(i) + success("Saved %s from %s" % (i, filemap[i][0])) + return else: incUsage("save") return @@ -525,6 +530,11 @@ class Helper(object): self.load(spl[1]) success("Loaded %s from %s" % (spl[1], filemap[spl[1]][0])) return + elif spl[1] == "all": + for i in filemap.keys(): + helper.load(i) + success("Loaded %s from %s" % (i, filemap[i][0])) + return else: incUsage("load") return From c1bbf3c0d03e39bf205fb880da17971704c20e7e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 2 Feb 2018 19:01:41 +0000 Subject: [PATCH 059/394] Make a folder for example configuration files --- conf/{example.json => example/config.json} | 0 conf/example/keyword.json | 6 ++++++ 2 files changed, 6 insertions(+) rename conf/{example.json => example/config.json} (100%) create mode 100644 conf/example/keyword.json diff --git a/conf/example.json b/conf/example/config.json similarity index 100% rename from conf/example.json rename to conf/example/config.json diff --git a/conf/example/keyword.json b/conf/example/keyword.json new file mode 100644 index 0000000..3980412 --- /dev/null +++ b/conf/example/keyword.json @@ -0,0 +1,6 @@ +{ + "Keywords": [ + "example" + ], + "KeywordsExcept": {} +} \ No newline at end of file From cd8c61cdc6ae9501adac1b219fe0713cc6f45a01 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 3 Feb 2018 16:59:59 +0000 Subject: [PATCH 060/394] Update gitignore --- .gitignore | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 91e5dd0..e6278e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ *.pyc *.db *.pem -config.json -pool.json -wholist.json -dist.sh +conf/config.json +conf/pool.json +conf/wholist.json +conf/keyword.json +conf/dist.sh env/ From 88426be62a8f464322ccfd7461a5602b63afd0e2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 3 Feb 2018 19:03:47 +0000 Subject: [PATCH 061/394] Move things into factories and implement reconnecting --- conf/example/config.json | 7 ++- threshold | 103 +++++++++++++++++++++++++-------------- 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index c4e6afe..5ead613 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -21,6 +21,10 @@ "protocol": null, "bind": null, "timeout": 30, + "maxdelay": 5, + "initialdelay": 1, + "factor": 2, + "jitter": 5, "nickname": null, "username": null, "realname": null, @@ -33,8 +37,7 @@ "password": null, "authentity": "NickServ", "key": "key.pem", - "certificate": "cert.pem", - "enabled": true + "certificate": "cert.pem" }, "Master": [null, null] } diff --git a/threshold b/threshold index f7c4b97..9d4df1c 100755 --- a/threshold +++ b/threshold @@ -2,8 +2,7 @@ from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.internet.ssl import DefaultOpenSSLContextFactory -from twisted.internet.protocol import Protocol, Factory -from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint, connectProtocol +from twisted.internet.protocol import Protocol, Factory, ClientFactory, ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient #from twisted.python import log @@ -19,6 +18,8 @@ numbers = "0123456789" listener = None connections = {} IRCPool = {} +ReactorPool = {} +FactoryPool = {} MonitorPool = [] @@ -246,19 +247,39 @@ class IRCBot(IRCClient): nick, ident, host = self.parsen(user) helper.setWhoSingle(self.name, nick, ident, host) - def connectionLost(self, reason): - self.connected = False - self.channels = [] +class IRCBotFactory(ReconnectingClientFactory): + def __init__(self, name): + self.instance = pool[name] + self.name = name + self.maxDelay = self.instance["maxdelay"] + self.initialDelay = self.instance["initialdelay"] + self.factor = self.instance["factor"] + self.jitter = self.instance["jitter"] + + def buildProtocol(self, addr): + global IRCPool + entry = IRCBot(self.name) + IRCPool[self.name] = entry + self.client = entry + return entry + + def clientConnectionLost(self, connector, reason): + self.client.connected = False + self.client.channels = [] error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) helper.sendAll("%s: connection lost: %s" % (self.name, error)) + self.retry(connector) + #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) - def connectionFailed(self, reason): - self.connected = False - self.channels = [] + def clientConnectionFailed(self, connector, reason): + self.client.connected = False + self.client.channels = [] error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) helper.sendAll("%s: connection failed: %s" % (self.name, error)) + self.retry(connector) + #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) class Base(Protocol): def __init__(self, addr): @@ -431,32 +452,37 @@ class Helper(object): log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) if instance["protocol"] == "plain": if instance["bind"] == None: - point = TCP4ClientEndpoint(reactor, instance["host"], int(instance["port"]), timeout=int(instance["timeout"])) - bot = IRCBot(name) - IRCPool[name] = bot - d = connectProtocol(point, bot) + bot = IRCBotFactory(name) + rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"])) + + ReactorPool[name] = rct + FactoryPool[name] = bot return else: - point = TCP4ClientEndpoint(reactor, instance["host"], int(instance["port"]), timeout=int(instance["timeout"]), bindAddress=instance["bind"]) - bot = IRCBot(name) - IRCPool[name] = bot - d = connectProtocol(point, bot) + bot = IRCBotFactory(name) + rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) + + ReactorPool[name] = rct + FactoryPool[name] = bot return elif instance["protocol"] == "ssl": keyFN = certPath+instance["key"] certFN = certPath+instance["certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) if instance["bind"] == None: - point = SSL4ClientEndpoint(reactor, instance["host"], int(instance["port"]), contextFactory, timeout=int(instance["timeout"])) - bot = IRCBot(name) - IRCPool[name] = bot - d = connectProtocol(point, bot) + bot = IRCBotFactory(name) + rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory) + + ReactorPool[name] = rct + FactoryPool[name] = bot return else: - point = SSL4ClientEndpoint(reactor, instance["host"], int(instance["port"]), contextFactory, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) - bot = IRCBot(name) - IRCPool[name] = bot - d = connectProtocol(point, bot) + + bot = IRCBotFactory(name) + rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory, bindAddress=instance["bind"]) + + ReactorPool[name] = rct + FactoryPool[name] = bot return def addKeyword(self, keyword): @@ -637,10 +663,13 @@ class Helper(object): return pool[spl[1]]["enabled"] = False self.save("pool") - if spl[1] in IRCPool.keys(): - if IRCPool[spl[1]].connected == True: - IRCPool[spl[1]].transport.loseConnection() - del IRCPool[spl[1]] + if spl[1] in ReactorPool.keys(): + if spl[1] in FactoryPool.keys(): + FactoryPool[spl[1]].stopTrying() + ReactorPool[spl[1]].disconnect() + del IRCPool[spl[1]] + del ReactorPool[spl[1]] + del FactoryPool[spl[1]] success("Successfully disabled bot %s" % spl[1]) return else: @@ -995,9 +1024,9 @@ class Helper(object): spl[2] = None toUnset = True - if spl[1] in ["port", "timeout"]: + if spl[1] in ["port", "timeout", "maxdelay", "initialdelay", "factor", "jitter"]: try: - int(spl[2]) + spl[2] = int(spl[2]) except: failure("Value must be an integer, not %s" % spl[2]) return @@ -1066,12 +1095,7 @@ class Helper(object): if not spl[2] in pool[spl[1]].keys(): failure("No such key: %s" % spl[2]) return - if spl[2] in ["port", "timeout"]: - try: - int(spl[3]) - except: - failure("Value must be an integer, not %s" % spl[3]) - return + if spl[2] == "protocol": if not spl[3] in ["ssl", "plain"]: failure("Protocol must be ssl or plain, not %s" % spl[3]) @@ -1084,6 +1108,13 @@ class Helper(object): spl[3] = None toUnset = True + if spl[2] in ["port", "timeout", "maxdelay", "initialdelay", "factor", "jitter"]: + try: + spl[3] = int(spl[3]) + except: + failure("Value must be an integer, not %s" % spl[3]) + return + if spl[2] == "authtype": if not toUnset: if not spl[3] in ["sp", "ns"]: From aacb50d5d23ac177825138d67adf0892868ad09d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 3 Feb 2018 19:31:59 +0000 Subject: [PATCH 062/394] Update examples and change how variables used when reconnecting are set --- conf/example/config.json | 8 ++++---- threshold | 33 +++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 5ead613..d2f87ff 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -21,10 +21,10 @@ "protocol": null, "bind": null, "timeout": 30, - "maxdelay": 5, - "initialdelay": 1, - "factor": 2, - "jitter": 5, + "maxdelay": 360, + "initialdelay": 1.0, + "factor": 2.7182818284590451, + "jitter": 0.11962656472, "nickname": null, "username": null, "realname": null, diff --git a/threshold b/threshold index 9d4df1c..f87a78f 100755 --- a/threshold +++ b/threshold @@ -251,6 +251,7 @@ class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): self.instance = pool[name] self.name = name + self.client = None self.maxDelay = self.instance["maxdelay"] self.initialDelay = self.instance["initialdelay"] self.factor = self.instance["factor"] @@ -264,8 +265,9 @@ class IRCBotFactory(ReconnectingClientFactory): return entry def clientConnectionLost(self, connector, reason): - self.client.connected = False - self.client.channels = [] + if not self.client == None: + self.client.connected = False + self.client.channels = [] error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) helper.sendAll("%s: connection lost: %s" % (self.name, error)) @@ -273,8 +275,9 @@ class IRCBotFactory(ReconnectingClientFactory): #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): - self.client.connected = False - self.client.channels = [] + if not self.client == None: + self.client.connected = False + self.client.channels = [] error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) helper.sendAll("%s: connection failed: %s" % (self.name, error)) @@ -963,6 +966,10 @@ class Helper(object): "protocol": protocol, "bind": config["Default"]["bind"], "timeout": config["Default"]["timeout"], + "maxdelay": config["Default"]["maxdelay"], + "initialdelay": config["Default"]["initialdelay"], + "factor": config["Default"]["factor"], + "jitter": config["Default"]["jitter"], "nickname": nickname, "username": config["Default"]["username"], "realname": config["Default"]["realname"], @@ -1024,13 +1031,20 @@ class Helper(object): spl[2] = None toUnset = True - if spl[1] in ["port", "timeout", "maxdelay", "initialdelay", "factor", "jitter"]: + if spl[1] in ["port", "timeout", "maxdelay"]: try: spl[2] = int(spl[2]) except: failure("Value must be an integer, not %s" % spl[2]) return + if spl[2] in ["initialdelay", "factor", "jitter"]: + try: + spl[3] = float(spl[3]) + except: + failure("Value must be a floating point integer, not %s" % spl[3]) + return + if spl[1] == "protocol": if not toUnset: if not spl[2] in ["ssl", "plain"]: @@ -1108,13 +1122,20 @@ class Helper(object): spl[3] = None toUnset = True - if spl[2] in ["port", "timeout", "maxdelay", "initialdelay", "factor", "jitter"]: + if spl[2] in ["port", "timeout", "maxdelay"]: try: spl[3] = int(spl[3]) except: failure("Value must be an integer, not %s" % spl[3]) return + if spl[2] in ["initialdelay", "factor", "jitter"]: + try: + spl[3] = float(spl[3]) + except: + failure("Value must be a floating point integer, not %s" % spl[3]) + return + if spl[2] == "authtype": if not toUnset: if not spl[3] in ["sp", "ns"]: From 57e54cbced88e78f2922e5115834f19089a3b346 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 3 Feb 2018 19:43:54 +0000 Subject: [PATCH 063/394] Additional checks when disabling or deleting bots --- threshold | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/threshold b/threshold index f87a78f..59dfebd 100755 --- a/threshold +++ b/threshold @@ -670,7 +670,8 @@ class Helper(object): if spl[1] in FactoryPool.keys(): FactoryPool[spl[1]].stopTrying() ReactorPool[spl[1]].disconnect() - del IRCPool[spl[1]] + if spl[1] in IRCPool.keys(): + del IRCPool[spl[1]] del ReactorPool[spl[1]] del FactoryPool[spl[1]] success("Successfully disabled bot %s" % spl[1]) @@ -998,10 +999,14 @@ class Helper(object): failure("Name does not exist: %s" % spl[1]) return del pool[spl[1]] - if spl[1] in IRCPool.keys(): - if IRCPool[spl[1]].connected == True: - IRCPool[spl[1]].transport.loseConnection() - del IRCPool[spl[1]] + if spl[1] in ReactorPool.keys(): + if spl[1] in FactoryPool.keys(): + FactoryPool[spl[1]].stopTrying() + ReactorPool[spl[1]].disconnect() + if spl[1] in IRCPool.keys(): + del IRCPool[spl[1]] + del ReactorPool[spl[1]] + del FactoryPool[spl[1]] success("Successfully removed bot") self.save("pool") return From f53cb060d088747f218395fbbc5231e79b4500ba Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 21 Feb 2018 20:24:08 +0000 Subject: [PATCH 064/394] Add connecting and disconnecting notifications --- conf/example/config.json | 1 + threshold | 3 +++ 2 files changed, 4 insertions(+) diff --git a/conf/example/config.json b/conf/example/config.json index d2f87ff..7a944c1 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -9,6 +9,7 @@ "UsePassword": true, "ConnectOnCreate": false, "HighlightNotifications": true, + "ConnectionNotifications": true, "Dist": { "Enabled": true, "SendOutput": false, diff --git a/threshold b/threshold index 59dfebd..cc7569f 100755 --- a/threshold +++ b/threshold @@ -193,6 +193,7 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True log("signed on: %s" % self.name) + helper.sendMaster("SIGNON: %s" % self.name) if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) for i in self.autojoin: @@ -271,6 +272,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) helper.sendAll("%s: connection lost: %s" % (self.name, error)) + helper.sendMaster("CONNLOST %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -281,6 +283,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) helper.sendAll("%s: connection failed: %s" % (self.name, error)) + helper.sendMaster("CONNFAIL %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) From 5d5b5d4d47c481486924b16ab64e07abe2a65993 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 21 Feb 2018 20:31:47 +0000 Subject: [PATCH 065/394] Make connection notifications configurable --- conf/example/config.json | 2 +- threshold | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 7a944c1..f7bc320 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -9,7 +9,7 @@ "UsePassword": true, "ConnectOnCreate": false, "HighlightNotifications": true, - "ConnectionNotifications": true, + "ConnectionNotifications": true, "Dist": { "Enabled": true, "SendOutput": false, diff --git a/threshold b/threshold index cc7569f..0fe6149 100755 --- a/threshold +++ b/threshold @@ -193,7 +193,8 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True log("signed on: %s" % self.name) - helper.sendMaster("SIGNON: %s" % self.name) + if config["ConnectionNotifications"]: + helper.sendMaster("SIGNON: %s" % self.name) if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) for i in self.autojoin: @@ -272,7 +273,8 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) helper.sendAll("%s: connection lost: %s" % (self.name, error)) - helper.sendMaster("CONNLOST %s: %s" % (self.name, error)) + if config["ConnectionNotifications"]: + helper.sendMaster("CONNLOST %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -283,7 +285,8 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) helper.sendAll("%s: connection failed: %s" % (self.name, error)) - helper.sendMaster("CONNFAIL %s: %s" % (self.name, error)) + if config["ConnectionNotifications"]: + helper.sendMaster("CONNFAIL %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) From 4b3541625a0ad1c799256afbd8a208bee8cd25a0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 22 Feb 2018 19:30:31 +0000 Subject: [PATCH 066/394] Separate out the logging functions --- threshold | 15 ++------------- utils/logging/log.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 utils/logging/log.py diff --git a/threshold b/threshold index 0fe6149..bc41686 100755 --- a/threshold +++ b/threshold @@ -13,6 +13,8 @@ from json import load, dump, loads from sys import exit from subprocess import run, PIPE +from utils.logging.log import * + numbers = "0123456789" listener = None @@ -34,19 +36,6 @@ filemap = { "wholist": ["wholist.json", "WHO lists"], } -def log(data): - print("[LOG]", data) - -def debug(data): - print("[DEBUG]", data) - -def warn(data): - print("[WARNING]", data) - -def error(data): - print("[ERROR]", data) - exit(1) - def sendData(addr, data): connections[addr].send(data) diff --git a/utils/logging/log.py b/utils/logging/log.py new file mode 100644 index 0000000..87d0ed5 --- /dev/null +++ b/utils/logging/log.py @@ -0,0 +1,12 @@ +def log(data): + print("[LOG]", data) + +def debug(data): + print("[DEBUG]", data) + +def warn(data): + print("[WARNING]", data) + +def error(data): + print("[ERROR]", data) + exit(1) From cb7142ef88024bdc07287b886604f28105df179e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 23 Feb 2018 22:05:40 +0000 Subject: [PATCH 067/394] Separate out everything into files and implement a modules system to segment commands --- commands/__init__.py | 0 commands/add.py | 110 +++ commands/default.py | 80 +++ commands/delete.py | 29 + commands/disable.py | 29 + commands/dist.py | 20 + commands/enable.py | 26 + commands/get.py | 22 + commands/help.py | 15 + commands/join.py | 33 + commands/key.py | 149 ++++ commands/list.py | 18 + commands/load.py | 26 + commands/logout.py | 15 + commands/mod.py | 89 +++ commands/part.py | 23 + commands/password.py | 22 + commands/save.py | 26 + commands/stats.py | 49 ++ commands/who.py | 26 + core/bot.py | 240 +++++++ core/helper.py | 44 ++ core/main.py | 50 ++ core/parser.py | 30 + modules/keyword.py | 74 ++ modules/userinfo.py | 31 + threshold | 1151 +------------------------------ utils/loaders/command_loader.py | 15 + utils/logging/log.py | 1 - utils/logging/send.py | 26 + 30 files changed, 1346 insertions(+), 1123 deletions(-) create mode 100644 commands/__init__.py create mode 100644 commands/add.py create mode 100644 commands/default.py create mode 100644 commands/delete.py create mode 100644 commands/disable.py create mode 100644 commands/dist.py create mode 100644 commands/enable.py create mode 100644 commands/get.py create mode 100644 commands/help.py create mode 100644 commands/join.py create mode 100644 commands/key.py create mode 100644 commands/list.py create mode 100644 commands/load.py create mode 100644 commands/logout.py create mode 100644 commands/mod.py create mode 100644 commands/part.py create mode 100644 commands/password.py create mode 100644 commands/save.py create mode 100644 commands/stats.py create mode 100644 commands/who.py create mode 100644 core/bot.py create mode 100644 core/helper.py create mode 100644 core/main.py create mode 100644 core/parser.py create mode 100644 modules/keyword.py create mode 100644 modules/userinfo.py create mode 100644 utils/loaders/command_loader.py create mode 100644 utils/logging/send.py diff --git a/commands/__init__.py b/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/commands/add.py b/commands/add.py new file mode 100644 index 0000000..b003c10 --- /dev/null +++ b/commands/add.py @@ -0,0 +1,110 @@ +from core.main import * +import core.helper as helper + +class Add: + def __init__(self, register): + register("add", self.add) + + def add(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 6: + failure("Too many arguments") + return + + if length > 1: + name = spl[1] + else: + incUsage("add") + return + if length > 2: + host = spl[2] + if length > 3: + port = spl[3] + if length > 4: + protocol = spl[4] + if length > 5: + nickname = spl[5] + + toFail = False + if length < 6: + if config["Default"]["nickname"] == None: + failure("Choose a nickname, or configure one in defaults") + toFail = True + else: + nickname = config["Default"]["nickname"] + + if length < 5: + if config["Default"]["protocol"] == None: + failure("Choose a protocol, or configure one in defaults") + toFail = True + else: + protocol = config["Default"]["protocol"] + + if length < 4: + if config["Default"]["port"] == None: + failure("Choose a port, or configure one in defaults") + toFail = True + else: + port = config["Default"]["port"] + + if length < 3: + if config["Default"]["host"] == None: + failure("Choose a host, or configure one in defaults") + toFail = True + else: + host = config["Default"]["host"] + if toFail: + failure("Stopping due to previous error(s)") + return + + if length < 2: + incUsage("add") + return + + if name in pool.keys(): + failure("Name already exists: %s" % name) + return + + protocol = protocol.lower() + + if not protocol in ["ssl", "plain"]: + failure("Protocol must be ssl or plain, not %s" % protocol) + return + + try: + int(port) + except: + failure("Port must be an integer, not %s" % port) + return + + pool[name] = { "host": host, + "port": port, + "protocol": protocol, + "bind": config["Default"]["bind"], + "timeout": config["Default"]["timeout"], + "maxdelay": config["Default"]["maxdelay"], + "initialdelay": config["Default"]["initialdelay"], + "factor": config["Default"]["factor"], + "jitter": config["Default"]["jitter"], + "nickname": nickname, + "username": config["Default"]["username"], + "realname": config["Default"]["realname"], + "userinfo": config["Default"]["userinfo"], + "finger": config["Default"]["finger"], + "version": config["Default"]["version"], + "source": config["Default"]["source"], + "autojoin": config["Default"]["autojoin"], + "authtype": config["Default"]["authtype"], + "password": config["Default"]["password"], + "authentity": config["Default"]["authentity"], + "key": config["Default"]["key"], + "certificate": config["Default"]["certificate"], + "enabled": config["ConnectOnCreate"], + } + if config["ConnectOnCreate"] == True: + helper.addBot(name) + success("Successfully created bot") + saveConf("pool") + return + else: + incUsage(None) diff --git a/commands/default.py b/commands/default.py new file mode 100644 index 0000000..7e5f2a5 --- /dev/null +++ b/commands/default.py @@ -0,0 +1,80 @@ +from core.main import * + +class Default: + def __init__(self, register): + register("default", self.default) + + def default(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + toUnset = False + if length == 1: + optionMap = ["Viewing defaults"] + for i in config["Default"].keys(): + optionMap.append("%s: %s" % (i, config["Default"][i])) + info("\n".join(optionMap)) + return + elif length == 2: + if not spl[1] in config["Default"].keys(): + failure("No such key: %s" % spl[1]) + return + info("%s: %s" % (spl[1], config["Default"][spl[1]])) + return + elif length == 3: + if not spl[1] in config["Default"].keys(): + failure("No such key: %s" % spl[1]) + return + + if spl[2].lower() in ["none", "nil"]: + spl[2] = None + toUnset = True + + if spl[1] in ["port", "timeout", "maxdelay"]: + try: + spl[2] = int(spl[2]) + except: + failure("Value must be an integer, not %s" % spl[2]) + return + + if spl[2] in ["initialdelay", "factor", "jitter"]: + try: + spl[3] = float(spl[3]) + except: + failure("Value must be a floating point integer, not %s" % spl[3]) + return + + if spl[1] == "protocol": + if not toUnset: + if not spl[2] in ["ssl", "plain"]: + failure("Protocol must be ssl or plain, not %s" % spl[2]) + return + + if spl[2] == config["Default"][spl[1]]: + failure("Value already exists: %s" % spl[2]) + return + + if spl[1] == "authtype": + if not toUnset: + if not spl[2] in ["sp", "ns"]: + failure("Authtype must be sp or ns, not %s" % spl[2]) + return + if spl[1] == "enabled": + failure("Use the ConnectOnCreate config parameter to set this") + return + if spl[1] == "autojoin": + if not toUnset: + spl[2] = spl[2].split(",") + else: + spl[2] = [] + + config["Default"][spl[1]] = spl[2] + saveConf("config") + if toUnset: + success("Successfully unset key %s" % spl[1]) + else: + success("Successfully set key %s to %s" % (spl[1], spl[2])) + return + else: + incUsage("default") + return + else: + incUsage(None) diff --git a/commands/delete.py b/commands/delete.py new file mode 100644 index 0000000..f47958a --- /dev/null +++ b/commands/delete.py @@ -0,0 +1,29 @@ +from core.main import * + +class Delete: + def __init__(self, register): + register("del", self.delete) + + def delete(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + del pool[spl[1]] + if spl[1] in ReactorPool.keys(): + if spl[1] in FactoryPool.keys(): + FactoryPool[spl[1]].stopTrying() + ReactorPool[spl[1]].disconnect() + if spl[1] in IRCPool.keys(): + del IRCPool[spl[1]] + del ReactorPool[spl[1]] + del FactoryPool[spl[1]] + success("Successfully removed bot") + saveConf("pool") + return + else: + incUsage("del") + return + else: + incUsage(None) diff --git a/commands/disable.py b/commands/disable.py new file mode 100644 index 0000000..f5b8b39 --- /dev/null +++ b/commands/disable.py @@ -0,0 +1,29 @@ +from core.main import * + +class Disable: + def __init__(self, register): + register("disable", self.disable) + + def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + pool[spl[1]]["enabled"] = False + saveConf("pool") + if spl[1] in ReactorPool.keys(): + if spl[1] in FactoryPool.keys(): + FactoryPool[spl[1]].stopTrying() + ReactorPool[spl[1]].disconnect() + if spl[1] in IRCPool.keys(): + del IRCPool[spl[1]] + del ReactorPool[spl[1]] + del FactoryPool[spl[1]] + success("Successfully disabled bot %s" % spl[1]) + return + else: + incUsage("disable") + return + else: + incUsage(None) diff --git a/commands/dist.py b/commands/dist.py new file mode 100644 index 0000000..0d4c325 --- /dev/null +++ b/commands/dist.py @@ -0,0 +1,20 @@ +from core.main import * +from subprocess import run, PIPE + +class Dist: + def __init__(self, register): + register("dist", self.dist) + + def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if config["Dist"]["Enabled"]: + rtrn = run([config["Dist"]["File"]], shell=True, stdout=PIPE) + if config["Dist"]["SendOutput"]: + info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) + else: + info("Exit code: %s" % rtrn.returncode) + else: + failure("The dist command is not enabled") + return + else: + incUsage(None) diff --git a/commands/enable.py b/commands/enable.py new file mode 100644 index 0000000..43ffba1 --- /dev/null +++ b/commands/enable.py @@ -0,0 +1,26 @@ +from core.main import * +import core.helper as helper + +class Enable: + def __init__(self, register): + register("enable", self.enable) + + def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + pool[spl[1]]["enabled"] = True + saveConf("pool") + if not spl[1] in IRCPool.keys(): + helper.addBot(spl[1]) + else: + pass + success("Successfully enabled bot %s" % spl[1]) + return + else: + incUsage("enable") + return + else: + incUsage(None) diff --git a/commands/get.py b/commands/get.py new file mode 100644 index 0000000..1f44169 --- /dev/null +++ b/commands/get.py @@ -0,0 +1,22 @@ +from core.main import * + +class Get: + def __init__(self, register): + register("get", self.get) + + def get(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + info(str(IRCPool[spl[1]].get(spl[2]))) + return + else: + incUsage("get") + return + else: + incUsage(None) diff --git a/commands/help.py b/commands/help.py new file mode 100644 index 0000000..e07b828 --- /dev/null +++ b/commands/help.py @@ -0,0 +1,15 @@ +from core.main import * + +class Help: + def __init__(self, register): + register("help", self.help) + + def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + helpMap = [] + for i in help.keys(): + helpMap.append("%s: %s" % (i, help[i])) + info("\n".join(helpMap)) + return + else: + incUsage(None) diff --git a/commands/join.py b/commands/join.py new file mode 100644 index 0000000..b68fcfd --- /dev/null +++ b/commands/join.py @@ -0,0 +1,33 @@ +from core.main import * + +class Join: + def __init__(self, register): + register("join", self.join) + + def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + IRCPool[spl[1]].join(spl[2]) + success("Joined %s" % spl[2]) + return + elif length == 4: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + IRCPool[spl[1]].join(spl[2], spl[3]) + success("Joined %s with key %s" % (spl[2], spl[3])) + return + else: + incUsage("join") + return + else: + incUsage(None) diff --git a/commands/key.py b/commands/key.py new file mode 100644 index 0000000..e523097 --- /dev/null +++ b/commands/key.py @@ -0,0 +1,149 @@ +from core.main import * +import modules.keyword as keyword + +class Key: + def __init__(self, register): + register("key", self.key) + + def key(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if data.startswith("key add"): + if not data in ["key add ", "key add"] and data[3] == " ": + keywordsToAdd = data[8:] + keywords = keywordsToAdd.split(",") + for i in keywords: + rtrn = keyword.addKeyword(i) + if rtrn == "EXISTS": + failure("Keyword already exists: %s" % i) + elif rtrn == "ISIN": + failure("Keyword already matched: %s" % i) + elif rtrn == True: + success("Keyword added: %s" % i) + return + else: + incUsage("key") + return + elif data.startswith("key del"): + if not data in ["key del ", "key del"] and data[3] == " ": + keywordsToDel = data[8:] + keywords = keywordsToDel.split(",") + for i in keywords: + rtrn = keyword.delKeyword(i) + if rtrn == "NOKEY": + failure("Keyword does not exist: %s" % i) + elif rtrn == True: + success("Keyword deleted: %s" % i) + return + else: + incUsage("key") + return + if length == 4: + if spl[1] == "except": + if not spl[2] in keyconf["Keywords"]: + failure("No such keyword: %s" % spl[2]) + return + if spl[2] in keyconf["KeywordsExcept"].keys(): + if spl[3] in keyconf["KeywordsExcept"][spl[2]]: + failure("Exception exists: %s" % spl[3]) + return + else: + if not spl[2] in spl[3]: + failure("Keyword %s not in exception %s. This won't work" % (spl[2], spl[3])) + return + keyconf["KeywordsExcept"][spl[2]] = [] + + keyconf["KeywordsExcept"][spl[2]].append(spl[3]) + saveConf("keyconf") + success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) + return + elif spl[1] == "master": + if not spl[2] in pool.keys(): + failure("Name does not exist: %s" % spl[2]) + return + if spl[2] in IRCPool.keys(): + if not spl[3] in IRCPool[spl[2]].channels: + info("Bot not on channel: %s" % spl[3]) + config["Master"] = [spl[2], spl[3]] + saveConf("config") + success("Master set to %s on %s" % (spl[3], spl[2])) + return + elif spl[1] == "unexcept": + if not spl[2] in keyconf["KeywordsExcept"].keys(): + failure("No such exception: %s" % spl[2]) + return + if not spl[3] in keyconf["KeywordsExcept"][spl[2]]: + failure("Exception %s has no attribute %s" % (spl[2], spl[3])) + return + keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) + if keyconf["KeywordsExcept"][spl[2]] == []: + del keyconf["KeywordsExcept"][spl[2]] + saveConf("keyconf") + success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) + return + else: + incUsage("key") + return + elif length == 3: + if spl[1] == "unexcept": + if not spl[2] in keyconf["KeywordsExcept"].keys(): + failure("No such exception: %s" % spl[2]) + return + del keyconf["KeywordsExcept"][spl[2]] + saveConf("keyconf") + success("Successfully removed exception list of %s" % spl[2]) + return + elif spl[1] == "monitor": + if spl[2] == "on": + if not obj.addr in MonitorPool: + MonitorPool.append(obj.addr) + success("Keyword monitoring enabled") + return + else: + failure("Keyword monitoring is already enabled") + return + elif spl[2] == "off": + if obj.addr in MonitorPool: + MonitorPool.remove(obj.addr) + success("Keyword monitoring disabled") + return + else: + failure("Keyword monitoring is already disabled") + return + else: + incUsage("key") + return + else: + incUsage("key") + return + elif length == 2: + if spl[1] == "show": + info(",".join(keyconf["Keywords"])) + return + + elif spl[1] == "showexcept": + exceptMap = [] + for i in keyconf["KeywordsExcept"].keys(): + exceptMap.append("Key: %s" % i) + exceptMap.append("%s: %s" % (i, ",".join(keyconf["KeywordsExcept"][i]))) + exceptMap.append("\n") + info("\n".join(exceptMap)) + return + elif spl[1] == "master": + info(" - ".join(config["Master"])) + return + elif spl[1] == "monitor": + if obj.addr in MonitorPool: + info("Keyword monitoring is enabled on this connection") + return + else: + info("Keyword monitoring is disabled on this connection") + return + else: + incUsage("key") + return + + else: + incUsage("key") + return + else: + incUsage(None) diff --git a/commands/list.py b/commands/list.py new file mode 100644 index 0000000..e39ee95 --- /dev/null +++ b/commands/list.py @@ -0,0 +1,18 @@ +from core.main import * + +class List: + def __init__(self, register): + register("list", self.list) + + def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + poolMap = [] + for i in pool.keys(): + poolMap.append("Server: %s" % i) + for x in pool[i].keys(): + poolMap.append("%s: %s" % (x, pool[i][x])) + poolMap.append("\n") + info("\n".join(poolMap)) + return + else: + incUsage(None) diff --git a/commands/load.py b/commands/load.py new file mode 100644 index 0000000..ccb1428 --- /dev/null +++ b/commands/load.py @@ -0,0 +1,26 @@ +from core.main import * + +class Load: + def __init__(self, register): + register("load", self.load) + + def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if spl[1] in filemap.keys(): + loadConf(spl[1]) + success("Loaded %s from %s" % (spl[1], filemap[spl[1]][0])) + return + elif spl[1] == "all": + for i in filemap.keys(): + loadConf(i) + success("Loaded %s from %s" % (i, filemap[i][0])) + return + else: + incUsage("load") + return + else: + incUsage("load") + return + else: + incUsage(None) diff --git a/commands/logout.py b/commands/logout.py new file mode 100644 index 0000000..f95a256 --- /dev/null +++ b/commands/logout.py @@ -0,0 +1,15 @@ +from core.main import * + +class Logout: + def __init__(self, register): + register("logout", self.logout) + + def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + obj.authed = False + if obj.addr in MonitorPool: + MonitorPool.remove(obj.addr) + success("Logged out") + return + else: + incUsage(None) diff --git a/commands/mod.py b/commands/mod.py new file mode 100644 index 0000000..8ffd9ad --- /dev/null +++ b/commands/mod.py @@ -0,0 +1,89 @@ +from core.main import * + +class Mod: + def __init__(self, register): + register("mod", self.mod) + + def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + toUnset = False + if length == 2: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + optionMap = ["Viewing options for %s" % spl[1]] + for i in pool[spl[1]].keys(): + optionMap.append("%s: %s" % (i, pool[spl[1]][i])) + info("\n".join(optionMap)) + return + + elif length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[2] in pool[spl[1]].keys(): + failure("No such key: %s" % spl[2]) + return + info("%s: %s" % (spl[2], pool[spl[1]][spl[2]])) + return + + elif length == 4: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[2] in pool[spl[1]].keys(): + failure("No such key: %s" % spl[2]) + return + + if spl[2] == "protocol": + if not spl[3] in ["ssl", "plain"]: + failure("Protocol must be ssl or plain, not %s" % spl[3]) + return + if spl[3] == pool[spl[1]][spl[2]]: + failure("Value already exists: %s" % spl[3]) + return + + if spl[3].lower() in ["none", "nil"]: + spl[3] = None + toUnset = True + + if spl[2] in ["port", "timeout", "maxdelay"]: + try: + spl[3] = int(spl[3]) + except: + failure("Value must be an integer, not %s" % spl[3]) + return + + if spl[2] in ["initialdelay", "factor", "jitter"]: + try: + spl[3] = float(spl[3]) + except: + failure("Value must be a floating point integer, not %s" % spl[3]) + return + + if spl[2] == "authtype": + if not toUnset: + if not spl[3] in ["sp", "ns"]: + failure("Authtype must be sp or ns, not %s" % spl[3]) + return + if spl[2] == "enabled": + failure("Use the enable and disable commands to manage this") + return + if spl[2] == "autojoin": + spl[3] = spl[3].split(",") + + pool[spl[1]][spl[2]] = spl[3] + if spl[1] in IRCPool.keys(): + IRCPool[spl[1]].refresh() + saveConf("pool") + if toUnset: + success("Successfully unset key %s on %s" % (spl[2], spl[1])) + else: + success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) + return + + else: + incUsage("mod") + return + else: + incUsage(None) diff --git a/commands/part.py b/commands/part.py new file mode 100644 index 0000000..0b145a9 --- /dev/null +++ b/commands/part.py @@ -0,0 +1,23 @@ +from core.main import * + +class Part: + def __init__(self, register): + register("part", self.part) + + def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 3: + if not spl[1] in pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + IRCPool[spl[1]].part(spl[2]) + success("Left %s" % spl[2]) + return + else: + incUsage("part") + return + else: + incUsage(None) diff --git a/commands/password.py b/commands/password.py new file mode 100644 index 0000000..b4b449e --- /dev/null +++ b/commands/password.py @@ -0,0 +1,22 @@ +from core.main import * + +class Password: + def __init__(self, register): + register("pass", self.password) + + def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + info("You are already authenticated") + return + else: + if length == 2: + if spl[1] == config["Password"]: + success("Authenticated successfully") + obj.authed = True + return + else: + failure("Password incorrect") + obj.transport.loseConnection() + return + else: + incUsage("pass") diff --git a/commands/save.py b/commands/save.py new file mode 100644 index 0000000..64c4cbf --- /dev/null +++ b/commands/save.py @@ -0,0 +1,26 @@ +from core.main import * + +class Save: + def __init__(self, register): + register("save", self.save) + + def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if spl[1] in filemap.keys(): + saveConf(spl[1]) + success("Saved %s to %s" % (spl[1], filemap[spl[1]][0])) + return + elif spl[1] == "all": + for i in filemap.keys(): + saveConf(i) + success("Saved %s from %s" % (i, filemap[i][0])) + return + else: + incUsage("save") + return + else: + incUsage("save") + return + else: + incUsage(None) diff --git a/commands/stats.py b/commands/stats.py new file mode 100644 index 0000000..9b1f76f --- /dev/null +++ b/commands/stats.py @@ -0,0 +1,49 @@ +from core.main import * + +class Stats: + def __init__(self, register): + register("stats", self.stats) + + def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 1: + stats = [] + numChannels = 0 + numWhoEntries = 0 + for i in IRCPool.keys(): + numChannels += len(IRCPool[i].channels) + for i in wholist.keys(): + numWhoEntries += len(wholist[i].keys()) + stats.append("Servers: %s" % len(IRCPool.keys())) + stats.append("Channels: %s" % numChannels) + stats.append("User records: %s" % numWhoEntries) + info("\n".join(stats)) + return + + elif length == 2: + stats = [] + numChannels = 0 + numWhoEntries = 0 + + failures = 0 + if spl[1] in IRCPool.keys(): + numChannels += len(IRCPool[spl[1]].channels) + else: + failure("Name does not exist: %s" % spl[1]) + failures += 1 + if spl[1] in wholist.keys(): + numWhoEntries += len(wholist[spl[1]].keys()) + else: + failure("Who entry does not exist: %s" % spl[1]) + failures += 1 + if failures == 2: + failure("No information found, aborting") + return + stats.append("Channels: %s" % numChannels) + stats.append("User records: %s" % numWhoEntries) + info("\n".join(stats)) + return + else: + incUsage("stats") + else: + incUsage(None) diff --git a/commands/who.py b/commands/who.py new file mode 100644 index 0000000..9a8256b --- /dev/null +++ b/commands/who.py @@ -0,0 +1,26 @@ +from core.main import * +import modules.userinfo as userinfo + +class Who: + def __init__(self, register): + register("who", self.who) + + def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + result = userinfo.getWho(spl[1]) + rtrn = "" + for i in result.keys(): + rtrn += "Matches from: %s" % i + rtrn += "\n" + for x in result[i]: + x = [y for y in x if not y == None] + rtrn += str((", ".join(x))) + rtrn += "\n" + info(rtrn) + return + else: + incUsage("who") + return + else: + incUsage(None) diff --git a/core/bot.py b/core/bot.py new file mode 100644 index 0000000..ec0894c --- /dev/null +++ b/core/bot.py @@ -0,0 +1,240 @@ +from twisted.internet.protocol import ReconnectingClientFactory +from twisted.words.protocols.irc import IRCClient +from twisted.internet.defer import Deferred + +import modules.keyword as keyword +import modules.userinfo as userinfo + +from core.main import * +from utils.logging.log import * +from utils.logging.send import * + +class IRCBot(IRCClient): + def __init__(self, name): + self.connected = False + self.channels = [] + + self.name = name + instance = pool[name] + + self.nickname = instance["nickname"] + self.realname = instance["realname"] + self.username = instance["username"] + self.userinfo = instance["userinfo"] + self.fingerReply = instance["finger"] + self.versionName = instance["version"] + self.versionNum = None + self.versionEnv = None + self.sourceURL = instance["source"] + self.autojoin = instance["autojoin"] + + self._who = {} + self._getWho = {} + + self.authtype = instance["authtype"] + if self.authtype == "ns": + self.authpass = instance["password"] + self.authentity = instance["authentity"] + else: + self.password = instance["password"] + + def refresh(self): + instance = pool[self.name] + if not instance["nickname"] == self.nickname: + self.nickname = instance["nickname"] + self.setNick(self.nickname) + + self.userinfo = instance["userinfo"] + self.fingerReply = instance["finger"] + self.versionName = instance["version"] + self.versionNum = None + self.versionEnv = None + self.sourceURL = instance["source"] + + def parsen(self, user): + step = user.split("!") + nick = step[0] + if len(step) == 2: + step2 = step[1].split("@") + ident, host = step2 + else: + ident = nick + host = nick + + return [nick, ident, host] + + def privmsg(self, user, channel, msg): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + keyword.actKeyword(user, channel, msg, self.nickname, "PRV", self.name) + + def noticed(self, user, channel, msg): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + keyword.actKeyword(user, channel, msg, self.nickname, "NOT", self.name) + + def action(self, user, channel, msg): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + keyword.actKeyword(user, channel, msg, self.nickname, "ACT", self.name) + + def get(self, var): + try: + result = getattr(self, var) + except AttributeError: + result = None + return result + + def setNick(self, nickname): + self._attemptedNick = nickname + self.sendLine("NICK %s" % nickname) + self.nickname = nickname + + def alterCollidedNick(self, nickname): + newnick = nickname + "_" + return newnick + + def irc_ERR_NICKNAMEINUSE(self, prefix, params): + self._attemptedNick = self.alterCollidedNick(self._attemptedNick) + self.setNick(self._attemptedNick) + + def irc_ERR_PASSWDMISMATCH(self, prefix, params): + log("%s: password mismatch" % self.name) + sendAll("%s: password mismatch" % self.name) + + def who(self, channel): + channel = channel + d = Deferred() + if channel not in self._who: + self._who[channel] = ([], {}) + self._who[channel][0].append(d) + self.sendLine("WHO %s" % channel) + return d + + def irc_RPL_WHOREPLY(self, prefix, params): + channel = params[1] + user = params[2] + host = params[3] + server = params[4] + nick = params[5] + status = params[6] + realname = params[7] + if channel not in self._who: + return + n = self._who[channel][1] + n[nick] = [nick, user, host, server, status, realname] + + def irc_RPL_ENDOFWHO(self, prefix, params): + channel = params[1] + if channel not in self._who: + return + callbacks, info = self._who[channel] + for cb in callbacks: + cb.callback((channel, info)) + del self._who[channel] + + def got_who(self, whoinfo): + userinfo.setWho(self.name, whoinfo[1]) + + def signedOn(self): + self.connected = True + log("signed on: %s" % self.name) + if config["ConnectionNotifications"]: + keyword.sendMaster("SIGNON: %s" % self.name) + if self.authtype == "ns": + self.msg(self.authentity, "IDENTIFY %s" % self.nspass) + for i in self.autojoin: + self.join(i) + + def joined(self, channel): + if not channel in self.channels: + self.channels.append(channel) + self.who(channel).addCallback(self.got_who) + + def left(self, channel): + if channel in self.channels: + self.channels.remove(channel) + + def kickedFrom(self, channel, kicker, message): + if channel in self.channels: + self.channels.remove(channel) + keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) + + def userJoined(self, user, channel): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + def userLeft(self, user, channel): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + def userQuit(self, user, quitMessage): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + keyword.actKeyword(user, None, quitMessage, self.nickname, "QUT", self.name) + + def userKicked(self, kickee, channel, kicker, message): + nick, ident, host = self.parsen(kicker) + userinfo.setWhoSingle(self.name, nick, ident, host) + + keyword.actKeyword(kicker, channel, message, self.nickname, "KCK", self.name) + + def userRenamed(self, oldname, newname): + nick, ident, host = self.parsen(oldname) + userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.name, newname, ident, host) + + def topicUpdated(self, user, channel, newTopic): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + + keyword.actKeyword(user, channel, newTopic, self.nickname, "TOP", self.name) + + def modeChanged(self, user, channel, toset, modes, args): + nick, ident, host = self.parsen(user) + userinfo.setWhoSingle(self.name, nick, ident, host) + +class IRCBotFactory(ReconnectingClientFactory): + def __init__(self, name): + self.instance = pool[name] + self.name = name + self.client = None + self.maxDelay = self.instance["maxdelay"] + self.initialDelay = self.instance["initialdelay"] + self.factor = self.instance["factor"] + self.jitter = self.instance["jitter"] + + def buildProtocol(self, addr): + entry = IRCBot(self.name) + IRCPool[self.name] = entry + self.client = entry + return entry + + def clientConnectionLost(self, connector, reason): + if not self.client == None: + self.client.connected = False + self.client.channels = [] + error = reason.getErrorMessage() + log("%s: connection lost: %s" % (self.name, error)) + sendAll("%s: connection lost: %s" % (self.name, error)) + if config["ConnectionNotifications"]: + keyword.sendMaster("CONNLOST %s: %s" % (self.name, error)) + self.retry(connector) + #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) + + def clientConnectionFailed(self, connector, reason): + if not self.client == None: + self.client.connected = False + self.client.channels = [] + error = reason.getErrorMessage() + log("%s: connection failed: %s" % (self.name, error)) + sendAll("%s: connection failed: %s" % (self.name, error)) + if config["ConnectionNotifications"]: + keyword.sendMaster("CONNFAIL %s: %s" % (self.name, error)) + self.retry(connector) + #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) + diff --git a/core/helper.py b/core/helper.py new file mode 100644 index 0000000..06a4c80 --- /dev/null +++ b/core/helper.py @@ -0,0 +1,44 @@ +from twisted.internet import reactor +from twisted.internet.ssl import DefaultOpenSSLContextFactory + +from core.bot import IRCBot, IRCBotFactory +from core.main import * +from utils.logging.log import * + +def addBot(name): + instance = pool[name] + log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) + if instance["protocol"] == "plain": + if instance["bind"] == None: + bot = IRCBotFactory(name) + rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"])) + + ReactorPool[name] = rct + FactoryPool[name] = bot + return + else: + bot = IRCBotFactory(name) + rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) + + ReactorPool[name] = rct + FactoryPool[name] = bot + return + elif instance["protocol"] == "ssl": + keyFN = certPath+instance["key"] + certFN = certPath+instance["certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) + if instance["bind"] == None: + bot = IRCBotFactory(name) + rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory) + + ReactorPool[name] = rct + FactoryPool[name] = bot + return + else: + + bot = IRCBotFactory(name) + rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory, bindAddress=instance["bind"]) + + ReactorPool[name] = rct + FactoryPool[name] = bot + return diff --git a/core/main.py b/core/main.py new file mode 100644 index 0000000..c732c09 --- /dev/null +++ b/core/main.py @@ -0,0 +1,50 @@ +from json import load, dump, loads +from utils.loaders.command_loader import loadCommands +from utils.logging.log import * + +configPath = "conf/" +certPath = "cert/" + +filemap = { + "config": ["config.json", "configuration"], + "keyconf": ["keyword.json", "keyword lists"], + "pool": ["pool.json", "pool"], + "help": ["help.json", "command help"], + "wholist": ["wholist.json", "WHO lists"], +} + +numbers = "0123456789" + +listener = None +connections = {} +IRCPool = {} +ReactorPool = {} +FactoryPool = {} + +MonitorPool = [] + +CommandMap = {} + +def register(command, function): + if not command in CommandMap: + CommandMap[command] = function + log("Registering command %s" % command) + else: + error("Duplicate command: %s" % (command)) + +def saveConf(var): + with open(configPath+filemap[var][0], "w") as f: + dump(globals()[var], f, indent=4) + return + +def loadConf(var): + with open(configPath+filemap[var][0], "r") as f: + globals()[var] = load(f) + +def initConf(): + for i in filemap.keys(): + loadConf(i) + +def initMain(): + initConf() + loadCommands(register) diff --git a/core/parser.py b/core/parser.py new file mode 100644 index 0000000..2845829 --- /dev/null +++ b/core/parser.py @@ -0,0 +1,30 @@ +from core.main import * +from utils.logging.log import * +from utils.logging.send import * + +def parseCommand(addr, authed, data): + #call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length) + spl = data.split() + if addr in connections.keys(): + obj = connections[addr] + else: + warn("Got connection object with no instance in the address pool") + return + + success = lambda data: sendSuccess(addr, data) + failure = lambda data: sendFailure(addr, data) + info = lambda data: sendInfo(addr, data) + + incUsage = lambda mode: incorrectUsage(addr, mode) + length = len(spl) + if len(spl) > 0: + cmd = spl[0] + else: + failure("No text was sent") + return + for i in CommandMap.keys(): + if data.startswith(i): + CommandMap[i](addr, authed, data, obj, spl, success, failure, info, incUsage, length) + return + incUsage(None) + return diff --git a/modules/keyword.py b/modules/keyword.py new file mode 100644 index 0000000..43547f6 --- /dev/null +++ b/modules/keyword.py @@ -0,0 +1,74 @@ +from core.main import * +from utils.logging.log import * + +def sendMaster(data): + if config["Master"][0] in IRCPool.keys(): + IRCPool[config["Master"][0]].msg(config["Master"][1], data) + else: + warn("Master with no IRC instance defined") + for i in MonitorPool: + connections[i].send(data) + +def isKeyword(msg): + message = msg.lower() + messageDuplicate = message + toUndo = False + uniqueNum = 0 + totalNum = 0 + for i in keyconf["Keywords"]: + if i in message: + if i in keyconf["KeywordsExcept"].keys(): + for x in keyconf["KeywordsExcept"][i]: + if x in message: + toUndo = True + messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") + for y in keyconf["Keywords"]: + if i in messageDuplicate: + totalNum += messageDuplicate.count(i) + message = messageDuplicate.replace(i, "{"+i+"}") + message = message.replace("\0\r\n\n\0", x) + uniqueNum += 1 + + if toUndo == False: + totalNum += message.count(i) + message = message.replace(i, "{"+i+"}") + uniqueNum += 1 + + toUndo = False + + if totalNum == 0: + return False + else: + return [message, uniqueNum, totalNum] + +def actKeyword(user, channel, message, nickname, actType, name): + toSend = isKeyword(message) + if name == config["Master"][0] and channel == config["Master"][1]: + pass + else: + if config["HighlightNotifications"]: + msgLower = message.lower() + nickLower = nickname.lower() + if nickLower in msgLower: + msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + sendMaster("NICK %s %s (T:%s): (%s/%s) %s" % (actType, name, msgLower.count(nickLower), user, channel, msgLower)) + if toSend: + sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) + +def addKeyword(keyword): + if keyword in keyconf["Keywords"]: + return "EXISTS" + else: + for i in keyconf["Keywords"]: + if i in keyword or keyword in i: + return "ISIN" + keyconf["Keywords"].append(keyword) + saveConf("keyconf") + return True + +def delKeyword(keyword): + if not keyword in keyconf["Keywords"]: + return "NOKEY" + keyconf["Keywords"].remove(keyword) + saveConf("keyconf") + return True diff --git a/modules/userinfo.py b/modules/userinfo.py new file mode 100644 index 0000000..f0403fe --- /dev/null +++ b/modules/userinfo.py @@ -0,0 +1,31 @@ +from core.main import * +#from utils.logging.log import * + +def setWho(network, newObjects): + network = "".join([x for x in network if not x in numbers]) + if not network in wholist.keys(): + wholist[network] = {} + for i in newObjects.keys(): + wholist[network][i] = newObjects[i] + + return + +def setWhoSingle(network, nick, ident, host): + network = "".join([x for x in network if not x in numbers]) + + if network in wholist.keys(): + if nick in wholist[network].keys(): + wholist[network][nick][1] = ident + wholist[network][nick][2] = host + else: + wholist[network][nick] = [nick, ident, host, None, None, None] + +def getWho(nick): + result = {} + for i in wholist.keys(): + for x in wholist[i].keys(): + if nick.lower() == x.lower(): + if not i in result.keys(): + result[i] = [] + result[i].append(wholist[i][x]) + return result diff --git a/threshold b/threshold index bc41686..9524f5a 100755 --- a/threshold +++ b/threshold @@ -1,283 +1,40 @@ #!/usr/bin/env python from twisted.internet import reactor -from twisted.internet.defer import Deferred from twisted.internet.ssl import DefaultOpenSSLContextFactory -from twisted.internet.protocol import Protocol, Factory, ClientFactory, ReconnectingClientFactory -from twisted.words.protocols.irc import IRCClient +from twisted.internet.protocol import Protocol, Factory, ClientFactory #from twisted.python import log #from sys import stdout #log.startLogging(stdout) -from json import load, dump, loads -from sys import exit -from subprocess import run, PIPE +from core.main import ( + numbers, + configPath, + certPath, + listener, + connections, + IRCPool, + ReactorPool, + FactoryPool, + MonitorPool, + saveConf, + loadConf, + initMain, +) +initMain() + +from core.main import ( + config, + keyconf, + pool, + help, + wholist, +) from utils.logging.log import * - -numbers = "0123456789" - -listener = None -connections = {} -IRCPool = {} -ReactorPool = {} -FactoryPool = {} - -MonitorPool = [] - -configPath = "conf/" -certPath = "cert/" - -filemap = { - "config": ["config.json", "configuration"], - "keyconf": ["keyword.json", "keyword lists"], - "pool": ["pool.json", "pool"], - "help": ["help.json", "command help"], - "wholist": ["wholist.json", "WHO lists"], -} - -def sendData(addr, data): - connections[addr].send(data) - -def sendSuccess(addr, data): - sendData(addr, "[y] " + data) - -def sendFailure(addr, data): - sendData(addr, "[n] " + data) - -def sendInfo(addr, data): - sendData(addr, "[i] " + data) - -class IRCBot(IRCClient): - def __init__(self, name): - self.connected = False - self.channels = [] - - self.name = name - instance = pool[name] - - self.nickname = instance["nickname"] - self.realname = instance["realname"] - self.username = instance["username"] - self.userinfo = instance["userinfo"] - self.fingerReply = instance["finger"] - self.versionName = instance["version"] - self.versionNum = None - self.versionEnv = None - self.sourceURL = instance["source"] - self.autojoin = instance["autojoin"] - - self._who = {} - self._getWho = {} - - self.authtype = instance["authtype"] - if self.authtype == "ns": - self.authpass = instance["password"] - self.authentity = instance["authentity"] - else: - self.password = instance["password"] - - def refresh(self): - instance = pool[self.name] - if not instance["nickname"] == self.nickname: - self.nickname = instance["nickname"] - self.setNick(self.nickname) - - self.userinfo = instance["userinfo"] - self.fingerReply = instance["finger"] - self.versionName = instance["version"] - self.versionNum = None - self.versionEnv = None - self.sourceURL = instance["source"] - - def parsen(self, user): - step = user.split("!") - nick = step[0] - if len(step) == 2: - step2 = step[1].split("@") - ident, host = step2 - else: - ident = nick - host = nick - - return [nick, ident, host] - - def privmsg(self, user, channel, msg): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - helper.actKeyword(user, channel, msg, self.nickname, "PRV", self.name) - - def noticed(self, user, channel, msg): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - helper.actKeyword(user, channel, msg, self.nickname, "NOT", self.name) - - def action(self, user, channel, msg): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - helper.actKeyword(user, channel, msg, self.nickname, "ACT", self.name) - - def get(self, var): - try: - result = getattr(self, var) - except AttributeError: - result = None - return result - - def setNick(self, nickname): - self._attemptedNick = nickname - self.sendLine("NICK %s" % nickname) - self.nickname = nickname - - def alterCollidedNick(self, nickname): - newnick = nickname + "_" - return newnick - - def irc_ERR_NICKNAMEINUSE(self, prefix, params): - self._attemptedNick = self.alterCollidedNick(self._attemptedNick) - self.setNick(self._attemptedNick) - - def irc_ERR_PASSWDMISMATCH(self, prefix, params): - log("%s: password mismatch" % self.name) - helper.sendAll("%s: password mismatch" % self.name) - - def who(self, channel): - channel = channel - d = Deferred() - if channel not in self._who: - self._who[channel] = ([], {}) - self._who[channel][0].append(d) - self.sendLine("WHO %s" % channel) - return d - - def irc_RPL_WHOREPLY(self, prefix, params): - channel = params[1] - user = params[2] - host = params[3] - server = params[4] - nick = params[5] - status = params[6] - realname = params[7] - if channel not in self._who: - return - n = self._who[channel][1] - n[nick] = [nick, user, host, server, status, realname] - - def irc_RPL_ENDOFWHO(self, prefix, params): - channel = params[1] - if channel not in self._who: - return - callbacks, info = self._who[channel] - for cb in callbacks: - cb.callback((channel, info)) - del self._who[channel] - - def got_who(self, whoinfo): - global wholist - helper.setWho(self.name, whoinfo[1]) - - def signedOn(self): - self.connected = True - log("signed on: %s" % self.name) - if config["ConnectionNotifications"]: - helper.sendMaster("SIGNON: %s" % self.name) - if self.authtype == "ns": - self.msg(self.authentity, "IDENTIFY %s" % self.nspass) - for i in self.autojoin: - self.join(i) - - def joined(self, channel): - if not channel in self.channels: - self.channels.append(channel) - self.who(channel).addCallback(self.got_who) - - def left(self, channel): - if channel in self.channels: - self.channels.remove(channel) - - def kickedFrom(self, channel, kicker, message): - if channel in self.channels: - self.channels.remove(channel) - helper.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) - - def userJoined(self, user, channel): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - def userLeft(self, user, channel): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - def userQuit(self, user, quitMessage): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - helper.actKeyword(user, None, quitMessage, self.nickname, "QUT", self.name) - - def userKicked(self, kickee, channel, kicker, message): - nick, ident, host = self.parsen(kicker) - helper.setWhoSingle(self.name, nick, ident, host) - - helper.actKeyword(kicker, channel, message, self.nickname, "KCK", self.name) - - def userRenamed(self, oldname, newname): - nick, ident, host = self.parsen(oldname) - helper.setWhoSingle(self.name, nick, ident, host) - helper.setWhoSingle(self.name, newname, ident, host) - - def topicUpdated(self, user, channel, newTopic): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - - helper.actKeyword(user, channel, newTopic, self.nickname, "TOP", self.name) - - def modeChanged(self, user, channel, toset, modes, args): - nick, ident, host = self.parsen(user) - helper.setWhoSingle(self.name, nick, ident, host) - -class IRCBotFactory(ReconnectingClientFactory): - def __init__(self, name): - self.instance = pool[name] - self.name = name - self.client = None - self.maxDelay = self.instance["maxdelay"] - self.initialDelay = self.instance["initialdelay"] - self.factor = self.instance["factor"] - self.jitter = self.instance["jitter"] - - def buildProtocol(self, addr): - global IRCPool - entry = IRCBot(self.name) - IRCPool[self.name] = entry - self.client = entry - return entry - - def clientConnectionLost(self, connector, reason): - if not self.client == None: - self.client.connected = False - self.client.channels = [] - error = reason.getErrorMessage() - log("%s: connection lost: %s" % (self.name, error)) - helper.sendAll("%s: connection lost: %s" % (self.name, error)) - if config["ConnectionNotifications"]: - helper.sendMaster("CONNLOST %s: %s" % (self.name, error)) - self.retry(connector) - #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) - - def clientConnectionFailed(self, connector, reason): - if not self.client == None: - self.client.connected = False - self.client.channels = [] - error = reason.getErrorMessage() - log("%s: connection failed: %s" % (self.name, error)) - helper.sendAll("%s: connection failed: %s" % (self.name, error)) - if config["ConnectionNotifications"]: - helper.sendMaster("CONNFAIL %s: %s" % (self.name, error)) - self.retry(connector) - #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) +import modules.userinfo as userinfo +import core.helper as helper +from core.parser import parseCommand class Base(Protocol): def __init__(self, addr): @@ -298,16 +55,15 @@ class Base(Protocol): splitData = [x for x in data.split("\n") if x] if "\n" in data: for i in splitData: - helper.parseCommand(self.addr, self.authed, i) + parseCommand(self.addr, self.authed, i) return - helper.parseCommand(self.addr, self.authed, data) + parseCommand(self.addr, self.authed, data) def connectionMade(self): log("Connection from %s:%s" % (self.addr.host, self.addr.port)) self.send("Hello.") def connectionLost(self, reason): - global connections, MonitorPool self.authed = False log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) if not listener == None: @@ -322,867 +78,18 @@ class Base(Protocol): class BaseFactory(Factory): def buildProtocol(self, addr): - global connections entry = Base(addr) connections[addr] = entry return entry def send(self, addr, data): - global connections if addr in connections.keys(): connection = connections[addr] connection.send(data) else: return -class Helper(object): - def setWho(self, network, newObjects): - global wholist - network = "".join([x for x in network if not x in numbers]) - if not network in wholist.keys(): - wholist[network] = {} - for i in newObjects.keys(): - wholist[network][i] = newObjects[i] - - return - - def setWhoSingle(self, network, nick, ident, host): - global wholist - network = "".join([x for x in network if not x in numbers]) - - if network in wholist.keys(): - if nick in wholist[network].keys(): - wholist[network][nick][1] = ident - wholist[network][nick][2] = host - else: - wholist[network][nick] = [nick, ident, host, None, None, None] - - def getWho(self, nick): - result = {} - for i in wholist.keys(): - for x in wholist[i].keys(): - if nick.lower() == x.lower(): - if not i in result.keys(): - result[i] = [] - result[i].append(wholist[i][x]) - return result - - def save(self, var): - with open(configPath+filemap[var][0], "w") as f: - dump(globals()[var], f, indent=4) - return - - def load(self, var): - with open(configPath+filemap[var][0], "r") as f: - globals()[var] = load(f) - - def incorrectUsage(self, addr, mode): - if mode == None: - sendFailure(addr, "Incorrect usage") - return - if mode in help.keys(): - sendFailure(addr, "Usage: " + help[mode]) - return - - def sendAll(self, data): - global connections - for i in connections: - connections[i].send(data) - return - - def sendMaster(self, data): - if config["Master"][0] in IRCPool.keys(): - IRCPool[config["Master"][0]].msg(config["Master"][1], data) - else: - warn("Master with no IRC instance defined") - for i in MonitorPool: - connections[i].send(data) - - def isKeyword(self, msg): - message = msg.lower() - messageDuplicate = message - toUndo = False - uniqueNum = 0 - totalNum = 0 - for i in keyconf["Keywords"]: - if i in message: - if i in keyconf["KeywordsExcept"].keys(): - for x in keyconf["KeywordsExcept"][i]: - if x in message: - toUndo = True - messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") - for y in keyconf["Keywords"]: - if i in messageDuplicate: - totalNum += messageDuplicate.count(i) - message = messageDuplicate.replace(i, "{"+i+"}") - message = message.replace("\0\r\n\n\0", x) - uniqueNum += 1 - - if toUndo == False: - totalNum += message.count(i) - message = message.replace(i, "{"+i+"}") - uniqueNum += 1 - - toUndo = False - - if totalNum == 0: - return False - else: - return [message, uniqueNum, totalNum] - - def actKeyword(self, user, channel, message, nickname, actType, name): - toSend = self.isKeyword(message) - if name == config["Master"][0] and channel == config["Master"][1]: - pass - else: - if config["HighlightNotifications"]: - msgLower = message.lower() - nickLower = nickname.lower() - if nickLower in msgLower: - msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") - self.sendMaster("NICK %s %s (T:%s): (%s/%s) %s" % (actType, name, msgLower.count(nickLower), user, channel, msgLower)) - if toSend: - helper.sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) - - def addBot(self, name): - global IRCPool, certPath - instance = pool[name] - log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) - if instance["protocol"] == "plain": - if instance["bind"] == None: - bot = IRCBotFactory(name) - rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"])) - - ReactorPool[name] = rct - FactoryPool[name] = bot - return - else: - bot = IRCBotFactory(name) - rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) - - ReactorPool[name] = rct - FactoryPool[name] = bot - return - elif instance["protocol"] == "ssl": - keyFN = certPath+instance["key"] - certFN = certPath+instance["certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - if instance["bind"] == None: - bot = IRCBotFactory(name) - rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory) - - ReactorPool[name] = rct - FactoryPool[name] = bot - return - else: - - bot = IRCBotFactory(name) - rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory, bindAddress=instance["bind"]) - - ReactorPool[name] = rct - FactoryPool[name] = bot - return - - def addKeyword(self, keyword): - if keyword in keyconf["Keywords"]: - return "EXISTS" - else: - for i in keyconf["Keywords"]: - if i in keyword or keyword in i: - return "ISIN" - keyconf["Keywords"].append(keyword) - self.save("keyconf") - return True - - def delKeyword(self, keyword): - if not keyword in keyconf["Keywords"]: - return "NOKEY" - keyconf["Keywords"].remove(keyword) - self.save("keyconf") - return True - - def parseCommand(self, addr, authed, data): - global pool, MonitorPool, wholist - spl = data.split() - if addr in connections.keys(): - obj = connections[addr] - else: - warn("Got connection object with no instance in the address pool") - return - - success = lambda data: sendSuccess(addr, data) - failure = lambda data: sendFailure(addr, data) - info = lambda data: sendInfo(addr, data) - - incUsage = lambda mode: self.incorrectUsage(addr, mode) - length = len(spl) - if len(spl) > 0: - cmd = spl[0] - else: - failure("No text was sent") - return - - if authed == True: - if cmd == "help": - helpMap = [] - for i in help.keys(): - helpMap.append("%s: %s" % (i, help[i])) - info("\n".join(helpMap)) - return - - elif cmd == "save": - if length == 2: - if spl[1] in filemap.keys(): - self.save(spl[1]) - success("Saved %s to %s" % (spl[1], filemap[spl[1]][0])) - return - elif spl[1] == "all": - for i in filemap.keys(): - helper.save(i) - success("Saved %s from %s" % (i, filemap[i][0])) - return - else: - incUsage("save") - return - else: - incUsage("save") - return - - elif cmd == "load": - if length == 2: - if spl[1] in filemap.keys(): - self.load(spl[1]) - success("Loaded %s from %s" % (spl[1], filemap[spl[1]][0])) - return - elif spl[1] == "all": - for i in filemap.keys(): - helper.load(i) - success("Loaded %s from %s" % (i, filemap[i][0])) - return - else: - incUsage("load") - return - else: - incUsage("load") - return - - elif cmd == "dist": - if config["Dist"]["Enabled"]: - rtrn = run([config["Dist"]["File"]], shell=True, stdout=PIPE) - if config["Dist"]["SendOutput"]: - info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) - else: - info("Exit code: %s" % rtrn.returncode) - else: - failure("The dist command is not enabled") - return - - elif cmd == "pass": - info("You are already authenticated") - return - - elif cmd == "logout": - obj.authed = False - success("Logged out") - return - - elif cmd == "list": - poolMap = [] - for i in pool.keys(): - poolMap.append("Server: %s" % i) - for x in pool[i].keys(): - poolMap.append("%s: %s" % (x, pool[i][x])) - poolMap.append("\n") - info("\n".join(poolMap)) - return - - elif cmd == "stats": - if length == 1: - stats = [] - numChannels = 0 - numWhoEntries = 0 - for i in IRCPool.keys(): - numChannels += len(IRCPool[i].channels) - for i in wholist.keys(): - numWhoEntries += len(wholist[i].keys()) - stats.append("Servers: %s" % len(IRCPool.keys())) - stats.append("Channels: %s" % numChannels) - stats.append("User records: %s" % numWhoEntries) - info("\n".join(stats)) - return - - elif length == 2: - stats = [] - numChannels = 0 - numWhoEntries = 0 - - failures = 0 - if spl[1] in IRCPool.keys(): - numChannels += len(IRCPool[spl[1]].channels) - else: - failure("Name does not exist: %s" % spl[1]) - failures += 1 - if spl[1] in wholist.keys(): - numWhoEntries += len(wholist[spl[1]].keys()) - else: - failure("Who entry does not exist: %s" % spl[1]) - failures += 1 - if failures == 2: - failure("No information found, aborting") - return - stats.append("Channels: %s" % numChannels) - stats.append("User records: %s" % numWhoEntries) - info("\n".join(stats)) - return - else: - incUsage("stats") - - elif cmd == "enable": - if length == 2: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - pool[spl[1]]["enabled"] = True - self.save("pool") - if not spl[1] in IRCPool.keys(): - self.addBot(spl[1]) - else: - pass - success("Successfully enabled bot %s" % spl[1]) - return - else: - incUsage("enable") - return - - elif cmd == "disable": - if length == 2: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - pool[spl[1]]["enabled"] = False - self.save("pool") - if spl[1] in ReactorPool.keys(): - if spl[1] in FactoryPool.keys(): - FactoryPool[spl[1]].stopTrying() - ReactorPool[spl[1]].disconnect() - if spl[1] in IRCPool.keys(): - del IRCPool[spl[1]] - del ReactorPool[spl[1]] - del FactoryPool[spl[1]] - success("Successfully disabled bot %s" % spl[1]) - return - else: - incUsage("disable") - return - - elif cmd == "join": - if length == 3: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[1] in IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]) - return - IRCPool[spl[1]].join(spl[2]) - success("Joined %s" % spl[2]) - return - elif length == 4: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[1] in IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]) - return - IRCPool[spl[1]].join(spl[2], spl[3]) - success("Joined %s with key %s" % (spl[2], spl[3])) - return - else: - incUsage("join") - return - - elif cmd == "part": - if length == 3: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[1] in IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]) - return - IRCPool[spl[1]].part(spl[2]) - success("Left %s" % spl[2]) - return - else: - incUsage("part") - return - - elif cmd == "get": - if length == 3: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[1] in IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]) - return - info(str(IRCPool[spl[1]].get(spl[2]))) - return - else: - incUsage("get") - return - - elif cmd == "who": - if length == 2: - result = self.getWho(spl[1]) - rtrn = "" - for i in result.keys(): - rtrn += "Matches from: %s" % i - rtrn += "\n" - for x in result[i]: - x = [y for y in x if not y == None] - rtrn += str((", ".join(x))) - rtrn += "\n" - info(rtrn) - return - else: - incUsage("who") - return - - elif cmd == "key": - if data.startswith("key add"): - if not data in ["key add ", "key add"] and data[3] == " ": - keywordsToAdd = data[8:] - keywords = keywordsToAdd.split(",") - for keyword in keywords: - rtrn = self.addKeyword(keyword) - if rtrn == "EXISTS": - failure("Keyword already exists: %s" % keyword) - elif rtrn == "ISIN": - failure("Keyword already matched: %s" % keyword) - elif rtrn == True: - success("Keyword added: %s" % keyword) - return - else: - incUsage("key") - return - elif data.startswith("key del"): - if not data in ["key del ", "key del"] and data[3] == " ": - keywordsToDel = data[8:] - keywords = keywordsToDel.split(",") - for keyword in keywords: - rtrn = self.delKeyword(keyword) - if rtrn == "NOKEY": - failure("Keyword does not exist: %s" % keyword) - elif rtrn == True: - success("Keyword deleted: %s" % keyword) - return - else: - incUsage("key") - return - if length == 4: - if spl[1] == "except": - if not spl[2] in keyconf["Keywords"]: - failure("No such keyword: %s" % spl[2]) - return - if spl[2] in keyconf["KeywordsExcept"].keys(): - if spl[3] in keyconf["KeywordsExcept"][spl[2]]: - failure("Exception exists: %s" % spl[3]) - return - else: - if not spl[2] in spl[3]: - failure("Keyword %s not in exception %s. This won't work" % (spl[2], spl[3])) - return - keyconf["KeywordsExcept"][spl[2]] = [] - - keyconf["KeywordsExcept"][spl[2]].append(spl[3]) - self.save("keyconf") - success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) - return - elif spl[1] == "master": - if not spl[2] in pool.keys(): - failure("Name does not exist: %s" % spl[2]) - return - if spl[2] in IRCPool.keys(): - if not spl[3] in IRCPool[spl[2]].channels: - info("Bot not on channel: %s" % spl[3]) - config["Master"] = [spl[2], spl[3]] - self.save("config") - success("Master set to %s on %s" % (spl[3], spl[2])) - return - elif spl[1] == "unexcept": - if not spl[2] in keyconf["KeywordsExcept"].keys(): - failure("No such exception: %s" % spl[2]) - return - if not spl[3] in keyconf["KeywordsExcept"][spl[2]]: - failure("Exception %s has no attribute %s" % (spl[2], spl[3])) - return - keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) - if keyconf["KeywordsExcept"][spl[2]] == []: - del keyconf["KeywordsExcept"][spl[2]] - self.save("keyconf") - success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) - return - else: - incUsage("key") - return - elif length == 3: - if spl[1] == "unexcept": - if not spl[2] in keyconf["KeywordsExcept"].keys(): - failure("No such exception: %s" % spl[2]) - return - del keyconf["KeywordsExcept"][spl[2]] - self.save("keyconf") - success("Successfully removed exception list of %s" % spl[2]) - return - elif spl[1] == "monitor": - if spl[2] == "on": - if not obj.addr in MonitorPool: - MonitorPool.append(obj.addr) - success("Keyword monitoring enabled") - return - else: - failure("Keyword monitoring is already enabled") - return - elif spl[2] == "off": - if obj.addr in MonitorPool: - MonitorPool.remove(obj.addr) - success("Keyword monitoring disabled") - return - else: - failure("Keyword monitoring is already disabled") - return - else: - incUsage("key") - return - else: - incUsage("key") - return - elif length == 2: - if spl[1] == "show": - info(",".join(keyconf["Keywords"])) - return - - elif spl[1] == "showexcept": - exceptMap = [] - for i in keyconf["KeywordsExcept"].keys(): - exceptMap.append("Key: %s" % i) - exceptMap.append("%s: %s" % (i, ",".join(keyconf["KeywordsExcept"][i]))) - exceptMap.append("\n") - info("\n".join(exceptMap)) - return - elif spl[1] == "master": - info(" - ".join(config["Master"])) - return - elif spl[1] == "monitor": - if obj.addr in MonitorPool: - info("Keyword monitoring is enabled on this connection") - return - else: - info("Keyword monitoring is disabled on this connection") - return - else: - incUsage("key") - return - - else: - incUsage("key") - return - - elif cmd == "add": - if length > 6: - failure("Too many arguments") - return - - if length > 1: - name = spl[1] - else: - incUsage("add") - return - if length > 2: - host = spl[2] - if length > 3: - port = spl[3] - if length > 4: - protocol = spl[4] - if length > 5: - nickname = spl[5] - - toFail = False - if length < 6: - if config["Default"]["nickname"] == None: - failure("Choose a nickname, or configure one in defaults") - toFail = True - else: - nickname = config["Default"]["nickname"] - - if length < 5: - if config["Default"]["protocol"] == None: - failure("Choose a protocol, or configure one in defaults") - toFail = True - else: - protocol = config["Default"]["protocol"] - - if length < 4: - if config["Default"]["port"] == None: - failure("Choose a port, or configure one in defaults") - toFail = True - else: - port = config["Default"]["port"] - - if length < 3: - if config["Default"]["host"] == None: - failure("Choose a host, or configure one in defaults") - toFail = True - else: - host = config["Default"]["host"] - if toFail: - failure("Stopping due to previous error(s)") - return - - if length < 2: - incUsage("add") - return - - if name in pool.keys(): - failure("Name already exists: %s" % name) - return - - protocol = protocol.lower() - - if not protocol in ["ssl", "plain"]: - failure("Protocol must be ssl or plain, not %s" % protocol) - return - - try: - int(port) - except: - failure("Port must be an integer, not %s" % port) - return - - pool[name] = { "host": host, - "port": port, - "protocol": protocol, - "bind": config["Default"]["bind"], - "timeout": config["Default"]["timeout"], - "maxdelay": config["Default"]["maxdelay"], - "initialdelay": config["Default"]["initialdelay"], - "factor": config["Default"]["factor"], - "jitter": config["Default"]["jitter"], - "nickname": nickname, - "username": config["Default"]["username"], - "realname": config["Default"]["realname"], - "userinfo": config["Default"]["userinfo"], - "finger": config["Default"]["finger"], - "version": config["Default"]["version"], - "source": config["Default"]["source"], - "autojoin": config["Default"]["autojoin"], - "authtype": config["Default"]["authtype"], - "password": config["Default"]["password"], - "authentity": config["Default"]["authentity"], - "key": config["Default"]["key"], - "certificate": config["Default"]["certificate"], - "enabled": config["ConnectOnCreate"], - } - if config["ConnectOnCreate"] == True: - self.addBot(name) - success("Successfully created bot") - self.save("pool") - return - - elif cmd == "del": - if length == 2: - - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - del pool[spl[1]] - if spl[1] in ReactorPool.keys(): - if spl[1] in FactoryPool.keys(): - FactoryPool[spl[1]].stopTrying() - ReactorPool[spl[1]].disconnect() - if spl[1] in IRCPool.keys(): - del IRCPool[spl[1]] - del ReactorPool[spl[1]] - del FactoryPool[spl[1]] - success("Successfully removed bot") - self.save("pool") - return - else: - incUsage("del") - return - elif cmd == "default": - toUnset = False - if length == 1: - optionMap = ["Viewing defaults"] - for i in config["Default"].keys(): - optionMap.append("%s: %s" % (i, config["Default"][i])) - info("\n".join(optionMap)) - return - elif length == 2: - if not spl[1] in config["Default"].keys(): - failure("No such key: %s" % spl[1]) - return - info("%s: %s" % (spl[1], config["Default"][spl[1]])) - return - elif length == 3: - if not spl[1] in config["Default"].keys(): - failure("No such key: %s" % spl[1]) - return - - if spl[2].lower() in ["none", "nil"]: - spl[2] = None - toUnset = True - - if spl[1] in ["port", "timeout", "maxdelay"]: - try: - spl[2] = int(spl[2]) - except: - failure("Value must be an integer, not %s" % spl[2]) - return - - if spl[2] in ["initialdelay", "factor", "jitter"]: - try: - spl[3] = float(spl[3]) - except: - failure("Value must be a floating point integer, not %s" % spl[3]) - return - - if spl[1] == "protocol": - if not toUnset: - if not spl[2] in ["ssl", "plain"]: - failure("Protocol must be ssl or plain, not %s" % spl[2]) - return - - if spl[2] == config["Default"][spl[1]]: - failure("Value already exists: %s" % spl[2]) - return - - if spl[1] == "authtype": - if not toUnset: - if not spl[2] in ["sp", "ns"]: - failure("Authtype must be sp or ns, not %s" % spl[2]) - return - if spl[1] == "enabled": - failure("Use the ConnectOnCreate config parameter to set this") - return - if spl[1] == "autojoin": - if not toUnset: - spl[2] = spl[2].split(",") - else: - spl[2] = [] - - config["Default"][spl[1]] = spl[2] - self.save("config") - if toUnset: - success("Successfully unset key %s" % spl[1]) - else: - success("Successfully set key %s to %s" % (spl[1], spl[2])) - return - else: - incUsage("default") - return - - elif cmd == "mod": - toUnset = False - if length == 2: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - optionMap = ["Viewing options for %s" % spl[1]] - for i in pool[spl[1]].keys(): - optionMap.append("%s: %s" % (i, pool[spl[1]][i])) - info("\n".join(optionMap)) - return - - elif length == 3: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[2] in pool[spl[1]].keys(): - failure("No such key: %s" % spl[2]) - return - info("%s: %s" % (spl[2], pool[spl[1]][spl[2]])) - return - - elif length == 4: - if not spl[1] in pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[2] in pool[spl[1]].keys(): - failure("No such key: %s" % spl[2]) - return - - if spl[2] == "protocol": - if not spl[3] in ["ssl", "plain"]: - failure("Protocol must be ssl or plain, not %s" % spl[3]) - return - if spl[3] == pool[spl[1]][spl[2]]: - failure("Value already exists: %s" % spl[3]) - return - - if spl[3].lower() in ["none", "nil"]: - spl[3] = None - toUnset = True - - if spl[2] in ["port", "timeout", "maxdelay"]: - try: - spl[3] = int(spl[3]) - except: - failure("Value must be an integer, not %s" % spl[3]) - return - - if spl[2] in ["initialdelay", "factor", "jitter"]: - try: - spl[3] = float(spl[3]) - except: - failure("Value must be a floating point integer, not %s" % spl[3]) - return - - if spl[2] == "authtype": - if not toUnset: - if not spl[3] in ["sp", "ns"]: - failure("Authtype must be sp or ns, not %s" % spl[3]) - return - if spl[2] == "enabled": - failure("Use the enable and disable commands to manage this") - return - if spl[2] == "autojoin": - spl[3] = spl[3].split(",") - - pool[spl[1]][spl[2]] = spl[3] - if spl[1] in IRCPool.keys(): - IRCPool[spl[1]].refresh() - self.save("pool") - if toUnset: - success("Successfully unset key %s on %s" % (spl[2], spl[1])) - else: - success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) - return - - else: - incUsage("mod") - return - - else: - incUsage(None) - return - else: - if cmd == "pass" and length == 2: - if spl[1] == config["Password"]: - success("Authenticated successfully") - obj.authed = True - return - else: - failure("Password incorrect") - obj.transport.loseConnection() - return - else: - incUsage(None) - return - if __name__ == "__main__": - helper = Helper() - for i in filemap.keys(): - helper.load(i) - for i in pool.keys(): if pool[i]["enabled"] == True: helper.addBot(i) diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py new file mode 100644 index 0000000..ce65a9c --- /dev/null +++ b/utils/loaders/command_loader.py @@ -0,0 +1,15 @@ +from os import listdir +from core.main import * +from utils.logging.log import * +import commands + +def loadCommands(func): + for filename in listdir('commands'): + if filename.endswith('.py') and filename != "__init__.py": + commandName = filename[0:-3] + className = commandName.capitalize() + #try: + __import__('commands.%s' % commandName) + eval('commands.%s.%s(func)' % (commandName, className)) + #except Exception as err: + # error("Exception while loading command %s:\n%s" % (commandName, err)) diff --git a/utils/logging/log.py b/utils/logging/log.py index 87d0ed5..5e1141f 100644 --- a/utils/logging/log.py +++ b/utils/logging/log.py @@ -9,4 +9,3 @@ def warn(data): def error(data): print("[ERROR]", data) - exit(1) diff --git a/utils/logging/send.py b/utils/logging/send.py new file mode 100644 index 0000000..2f894f1 --- /dev/null +++ b/utils/logging/send.py @@ -0,0 +1,26 @@ +from core.main import connections, help + +def sendData(addr, data): + connections[addr].send(data) + +def sendSuccess(addr, data): + sendData(addr, "[y] " + data) + +def sendFailure(addr, data): + sendData(addr, "[n] " + data) + +def sendInfo(addr, data): + sendData(addr, "[i] " + data) + +def sendAll(data): + for i in connections: + connections[i].send(data) + return + +def incorrectUsage(addr, mode): + if mode == None: + sendFailure(addr, "Incorrect usage") + return + if mode in help.keys(): + sendFailure(addr, "Usage: " + help[mode]) + return From 378c4d9bba14e92d4978a34f9630a1e66385a788 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 23 Feb 2018 23:26:21 +0000 Subject: [PATCH 068/394] Implement loading new modules at runtime --- commands/loadmod.py | 23 +++++++++++++++++++++++ conf/help.json | 5 +++-- core/parser.py | 2 +- utils/loaders/command_loader.py | 10 +++++----- utils/loaders/single_loader.py | 17 +++++++++++++++++ 5 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 commands/loadmod.py create mode 100644 utils/loaders/single_loader.py diff --git a/commands/loadmod.py b/commands/loadmod.py new file mode 100644 index 0000000..9fb3697 --- /dev/null +++ b/commands/loadmod.py @@ -0,0 +1,23 @@ +from core.main import * +from utils.loaders.single_loader import loadSingle + +class Loadmod: + def __init__(self, register): + register("loadmod", self.loadmod) + + def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + rtrn = loadSingle(spl[1], register) + if rtrn == True: + success("Loaded module %s" % spl[1]) + return + else: + failure("Error loading module %s: %s" % (spl[1], rtrn)) + return + else: + incUsage("loadmod") + return + else: + incUsage(None) + return diff --git a/conf/help.json b/conf/help.json index b505b6c..dba2f80 100644 --- a/conf/help.json +++ b/conf/help.json @@ -15,5 +15,6 @@ "stats": "stats []", "save": "save ", "load": "load ", - "dist": "dist" -} \ No newline at end of file + "dist": "dist", + "loadmod": "loadmod " +} diff --git a/core/parser.py b/core/parser.py index 2845829..ecfd18a 100644 --- a/core/parser.py +++ b/core/parser.py @@ -23,7 +23,7 @@ def parseCommand(addr, authed, data): failure("No text was sent") return for i in CommandMap.keys(): - if data.startswith(i): + if spl[0] == i: CommandMap[i](addr, authed, data, obj, spl, success, failure, info, incUsage, length) return incUsage(None) diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index ce65a9c..61eb93e 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -8,8 +8,8 @@ def loadCommands(func): if filename.endswith('.py') and filename != "__init__.py": commandName = filename[0:-3] className = commandName.capitalize() - #try: - __import__('commands.%s' % commandName) - eval('commands.%s.%s(func)' % (commandName, className)) - #except Exception as err: - # error("Exception while loading command %s:\n%s" % (commandName, err)) + try: + __import__('commands.%s' % commandName) + eval('commands.%s.%s(func)' % (commandName, className)) + except Exception as err: + error("Exception while loading command %s:\n%s" % (commandName, err)) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py new file mode 100644 index 0000000..11de8ef --- /dev/null +++ b/utils/loaders/single_loader.py @@ -0,0 +1,17 @@ +from os import listdir +from core.main import * +from utils.logging.log import * +import commands + +def loadSingle(command, func): + if command+".py" in listdir("commands"): + try: + if command in CommandMap.keys(): + return "Cannot reload modules" + else: + className = command.capitalize() + __import__("commands.%s" % command) + eval("commands.%s.%s(func)" % (command, className)) + return True + except Exception as err: + return err From 8dec0b6828bac6a23f92d43979ea03571fbd560b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 24 Feb 2018 12:42:27 +0000 Subject: [PATCH 069/394] Implement counting of various IRC events --- commands/list.py | 2 +- commands/mod.py | 2 +- commands/stats.py | 15 +++++++++++++++ conf/help.json | 6 +++--- core/bot.py | 16 ++++++++++++++++ core/main.py | 3 ++- modules/counters.py | 29 +++++++++++++++++++++++++++++ threshold | 6 ++++-- 8 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 modules/counters.py diff --git a/commands/list.py b/commands/list.py index e39ee95..996f5d1 100644 --- a/commands/list.py +++ b/commands/list.py @@ -10,7 +10,7 @@ class List: for i in pool.keys(): poolMap.append("Server: %s" % i) for x in pool[i].keys(): - poolMap.append("%s: %s" % (x, pool[i][x])) + poolMap.append(" %s: %s" % (x, pool[i][x])) poolMap.append("\n") info("\n".join(poolMap)) return diff --git a/commands/mod.py b/commands/mod.py index 8ffd9ad..6bb408a 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -13,7 +13,7 @@ class Mod: return optionMap = ["Viewing options for %s" % spl[1]] for i in pool[spl[1]].keys(): - optionMap.append("%s: %s" % (i, pool[spl[1]][i])) + optionMap.append(" %s: %s" % (i, pool[spl[1]][i])) info("\n".join(optionMap)) return diff --git a/commands/stats.py b/commands/stats.py index 9b1f76f..d1cc14e 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,4 +1,5 @@ from core.main import * +import modules.counters as count class Stats: def __init__(self, register): @@ -17,6 +18,13 @@ class Stats: stats.append("Servers: %s" % len(IRCPool.keys())) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) + counterEvents = count.getEvents() + if counterEvents == None: + stats.append("No counters records") + else: + stats.append("Counters:") + for i in counterEvents.keys(): + stats.append(" %s: %s" % (i, counterEvents[i])) info("\n".join(stats)) return @@ -41,6 +49,13 @@ class Stats: return stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) + counterEvents = count.getEvents(spl[1]) + if counterEvents == None: + stats.append("No counters records") + else: + stats.append("Counters:") + for i in counterEvents.keys(): + stats.append(" %s: %s" % (i, counterEvents[i])) info("\n".join(stats)) return else: diff --git a/conf/help.json b/conf/help.json index dba2f80..30ef842 100644 --- a/conf/help.json +++ b/conf/help.json @@ -13,8 +13,8 @@ "disable": "disable ", "list": "list", "stats": "stats []", - "save": "save ", - "load": "load ", + "save": "save ", + "load": "load ", "dist": "dist", "loadmod": "loadmod " -} +} \ No newline at end of file diff --git a/core/bot.py b/core/bot.py index ec0894c..4b33fce 100644 --- a/core/bot.py +++ b/core/bot.py @@ -4,6 +4,7 @@ from twisted.internet.defer import Deferred import modules.keyword as keyword import modules.userinfo as userinfo +import modules.counters as count from core.main import * from utils.logging.log import * @@ -66,18 +67,21 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "PRV", self.name) def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOT", self.name) def action(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACT", self.name) @@ -126,6 +130,7 @@ class IRCBot(IRCClient): return n = self._who[channel][1] n[nick] = [nick, user, host, server, status, realname] + count.event(self.name, "whoreply") def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -148,38 +153,46 @@ class IRCBot(IRCClient): self.msg(self.authentity, "IDENTIFY %s" % self.nspass) for i in self.autojoin: self.join(i) + count.event(self.name, "signedon") def joined(self, channel): if not channel in self.channels: self.channels.append(channel) self.who(channel).addCallback(self.got_who) + count.event(self.name, "selfjoin") def left(self, channel): if channel in self.channels: self.channels.remove(channel) + count.event(self.name, "selfpart") def kickedFrom(self, channel, kicker, message): if channel in self.channels: self.channels.remove(channel) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) + count.event(self.name, "selfkick") def userJoined(self, user, channel): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "join") def userLeft(self, user, channel): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "part") def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "quit") keyword.actKeyword(user, None, quitMessage, self.nickname, "QUT", self.name) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "kick") keyword.actKeyword(kicker, channel, message, self.nickname, "KCK", self.name) @@ -187,16 +200,19 @@ class IRCBot(IRCClient): nick, ident, host = self.parsen(oldname) userinfo.setWhoSingle(self.name, nick, ident, host) userinfo.setWhoSingle(self.name, newname, ident, host) + count.event(self.name, "nick") def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "topic") keyword.actKeyword(user, channel, newTopic, self.nickname, "TOP", self.name) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + count.event(self.name, "mode") class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): diff --git a/core/main.py b/core/main.py index c732c09..b393d7f 100644 --- a/core/main.py +++ b/core/main.py @@ -11,6 +11,7 @@ filemap = { "pool": ["pool.json", "pool"], "help": ["help.json", "command help"], "wholist": ["wholist.json", "WHO lists"], + "counters": ["counters.json", "counters file"], } numbers = "0123456789" @@ -28,7 +29,7 @@ CommandMap = {} def register(command, function): if not command in CommandMap: CommandMap[command] = function - log("Registering command %s" % command) + debug("Registered command: %s" % command) else: error("Duplicate command: %s" % (command)) diff --git a/modules/counters.py b/modules/counters.py new file mode 100644 index 0000000..d832d46 --- /dev/null +++ b/modules/counters.py @@ -0,0 +1,29 @@ +from core.main import * + +def event(name, eventType): + if not "local" in counters.keys(): + counters["local"] = {} + if not "global" in counters.keys(): + counters["global"] = {} + if not name in counters["local"].keys(): + counters["local"][name] = {} + if eventType not in counters["local"][name].keys(): + counters["local"][name][eventType] = 0 + + if eventType not in counters["global"]: + counters["global"][eventType] = 0 + + counters["local"][name][eventType] += 1 + counters["global"][eventType] += 1 + +def getEvents(name=None): + if name == None: + if "global" in counters.keys(): + return counters["global"] + else: + return None + else: + if name in counters["local"].keys(): + return counters["local"][name] + else: + return None diff --git a/threshold b/threshold index 9524f5a..0439b1a 100755 --- a/threshold +++ b/threshold @@ -20,7 +20,8 @@ from core.main import ( saveConf, loadConf, initMain, -) + ) + initMain() from core.main import ( @@ -29,7 +30,8 @@ from core.main import ( pool, help, wholist, -) + counters, + ) from utils.logging.log import * import modules.userinfo as userinfo From 8940aeeb80752e9c6d65cb79d1130fa328f7327b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 24 Feb 2018 12:44:30 +0000 Subject: [PATCH 070/394] Add the counters to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e6278e1..909e3f8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ conf/config.json conf/pool.json conf/wholist.json conf/keyword.json +conf/counters.json conf/dist.sh env/ From badfe46694a077bae4aa54c2bc15c971fd91afc8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 4 Mar 2018 13:17:04 +0000 Subject: [PATCH 071/394] Add __pycache__ to gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 909e3f8..0f1e65e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *.pyc -*.db *.pem +__pycache__/ conf/config.json conf/pool.json conf/wholist.json From eae43155621b06487b6083e0a65f0072a547e4ef Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 4 Mar 2018 13:26:53 +0000 Subject: [PATCH 072/394] Move server functions to a separate file --- conf/keyword.json | 4 +--- core/server.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++ threshold | 57 ++-------------------------------------------- 3 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 core/server.py diff --git a/conf/keyword.json b/conf/keyword.json index 3980412..fc70716 100644 --- a/conf/keyword.json +++ b/conf/keyword.json @@ -1,6 +1,4 @@ { - "Keywords": [ - "example" - ], + "Keywords": [], "KeywordsExcept": {} } \ No newline at end of file diff --git a/core/server.py b/core/server.py new file mode 100644 index 0000000..71e9056 --- /dev/null +++ b/core/server.py @@ -0,0 +1,58 @@ +from twisted.internet.protocol import Protocol, Factory, ClientFactory +from core.main import * +from utils.logging.log import * + +from core.parser import parseCommand + +class Server(Protocol): + def __init__(self, addr): + self.addr = addr + self.authed = False + if config["UsePassword"] == False: + self.authed = True + + def send(self, data): + data += "\r\n" + data = data.encode("utf-8", "replace") + self.transport.write(data) + + def dataReceived(self, data): + data = data.decode("utf-8", "replace") + #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) + if "\n" in data: + splitData = [x for x in data.split("\n") if x] + if "\n" in data: + for i in splitData: + parseCommand(self.addr, self.authed, i) + return + parseCommand(self.addr, self.authed, data) + + def connectionMade(self): + log("Connection from %s:%s" % (self.addr.host, self.addr.port)) + self.send("Hello.") + + def connectionLost(self, reason): + self.authed = False + log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) + if not listener == None: + if self.addr in connections.keys(): + del connections[self.addr] + else: + warn("Tried to remove a non-existant connection.") + else: + warn("Tried to remove a connection from a listener that wasn't running.") + if self.addr in MonitorPool: + MonitorPool.remove(self.addr) + +class ServerFactory(Factory): + def buildProtocol(self, addr): + entry = Server(addr) + connections[addr] = entry + return entry + + def send(self, addr, data): + if addr in connections.keys(): + connection = connections[addr] + connection.send(data) + else: + return diff --git a/threshold b/threshold index 0439b1a..eb72cf3 100755 --- a/threshold +++ b/threshold @@ -1,7 +1,6 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory -from twisted.internet.protocol import Protocol, Factory, ClientFactory #from twisted.python import log #from sys import stdout @@ -36,67 +35,15 @@ from core.main import ( from utils.logging.log import * import modules.userinfo as userinfo import core.helper as helper -from core.parser import parseCommand +from core.server import Server, ServerFactory -class Base(Protocol): - def __init__(self, addr): - self.addr = addr - self.authed = False - if config["UsePassword"] == False: - self.authed = True - - def send(self, data): - data += "\r\n" - data = data.encode("utf-8", "replace") - self.transport.write(data) - - def dataReceived(self, data): - data = data.decode("utf-8", "replace") - #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) - if "\n" in data: - splitData = [x for x in data.split("\n") if x] - if "\n" in data: - for i in splitData: - parseCommand(self.addr, self.authed, i) - return - parseCommand(self.addr, self.authed, data) - - def connectionMade(self): - log("Connection from %s:%s" % (self.addr.host, self.addr.port)) - self.send("Hello.") - - def connectionLost(self, reason): - self.authed = False - log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) - if not listener == None: - if self.addr in connections.keys(): - del connections[self.addr] - else: - warn("Tried to remove a non-existant connection.") - else: - warn("Tried to remove a connection from a listener that wasn't running.") - if self.addr in MonitorPool: - MonitorPool.remove(self.addr) - -class BaseFactory(Factory): - def buildProtocol(self, addr): - entry = Base(addr) - connections[addr] = entry - return entry - - def send(self, addr, data): - if addr in connections.keys(): - connection = connections[addr] - connection.send(data) - else: - return if __name__ == "__main__": for i in pool.keys(): if pool[i]["enabled"] == True: helper.addBot(i) - listener = BaseFactory() + listener = ServerFactory() if config["Listener"]["UseSSL"] == True: reactor.listenSSL(config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(certPath+config["Listener"]["Key"], certPath+config["Listener"]["Certificate"]), interface=config["Listener"]["Address"]) log("Threshold running with SSL on %s:%s" % (config["Listener"]["Address"], config["Listener"]["Port"])) From da6c45f0933fff83a763c91843f10bd0c71adf04 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 4 Mar 2018 17:25:57 +0000 Subject: [PATCH 073/394] Implement counting keyword events and a unified buffers system for when the master channel is unavailable --- .gitignore | 1 + commands/key.py | 9 ++++++++- commands/save.py | 2 +- conf/example/counters.json | 1 + conf/example/masterbuf.json | 1 + conf/example/pool.json | 1 + conf/example/wholist.json | 1 + conf/help.json | 4 ++-- conf/keyword.json | 4 +++- core/bot.py | 4 ++++ core/main.py | 5 ++--- modules/keyword.py | 31 +++++++++++++++++++++++++++++-- modules/userinfo.py | 5 +++-- threshold | 10 +++------- utils/fileutil/read_line.py | 0 utils/fileutil/write_line.py | 0 16 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 conf/example/counters.json create mode 100644 conf/example/masterbuf.json create mode 100644 conf/example/pool.json create mode 100644 conf/example/wholist.json create mode 100644 utils/fileutil/read_line.py create mode 100644 utils/fileutil/write_line.py diff --git a/.gitignore b/.gitignore index 0f1e65e..f3d7146 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ conf/pool.json conf/wholist.json conf/keyword.json conf/counters.json +conf/masterbuf.json conf/dist.sh env/ diff --git a/commands/key.py b/commands/key.py index e523097..7a5ccb0 100644 --- a/commands/key.py +++ b/commands/key.py @@ -97,6 +97,13 @@ class Key: if not obj.addr in MonitorPool: MonitorPool.append(obj.addr) success("Keyword monitoring enabled") + if len(masterbuf) == 0: + return + rtrn = [] + for i in range(len(masterbuf)): + rtrn.append(masterbuf.pop(0)) + saveConf("masterbuf") + info("\n".join(rtrn)) return else: failure("Keyword monitoring is already enabled") @@ -129,7 +136,7 @@ class Key: info("\n".join(exceptMap)) return elif spl[1] == "master": - info(" - ".join(config["Master"])) + info(" - ".join([str(i) for i in config["Master"]])) return elif spl[1] == "monitor": if obj.addr in MonitorPool: diff --git a/commands/save.py b/commands/save.py index 64c4cbf..7fbb85c 100644 --- a/commands/save.py +++ b/commands/save.py @@ -14,7 +14,7 @@ class Save: elif spl[1] == "all": for i in filemap.keys(): saveConf(i) - success("Saved %s from %s" % (i, filemap[i][0])) + success("Saved %s to %s" % (i, filemap[i][0])) return else: incUsage("save") diff --git a/conf/example/counters.json b/conf/example/counters.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/counters.json @@ -0,0 +1 @@ +{} diff --git a/conf/example/masterbuf.json b/conf/example/masterbuf.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/conf/example/masterbuf.json @@ -0,0 +1 @@ +[] diff --git a/conf/example/pool.json b/conf/example/pool.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/pool.json @@ -0,0 +1 @@ +{} diff --git a/conf/example/wholist.json b/conf/example/wholist.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/wholist.json @@ -0,0 +1 @@ +{} diff --git a/conf/help.json b/conf/help.json index 30ef842..e251599 100644 --- a/conf/help.json +++ b/conf/help.json @@ -13,8 +13,8 @@ "disable": "disable ", "list": "list", "stats": "stats []", - "save": "save ", - "load": "load ", + "save": "save ", + "load": "load ", "dist": "dist", "loadmod": "loadmod " } \ No newline at end of file diff --git a/conf/keyword.json b/conf/keyword.json index fc70716..3980412 100644 --- a/conf/keyword.json +++ b/conf/keyword.json @@ -1,4 +1,6 @@ { - "Keywords": [], + "Keywords": [ + "example" + ], "KeywordsExcept": {} } \ No newline at end of file diff --git a/core/bot.py b/core/bot.py index 4b33fce..b5f6acb 100644 --- a/core/bot.py +++ b/core/bot.py @@ -160,6 +160,10 @@ class IRCBot(IRCClient): self.channels.append(channel) self.who(channel).addCallback(self.got_who) count.event(self.name, "selfjoin") + if self.name == config["Master"][0] and channel == config["Master"][1]: + for i in range(len(masterbuf)): + self.msg(channel, masterbuf.pop(0)) + saveConf("masterbuf") def left(self, channel): if channel in self.channels: diff --git a/core/main.py b/core/main.py index b393d7f..17cfaef 100644 --- a/core/main.py +++ b/core/main.py @@ -1,4 +1,5 @@ from json import load, dump, loads + from utils.loaders.command_loader import loadCommands from utils.logging.log import * @@ -12,11 +13,9 @@ filemap = { "help": ["help.json", "command help"], "wholist": ["wholist.json", "WHO lists"], "counters": ["counters.json", "counters file"], + "masterbuf": ["masterbuf.json", "master buffer"], } -numbers = "0123456789" - -listener = None connections = {} IRCPool = {} ReactorPool = {} diff --git a/modules/keyword.py b/modules/keyword.py index 43547f6..25207bc 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -1,11 +1,36 @@ from core.main import * from utils.logging.log import * +import modules.counters as count def sendMaster(data): - if config["Master"][0] in IRCPool.keys(): - IRCPool[config["Master"][0]].msg(config["Master"][1], data) + if not len(MonitorPool) == 0: + hasMonitors = True else: + hasMonitors = False + + if config["Master"] == [None, None]: + if hasMonitors: + for i in MonitorPool: + connections[i].send(data) + return + else: + masterbuf.append(data) + saveConf("masterbuf") + return + + if config["Master"][0] in IRCPool.keys(): + if config["Master"][1] in IRCPool[config["Master"][0]].channels: + IRCPool[config["Master"][0]].msg(config["Master"][1], data) + else: + if not hasMonitors: + masterbuf.append(data) + saveConf("masterbuf") + else: + if not hasMonitors: + masterbuf.append(data) + saveConf("masterbuf") warn("Master with no IRC instance defined") + for i in MonitorPool: connections[i].send(data) @@ -52,8 +77,10 @@ def actKeyword(user, channel, message, nickname, actType, name): if nickLower in msgLower: msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") sendMaster("NICK %s %s (T:%s): (%s/%s) %s" % (actType, name, msgLower.count(nickLower), user, channel, msgLower)) + count.event(name, "nickhighlight") if toSend: sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) + count.event(name, "keymatch") def addKeyword(keyword): if keyword in keyconf["Keywords"]: diff --git a/modules/userinfo.py b/modules/userinfo.py index f0403fe..ad7fe04 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,8 +1,9 @@ from core.main import * +from string import digits #from utils.logging.log import * def setWho(network, newObjects): - network = "".join([x for x in network if not x in numbers]) + network = "".join([x for x in network if not x in digits]) if not network in wholist.keys(): wholist[network] = {} for i in newObjects.keys(): @@ -11,7 +12,7 @@ def setWho(network, newObjects): return def setWhoSingle(network, nick, ident, host): - network = "".join([x for x in network if not x in numbers]) + network = "".join([x for x in network if not x in digits]) if network in wholist.keys(): if nick in wholist[network].keys(): diff --git a/threshold b/threshold index eb72cf3..f83795b 100755 --- a/threshold +++ b/threshold @@ -7,10 +7,8 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory #log.startLogging(stdout) from core.main import ( - numbers, configPath, certPath, - listener, connections, IRCPool, ReactorPool, @@ -37,12 +35,7 @@ import modules.userinfo as userinfo import core.helper as helper from core.server import Server, ServerFactory - if __name__ == "__main__": - for i in pool.keys(): - if pool[i]["enabled"] == True: - helper.addBot(i) - listener = ServerFactory() if config["Listener"]["UseSSL"] == True: reactor.listenSSL(config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(certPath+config["Listener"]["Key"], certPath+config["Listener"]["Certificate"]), interface=config["Listener"]["Address"]) @@ -50,5 +43,8 @@ if __name__ == "__main__": else: reactor.listenTCP(config["Listener"]["Port"], listener, interface=config["Listener"]["Address"]) log("Threshold running on %s:%s" % (config["Listener"]["Address"], config["Listener"]["Port"])) + for i in pool.keys(): + if pool[i]["enabled"] == True: + helper.addBot(i) reactor.run() diff --git a/utils/fileutil/read_line.py b/utils/fileutil/read_line.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/fileutil/write_line.py b/utils/fileutil/write_line.py new file mode 100644 index 0000000..e69de29 From 5b1e3c6fb15674248ecaddff24e2e485823a2549 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 10 Mar 2018 14:01:43 +0000 Subject: [PATCH 074/394] Implement counting nickname changes and remove check on listener --- core/bot.py | 4 ++++ core/server.py | 9 +++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/bot.py b/core/bot.py index b5f6acb..a704ae7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -101,6 +101,10 @@ class IRCBot(IRCClient): newnick = nickname + "_" return newnick + def nickChanged(self, nick): + self.nickname = nick + count.event(self.name, "selfnick") + def irc_ERR_NICKNAMEINUSE(self, prefix, params): self._attemptedNick = self.alterCollidedNick(self._attemptedNick) self.setNick(self._attemptedNick) diff --git a/core/server.py b/core/server.py index 71e9056..4aa81cf 100644 --- a/core/server.py +++ b/core/server.py @@ -34,13 +34,10 @@ class Server(Protocol): def connectionLost(self, reason): self.authed = False log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) - if not listener == None: - if self.addr in connections.keys(): - del connections[self.addr] - else: - warn("Tried to remove a non-existant connection.") + if self.addr in connections.keys(): + del connections[self.addr] else: - warn("Tried to remove a connection from a listener that wasn't running.") + warn("Tried to remove a non-existant connection.") if self.addr in MonitorPool: MonitorPool.remove(self.addr) From d168d69732743d160e2275e41d9d6bb59df9e64a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 14 Mar 2018 20:13:40 +0000 Subject: [PATCH 075/394] Import the main module properly and fix some oddities in Twisted to prevent it from discarding some data --- commands/add.py | 62 +++++++++--------- commands/default.py | 20 +++--- commands/delete.py | 22 +++---- commands/disable.py | 24 +++---- commands/dist.py | 8 +-- commands/enable.py | 8 +-- commands/get.py | 8 +-- commands/help.py | 6 +- commands/join.py | 14 ++-- commands/key.py | 69 ++++++++++--------- commands/list.py | 8 +-- commands/load.py | 14 ++-- commands/loadmod.py | 4 +- commands/logout.py | 6 +- commands/mod.py | 28 ++++---- commands/part.py | 8 +-- commands/password.py | 4 +- commands/save.py | 14 ++-- commands/stats.py | 20 +++--- commands/who.py | 2 +- conf/keyword.json | 3 +- core/bot.py | 113 ++++++++++++++++++++++++++------ core/helper.py | 24 +++---- core/parser.py | 10 +-- core/server.py | 18 ++--- core/main.py => main.py | 0 modules/counters.py | 34 +++++----- modules/keyword.py | 56 ++++++++-------- modules/userinfo.py | 26 ++++---- threshold | 38 +++-------- utils/loaders/command_loader.py | 1 - utils/loaders/single_loader.py | 4 +- utils/logging/send.py | 12 ++-- 33 files changed, 370 insertions(+), 318 deletions(-) rename core/main.py => main.py (100%) diff --git a/commands/add.py b/commands/add.py index b003c10..1351fd1 100644 --- a/commands/add.py +++ b/commands/add.py @@ -1,4 +1,4 @@ -from core.main import * +import main import core.helper as helper class Add: @@ -27,32 +27,32 @@ class Add: toFail = False if length < 6: - if config["Default"]["nickname"] == None: + if main.config["Default"]["nickname"] == None: failure("Choose a nickname, or configure one in defaults") toFail = True else: - nickname = config["Default"]["nickname"] + nickname = main.config["Default"]["nickname"] if length < 5: - if config["Default"]["protocol"] == None: + if main.config["Default"]["protocol"] == None: failure("Choose a protocol, or configure one in defaults") toFail = True else: - protocol = config["Default"]["protocol"] + protocol = main.config["Default"]["protocol"] if length < 4: - if config["Default"]["port"] == None: + if main.config["Default"]["port"] == None: failure("Choose a port, or configure one in defaults") toFail = True else: - port = config["Default"]["port"] + port = main.config["Default"]["port"] if length < 3: - if config["Default"]["host"] == None: + if main.config["Default"]["host"] == None: failure("Choose a host, or configure one in defaults") toFail = True else: - host = config["Default"]["host"] + host = main.config["Default"]["host"] if toFail: failure("Stopping due to previous error(s)") return @@ -61,7 +61,7 @@ class Add: incUsage("add") return - if name in pool.keys(): + if name in main.pool.keys(): failure("Name already exists: %s" % name) return @@ -77,34 +77,34 @@ class Add: failure("Port must be an integer, not %s" % port) return - pool[name] = { "host": host, + main.pool[name] = { "host": host, "port": port, "protocol": protocol, - "bind": config["Default"]["bind"], - "timeout": config["Default"]["timeout"], - "maxdelay": config["Default"]["maxdelay"], - "initialdelay": config["Default"]["initialdelay"], - "factor": config["Default"]["factor"], - "jitter": config["Default"]["jitter"], + "bind": main.config["Default"]["bind"], + "timeout": main.config["Default"]["timeout"], + "maxdelay": main.config["Default"]["maxdelay"], + "initialdelay": main.config["Default"]["initialdelay"], + "factor": main.config["Default"]["factor"], + "jitter": main.config["Default"]["jitter"], "nickname": nickname, - "username": config["Default"]["username"], - "realname": config["Default"]["realname"], - "userinfo": config["Default"]["userinfo"], - "finger": config["Default"]["finger"], - "version": config["Default"]["version"], - "source": config["Default"]["source"], - "autojoin": config["Default"]["autojoin"], - "authtype": config["Default"]["authtype"], - "password": config["Default"]["password"], - "authentity": config["Default"]["authentity"], - "key": config["Default"]["key"], - "certificate": config["Default"]["certificate"], - "enabled": config["ConnectOnCreate"], + "username": main.config["Default"]["username"], + "realname": main.config["Default"]["realname"], + "userinfo": main.config["Default"]["userinfo"], + "finger": main.config["Default"]["finger"], + "version": main.config["Default"]["version"], + "source": main.config["Default"]["source"], + "autojoin": main.config["Default"]["autojoin"], + "authtype": main.config["Default"]["authtype"], + "password": main.config["Default"]["password"], + "authentity": main.config["Default"]["authentity"], + "key": main.config["Default"]["key"], + "certificate": main.config["Default"]["certificate"], + "enabled": main.config["ConnectOnCreate"], } if config["ConnectOnCreate"] == True: helper.addBot(name) success("Successfully created bot") - saveConf("pool") + main.saveConf("pool") return else: incUsage(None) diff --git a/commands/default.py b/commands/default.py index 7e5f2a5..82387af 100644 --- a/commands/default.py +++ b/commands/default.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Default: def __init__(self, register): @@ -9,18 +9,18 @@ class Default: toUnset = False if length == 1: optionMap = ["Viewing defaults"] - for i in config["Default"].keys(): - optionMap.append("%s: %s" % (i, config["Default"][i])) + for i in main.config["Default"].keys(): + optionMap.append("%s: %s" % (i, main.config["Default"][i])) info("\n".join(optionMap)) return elif length == 2: - if not spl[1] in config["Default"].keys(): + if not spl[1] in main.config["Default"].keys(): failure("No such key: %s" % spl[1]) return - info("%s: %s" % (spl[1], config["Default"][spl[1]])) + info("%s: %s" % (spl[1], main.config["Default"][spl[1]])) return elif length == 3: - if not spl[1] in config["Default"].keys(): + if not spl[1] in main.config["Default"].keys(): failure("No such key: %s" % spl[1]) return @@ -48,7 +48,7 @@ class Default: failure("Protocol must be ssl or plain, not %s" % spl[2]) return - if spl[2] == config["Default"][spl[1]]: + if spl[2] == main.config["Default"][spl[1]]: failure("Value already exists: %s" % spl[2]) return @@ -58,7 +58,7 @@ class Default: failure("Authtype must be sp or ns, not %s" % spl[2]) return if spl[1] == "enabled": - failure("Use the ConnectOnCreate config parameter to set this") + failure("Use the ConnectOnCreate main.config parameter to set this") return if spl[1] == "autojoin": if not toUnset: @@ -66,8 +66,8 @@ class Default: else: spl[2] = [] - config["Default"][spl[1]] = spl[2] - saveConf("config") + main.config["Default"][spl[1]] = spl[2] + main.saveConf("main.config") if toUnset: success("Successfully unset key %s" % spl[1]) else: diff --git a/commands/delete.py b/commands/delete.py index f47958a..18d2189 100644 --- a/commands/delete.py +++ b/commands/delete.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Delete: def __init__(self, register): @@ -7,20 +7,20 @@ class Delete: def delete(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return del pool[spl[1]] - if spl[1] in ReactorPool.keys(): - if spl[1] in FactoryPool.keys(): - FactoryPool[spl[1]].stopTrying() - ReactorPool[spl[1]].disconnect() - if spl[1] in IRCPool.keys(): - del IRCPool[spl[1]] - del ReactorPool[spl[1]] - del FactoryPool[spl[1]] + if spl[1] in main.ReactorPool.keys(): + if spl[1] in main.FactoryPool.keys(): + main.FactoryPool[spl[1]].stopTrying() + main.ReactorPool[spl[1]].disconnect() + if spl[1] in main.IRCPool.keys(): + del main.IRCPool[spl[1]] + del main.ReactorPool[spl[1]] + del main.FactoryPool[spl[1]] success("Successfully removed bot") - saveConf("pool") + main.saveConf("pool") return else: incUsage("del") diff --git a/commands/disable.py b/commands/disable.py index f5b8b39..681f155 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Disable: def __init__(self, register): @@ -7,19 +7,19 @@ class Disable: def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - pool[spl[1]]["enabled"] = False - saveConf("pool") - if spl[1] in ReactorPool.keys(): - if spl[1] in FactoryPool.keys(): - FactoryPool[spl[1]].stopTrying() - ReactorPool[spl[1]].disconnect() - if spl[1] in IRCPool.keys(): - del IRCPool[spl[1]] - del ReactorPool[spl[1]] - del FactoryPool[spl[1]] + main.pool[spl[1]]["enabled"] = False + main.saveConf("pool") + if spl[1] in main.ReactorPool.keys(): + if spl[1] in main.FactoryPool.keys(): + main.FactoryPool[spl[1]].stopTrying() + main.ReactorPool[spl[1]].disconnect() + if spl[1] in main.IRCPool.keys(): + del main.IRCPool[spl[1]] + del main.ReactorPool[spl[1]] + del main.FactoryPool[spl[1]] success("Successfully disabled bot %s" % spl[1]) return else: diff --git a/commands/dist.py b/commands/dist.py index 0d4c325..6fd579c 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -1,4 +1,4 @@ -from core.main import * +import main from subprocess import run, PIPE class Dist: @@ -7,9 +7,9 @@ class Dist: def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if config["Dist"]["Enabled"]: - rtrn = run([config["Dist"]["File"]], shell=True, stdout=PIPE) - if config["Dist"]["SendOutput"]: + if main.config["Dist"]["Enabled"]: + rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE) + if main.config["Dist"]["SendOutput"]: info("Exit code: %s -- Stdout: %s" % (rtrn.returncode, rtrn.stdout)) else: info("Exit code: %s" % rtrn.returncode) diff --git a/commands/enable.py b/commands/enable.py index 43ffba1..9078815 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -1,4 +1,4 @@ -from core.main import * +import main import core.helper as helper class Enable: @@ -11,9 +11,9 @@ class Enable: if not spl[1] in pool.keys(): failure("Name does not exist: %s" % spl[1]) return - pool[spl[1]]["enabled"] = True - saveConf("pool") - if not spl[1] in IRCPool.keys(): + main.pool[spl[1]]["enabled"] = True + main.saveConf("pool") + if not spl[1] in main.IRCPool.keys(): helper.addBot(spl[1]) else: pass diff --git a/commands/get.py b/commands/get.py index 1f44169..3c302b8 100644 --- a/commands/get.py +++ b/commands/get.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Get: def __init__(self, register): @@ -7,13 +7,13 @@ class Get: def get(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - if not spl[1] in IRCPool.keys(): + if not spl[1] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - info(str(IRCPool[spl[1]].get(spl[2]))) + info(str(main.IRCPool[spl[1]].get(spl[2]))) return else: incUsage("get") diff --git a/commands/help.py b/commands/help.py index e07b828..46a9782 100644 --- a/commands/help.py +++ b/commands/help.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Help: def __init__(self, register): @@ -7,8 +7,8 @@ class Help: def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: helpMap = [] - for i in help.keys(): - helpMap.append("%s: %s" % (i, help[i])) + for i in main.help.keys(): + helpMap.append("%s: %s" % (i, main.help[i])) info("\n".join(helpMap)) return else: diff --git a/commands/join.py b/commands/join.py index b68fcfd..3be5708 100644 --- a/commands/join.py +++ b/commands/join.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Join: def __init__(self, register): @@ -7,23 +7,23 @@ class Join: def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - if not spl[1] in IRCPool.keys(): + if not spl[1] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - IRCPool[spl[1]].join(spl[2]) + main.IRCPool[spl[1]].join(spl[2]) success("Joined %s" % spl[2]) return elif length == 4: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - if not spl[1] in IRCPool.keys(): + if not spl[1] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - IRCPool[spl[1]].join(spl[2], spl[3]) + main.IRCPool[spl[1]].join(spl[2], spl[3]) success("Joined %s with key %s" % (spl[2], spl[3])) return else: diff --git a/commands/key.py b/commands/key.py index 7a5ccb0..943d03e 100644 --- a/commands/key.py +++ b/commands/key.py @@ -1,4 +1,4 @@ -from core.main import * +import main import modules.keyword as keyword class Key: @@ -39,45 +39,45 @@ class Key: return if length == 4: if spl[1] == "except": - if not spl[2] in keyconf["Keywords"]: + if not spl[2] in main.keyconf["Keywords"]: failure("No such keyword: %s" % spl[2]) return - if spl[2] in keyconf["KeywordsExcept"].keys(): - if spl[3] in keyconf["KeywordsExcept"][spl[2]]: + if spl[2] in main.keyconf["KeywordsExcept"].keys(): + if spl[3] in main.keyconf["KeywordsExcept"][spl[2]]: failure("Exception exists: %s" % spl[3]) return else: if not spl[2] in spl[3]: failure("Keyword %s not in exception %s. This won't work" % (spl[2], spl[3])) return - keyconf["KeywordsExcept"][spl[2]] = [] + main.keyconf["KeywordsExcept"][spl[2]] = [] - keyconf["KeywordsExcept"][spl[2]].append(spl[3]) - saveConf("keyconf") + main.keyconf["KeywordsExcept"][spl[2]].append(spl[3]) + main.saveConf("keyconf") success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) return elif spl[1] == "master": - if not spl[2] in pool.keys(): + if not spl[2] in main.pool.keys(): failure("Name does not exist: %s" % spl[2]) return - if spl[2] in IRCPool.keys(): - if not spl[3] in IRCPool[spl[2]].channels: + if spl[2] in main.IRCPool.keys(): + if not spl[3] in main.IRCPool[spl[2]].channels: info("Bot not on channel: %s" % spl[3]) - config["Master"] = [spl[2], spl[3]] - saveConf("config") + main.config["Master"] = [spl[2], spl[3]] + main.saveConf("config") success("Master set to %s on %s" % (spl[3], spl[2])) return elif spl[1] == "unexcept": - if not spl[2] in keyconf["KeywordsExcept"].keys(): + if not spl[2] in main.keyconf["KeywordsExcept"].keys(): failure("No such exception: %s" % spl[2]) return - if not spl[3] in keyconf["KeywordsExcept"][spl[2]]: + if not spl[3] in main.keyconf["KeywordsExcept"][spl[2]]: failure("Exception %s has no attribute %s" % (spl[2], spl[3])) return - keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) - if keyconf["KeywordsExcept"][spl[2]] == []: - del keyconf["KeywordsExcept"][spl[2]] - saveConf("keyconf") + main.keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) + if main.keyconf["KeywordsExcept"][spl[2]] == []: + del main.keyconf["KeywordsExcept"][spl[2]] + main.saveConf("keyconf") success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) return else: @@ -85,32 +85,32 @@ class Key: return elif length == 3: if spl[1] == "unexcept": - if not spl[2] in keyconf["KeywordsExcept"].keys(): + if not spl[2] in main.keyconf["KeywordsExcept"].keys(): failure("No such exception: %s" % spl[2]) return - del keyconf["KeywordsExcept"][spl[2]] - saveConf("keyconf") + del main.keyconf["KeywordsExcept"][spl[2]] + main.saveConf("keyconf") success("Successfully removed exception list of %s" % spl[2]) return elif spl[1] == "monitor": if spl[2] == "on": - if not obj.addr in MonitorPool: - MonitorPool.append(obj.addr) + if not obj.addr in main.MonitorPool: + main.MonitorPool.append(obj.addr) success("Keyword monitoring enabled") - if len(masterbuf) == 0: + if len(main.masterbuf) == 0: return rtrn = [] - for i in range(len(masterbuf)): - rtrn.append(masterbuf.pop(0)) - saveConf("masterbuf") + for i in range(len(main.masterbuf)): + rtrn.append(main.masterbuf.pop(0)) + main.saveConf("masterbuf") info("\n".join(rtrn)) return else: failure("Keyword monitoring is already enabled") return elif spl[2] == "off": - if obj.addr in MonitorPool: - MonitorPool.remove(obj.addr) + if obj.addr in main.MonitorPool: + main.MonitorPool.remove(obj.addr) success("Keyword monitoring disabled") return else: @@ -124,22 +124,22 @@ class Key: return elif length == 2: if spl[1] == "show": - info(",".join(keyconf["Keywords"])) + info(",".join(main.keyconf["Keywords"])) return elif spl[1] == "showexcept": exceptMap = [] - for i in keyconf["KeywordsExcept"].keys(): + for i in main.keyconf["KeywordsExcept"].keys(): exceptMap.append("Key: %s" % i) - exceptMap.append("%s: %s" % (i, ",".join(keyconf["KeywordsExcept"][i]))) + exceptMap.append("%s: %s" % (i, ",".join(main.keyconf["KeywordsExcept"][i]))) exceptMap.append("\n") info("\n".join(exceptMap)) return elif spl[1] == "master": - info(" - ".join([str(i) for i in config["Master"]])) + info(" - ".join([str(i) for i in main.config["Master"]])) return elif spl[1] == "monitor": - if obj.addr in MonitorPool: + if obj.addr in main.MonitorPool: info("Keyword monitoring is enabled on this connection") return else: @@ -148,7 +148,6 @@ class Key: else: incUsage("key") return - else: incUsage("key") return diff --git a/commands/list.py b/commands/list.py index 996f5d1..000660e 100644 --- a/commands/list.py +++ b/commands/list.py @@ -1,4 +1,4 @@ -from core.main import * +import main class List: def __init__(self, register): @@ -7,10 +7,10 @@ class List: def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: poolMap = [] - for i in pool.keys(): + for i in main.pool.keys(): poolMap.append("Server: %s" % i) - for x in pool[i].keys(): - poolMap.append(" %s: %s" % (x, pool[i][x])) + for x in main.pool[i].keys(): + poolMap.append(" %s: %s" % (x, main.pool[i][x])) poolMap.append("\n") info("\n".join(poolMap)) return diff --git a/commands/load.py b/commands/load.py index ccb1428..91e168a 100644 --- a/commands/load.py +++ b/commands/load.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Load: def __init__(self, register): @@ -7,14 +7,14 @@ class Load: def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - if spl[1] in filemap.keys(): - loadConf(spl[1]) - success("Loaded %s from %s" % (spl[1], filemap[spl[1]][0])) + if spl[1] in main.filemap.keys(): + main.loadConf(spl[1]) + success("Loaded %s from %s" % (spl[1], main.filemap[spl[1]][0])) return elif spl[1] == "all": - for i in filemap.keys(): - loadConf(i) - success("Loaded %s from %s" % (i, filemap[i][0])) + for i in main.filemap.keys(): + main.loadConf(i) + success("Loaded %s from %s" % (i, main.filemap[i][0])) return else: incUsage("load") diff --git a/commands/loadmod.py b/commands/loadmod.py index 9fb3697..18a403e 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -1,4 +1,4 @@ -from core.main import * +import main from utils.loaders.single_loader import loadSingle class Loadmod: @@ -8,7 +8,7 @@ class Loadmod: def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - rtrn = loadSingle(spl[1], register) + rtrn = loadSingle(spl[1], main.register) if rtrn == True: success("Loaded module %s" % spl[1]) return diff --git a/commands/logout.py b/commands/logout.py index f95a256..a50ea02 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Logout: def __init__(self, register): @@ -7,8 +7,8 @@ class Logout: def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: obj.authed = False - if obj.addr in MonitorPool: - MonitorPool.remove(obj.addr) + if obj.addr in main.MonitorPool: + main.MonitorPool.remove(obj.addr) success("Logged out") return else: diff --git a/commands/mod.py b/commands/mod.py index 6bb408a..95af432 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Mod: def __init__(self, register): @@ -8,30 +8,30 @@ class Mod: if authed: toUnset = False if length == 2: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return optionMap = ["Viewing options for %s" % spl[1]] - for i in pool[spl[1]].keys(): - optionMap.append(" %s: %s" % (i, pool[spl[1]][i])) + for i in main.pool[spl[1]].keys(): + optionMap.append(" %s: %s" % (i, main.pool[spl[1]][i])) info("\n".join(optionMap)) return elif length == 3: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - if not spl[2] in pool[spl[1]].keys(): + if not spl[2] in main.pool[spl[1]].keys(): failure("No such key: %s" % spl[2]) return - info("%s: %s" % (spl[2], pool[spl[1]][spl[2]])) + info("%s: %s" % (spl[2], main.pool[spl[1]][spl[2]])) return elif length == 4: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - if not spl[2] in pool[spl[1]].keys(): + if not spl[2] in main.pool[spl[1]].keys(): failure("No such key: %s" % spl[2]) return @@ -39,7 +39,7 @@ class Mod: if not spl[3] in ["ssl", "plain"]: failure("Protocol must be ssl or plain, not %s" % spl[3]) return - if spl[3] == pool[spl[1]][spl[2]]: + if spl[3] == main.pool[spl[1]][spl[2]]: failure("Value already exists: %s" % spl[3]) return @@ -72,10 +72,10 @@ class Mod: if spl[2] == "autojoin": spl[3] = spl[3].split(",") - pool[spl[1]][spl[2]] = spl[3] - if spl[1] in IRCPool.keys(): - IRCPool[spl[1]].refresh() - saveConf("pool") + main.pool[spl[1]][spl[2]] = spl[3] + if spl[1] in main.IRCPool.keys(): + main.IRCPool[spl[1]].refresh() + main.saveConf("pool") if toUnset: success("Successfully unset key %s on %s" % (spl[2], spl[1])) else: diff --git a/commands/part.py b/commands/part.py index 0b145a9..a1adacb 100644 --- a/commands/part.py +++ b/commands/part.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Part: def __init__(self, register): @@ -7,13 +7,13 @@ class Part: def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - if not spl[1] in IRCPool.keys(): + if not spl[1] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - IRCPool[spl[1]].part(spl[2]) + main.IRCPool[spl[1]].part(spl[2]) success("Left %s" % spl[2]) return else: diff --git a/commands/password.py b/commands/password.py index b4b449e..6f88617 100644 --- a/commands/password.py +++ b/commands/password.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Password: def __init__(self, register): @@ -10,7 +10,7 @@ class Password: return else: if length == 2: - if spl[1] == config["Password"]: + if spl[1] == main.config["Password"]: success("Authenticated successfully") obj.authed = True return diff --git a/commands/save.py b/commands/save.py index 7fbb85c..aebec15 100644 --- a/commands/save.py +++ b/commands/save.py @@ -1,4 +1,4 @@ -from core.main import * +import main class Save: def __init__(self, register): @@ -7,14 +7,14 @@ class Save: def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - if spl[1] in filemap.keys(): - saveConf(spl[1]) - success("Saved %s to %s" % (spl[1], filemap[spl[1]][0])) + if spl[1] in main.filemap.keys(): + main.saveConf(spl[1]) + success("Saved %s to %s" % (spl[1], main.filemap[spl[1]][0])) return elif spl[1] == "all": - for i in filemap.keys(): - saveConf(i) - success("Saved %s to %s" % (i, filemap[i][0])) + for i in main.filemap.keys(): + main.saveConf(i) + success("Saved %s to %s" % (i, main.filemap[i][0])) return else: incUsage("save") diff --git a/commands/stats.py b/commands/stats.py index d1cc14e..342f389 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,4 +1,4 @@ -from core.main import * +import main import modules.counters as count class Stats: @@ -11,11 +11,11 @@ class Stats: stats = [] numChannels = 0 numWhoEntries = 0 - for i in IRCPool.keys(): - numChannels += len(IRCPool[i].channels) - for i in wholist.keys(): - numWhoEntries += len(wholist[i].keys()) - stats.append("Servers: %s" % len(IRCPool.keys())) + for i in main.IRCPool.keys(): + numChannels += len(main.IRCPool[i].channels) + for i in main.wholist.keys(): + numWhoEntries += len(main.wholist[i].keys()) + stats.append("Servers: %s" % len(main.IRCPool.keys())) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) counterEvents = count.getEvents() @@ -34,13 +34,13 @@ class Stats: numWhoEntries = 0 failures = 0 - if spl[1] in IRCPool.keys(): - numChannels += len(IRCPool[spl[1]].channels) + if spl[1] in main.IRCPool.keys(): + numChannels += len(main.IRCPool[spl[1]].channels) else: failure("Name does not exist: %s" % spl[1]) failures += 1 - if spl[1] in wholist.keys(): - numWhoEntries += len(wholist[spl[1]].keys()) + if spl[1] in main.wholist.keys(): + numWhoEntries += len(main.wholist[spl[1]].keys()) else: failure("Who entry does not exist: %s" % spl[1]) failures += 1 diff --git a/commands/who.py b/commands/who.py index 9a8256b..a2b4cfb 100644 --- a/commands/who.py +++ b/commands/who.py @@ -1,4 +1,4 @@ -from core.main import * +import main import modules.userinfo as userinfo class Who: diff --git a/conf/keyword.json b/conf/keyword.json index 3980412..b5e497e 100644 --- a/conf/keyword.json +++ b/conf/keyword.json @@ -1,6 +1,7 @@ { "Keywords": [ - "example" + "example", + "s" ], "KeywordsExcept": {} } \ No newline at end of file diff --git a/core/bot.py b/core/bot.py index a704ae7..60eeb8c 100644 --- a/core/bot.py +++ b/core/bot.py @@ -6,7 +6,7 @@ import modules.keyword as keyword import modules.userinfo as userinfo import modules.counters as count -from core.main import * +import main from utils.logging.log import * from utils.logging.send import * @@ -14,9 +14,9 @@ class IRCBot(IRCClient): def __init__(self, name): self.connected = False self.channels = [] - + self.buffer = "" self.name = name - instance = pool[name] + instance = main.pool[name] self.nickname = instance["nickname"] self.realname = instance["realname"] @@ -40,7 +40,7 @@ class IRCBot(IRCClient): self.password = instance["password"] def refresh(self): - instance = pool[self.name] + instance = main.pool[self.name] if not instance["nickname"] == self.nickname: self.nickname = instance["nickname"] self.setNick(self.nickname) @@ -69,21 +69,21 @@ class IRCBot(IRCClient): userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "privmsg") - keyword.actKeyword(user, channel, msg, self.nickname, "PRV", self.name) + keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "notice") - keyword.actKeyword(user, channel, msg, self.nickname, "NOT", self.name) + keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) def action(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "action") - keyword.actKeyword(user, channel, msg, self.nickname, "ACT", self.name) + keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) def get(self, var): try: @@ -148,10 +148,79 @@ class IRCBot(IRCClient): def got_who(self, whoinfo): userinfo.setWho(self.name, whoinfo[1]) + + #twisted sucks so i have to do this to actually get the user info + def irc_JOIN(self, prefix, params): + """ + Called when a user joins a channel. + """ + nick = prefix.split('!')[0] + channel = params[-1] + if nick == self.nickname: + self.joined(channel) + else: + self.userJoined(prefix, channel) + + def irc_PART(self, prefix, params): + """ + Called when a user leaves a channel. + """ + nick = prefix.split('!')[0] + channel = params[0] + if len(params) >= 2: + message = params[1] + else: + message = None + if nick == self.nickname: + self.left(channel, message) + else: + self.userLeft(prefix, channel, message) + + def irc_QUIT(self, prefix, params): + """ + Called when a user has quit. + """ + #nick = prefix.split('!')[0] + self.userQuit(prefix, params[0]) + + def irc_NICK(self, prefix, params): + """ + Called when a user changes their nickname. + """ + nick = prefix.split('!', 1)[0] + if nick == self.nickname: + self.nickChanged(params[0]) + else: + self.userRenamed(prefix, params[0]) + + def irc_KICK(self, prefix, params): + """ + Called when a user is kicked from a channel. + """ + #kicker = prefix.split('!')[0] + channel = params[0] + kicked = params[1] + message = params[-1] + if kicked.lower() == self.nickname.lower(): + # Yikes! + self.kickedFrom(channel, prefix, message) + else: + self.userKicked(kicked, channel, prefix, message) + + def irc_TOPIC(self, prefix, params): + """ + Someone in the channel set the topic. + """ + #user = prefix.split('!')[0] + channel = params[0] + newtopic = params[1] + self.topicUpdated(prefix, channel, newtopic) + #END hacks + def signedOn(self): self.connected = True log("signed on: %s" % self.name) - if config["ConnectionNotifications"]: + if main.config["ConnectionNotifications"]: keyword.sendMaster("SIGNON: %s" % self.name) if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) @@ -164,20 +233,21 @@ class IRCBot(IRCClient): self.channels.append(channel) self.who(channel).addCallback(self.got_who) count.event(self.name, "selfjoin") - if self.name == config["Master"][0] and channel == config["Master"][1]: - for i in range(len(masterbuf)): - self.msg(channel, masterbuf.pop(0)) - saveConf("masterbuf") + if self.name == main.config["Master"][0] and channel == main.config["Master"][1]: + for i in range(len(main.masterbuf)): + self.msg(channel, main.masterbuf.pop(0)) + main.saveConf("masterbuf") - def left(self, channel): + def left(self, channel, message): if channel in self.channels: self.channels.remove(channel) + keyword.actKeyword(user, channel, message, self.nickname, "SELFPART", self.name) count.event(self.name, "selfpart") def kickedFrom(self, channel, kicker, message): if channel in self.channels: self.channels.remove(channel) - keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) + keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.name, "selfkick") def userJoined(self, user, channel): @@ -185,9 +255,10 @@ class IRCBot(IRCClient): userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "join") - def userLeft(self, user, channel): + def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) + keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) count.event(self.name, "part") def userQuit(self, user, quitMessage): @@ -195,14 +266,14 @@ class IRCBot(IRCClient): userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "quit") - keyword.actKeyword(user, None, quitMessage, self.nickname, "QUT", self.name) + keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "kick") - keyword.actKeyword(kicker, channel, message, self.nickname, "KCK", self.name) + keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) @@ -215,7 +286,7 @@ class IRCBot(IRCClient): userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "topic") - keyword.actKeyword(user, channel, newTopic, self.nickname, "TOP", self.name) + keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) @@ -224,7 +295,7 @@ class IRCBot(IRCClient): class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): - self.instance = pool[name] + self.instance = main.pool[name] self.name = name self.client = None self.maxDelay = self.instance["maxdelay"] @@ -234,7 +305,7 @@ class IRCBotFactory(ReconnectingClientFactory): def buildProtocol(self, addr): entry = IRCBot(self.name) - IRCPool[self.name] = entry + main.IRCPool[self.name] = entry self.client = entry return entry @@ -245,7 +316,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) sendAll("%s: connection lost: %s" % (self.name, error)) - if config["ConnectionNotifications"]: + if main.config["ConnectionNotifications"]: keyword.sendMaster("CONNLOST %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) diff --git a/core/helper.py b/core/helper.py index 06a4c80..98e6364 100644 --- a/core/helper.py +++ b/core/helper.py @@ -2,43 +2,43 @@ from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory from core.bot import IRCBot, IRCBotFactory -from core.main import * +import main from utils.logging.log import * def addBot(name): - instance = pool[name] + instance = main.pool[name] log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) if instance["protocol"] == "plain": if instance["bind"] == None: bot = IRCBotFactory(name) rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"])) - ReactorPool[name] = rct - FactoryPool[name] = bot + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot return else: bot = IRCBotFactory(name) rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) - ReactorPool[name] = rct - FactoryPool[name] = bot + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot return elif instance["protocol"] == "ssl": - keyFN = certPath+instance["key"] - certFN = certPath+instance["certificate"] + keyFN = main.certPath+instance["key"] + certFN = main.certPath+instance["certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) if instance["bind"] == None: bot = IRCBotFactory(name) rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory) - ReactorPool[name] = rct - FactoryPool[name] = bot + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot return else: bot = IRCBotFactory(name) rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory, bindAddress=instance["bind"]) - ReactorPool[name] = rct - FactoryPool[name] = bot + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot return diff --git a/core/parser.py b/core/parser.py index ecfd18a..a8a001e 100644 --- a/core/parser.py +++ b/core/parser.py @@ -1,12 +1,12 @@ -from core.main import * +import main from utils.logging.log import * from utils.logging.send import * def parseCommand(addr, authed, data): #call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length) spl = data.split() - if addr in connections.keys(): - obj = connections[addr] + if addr in main.connections.keys(): + obj = main.connections[addr] else: warn("Got connection object with no instance in the address pool") return @@ -22,9 +22,9 @@ def parseCommand(addr, authed, data): else: failure("No text was sent") return - for i in CommandMap.keys(): + for i in main.CommandMap.keys(): if spl[0] == i: - CommandMap[i](addr, authed, data, obj, spl, success, failure, info, incUsage, length) + main.CommandMap[i](addr, authed, data, obj, spl, success, failure, info, incUsage, length) return incUsage(None) return diff --git a/core/server.py b/core/server.py index 4aa81cf..2f40ccf 100644 --- a/core/server.py +++ b/core/server.py @@ -1,5 +1,5 @@ from twisted.internet.protocol import Protocol, Factory, ClientFactory -from core.main import * +import main from utils.logging.log import * from core.parser import parseCommand @@ -8,7 +8,7 @@ class Server(Protocol): def __init__(self, addr): self.addr = addr self.authed = False - if config["UsePassword"] == False: + if main.config["UsePassword"] == False: self.authed = True def send(self, data): @@ -34,22 +34,22 @@ class Server(Protocol): def connectionLost(self, reason): self.authed = False log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) - if self.addr in connections.keys(): - del connections[self.addr] + if self.addr in main.connections.keys(): + del main.connections[self.addr] else: warn("Tried to remove a non-existant connection.") - if self.addr in MonitorPool: - MonitorPool.remove(self.addr) + if self.addr in main.MonitorPool: + main.MonitorPool.remove(self.addr) class ServerFactory(Factory): def buildProtocol(self, addr): entry = Server(addr) - connections[addr] = entry + main.connections[addr] = entry return entry def send(self, addr, data): - if addr in connections.keys(): - connection = connections[addr] + if addr in main.connections.keys(): + connection = main.connections[addr] connection.send(data) else: return diff --git a/core/main.py b/main.py similarity index 100% rename from core/main.py rename to main.py diff --git a/modules/counters.py b/modules/counters.py index d832d46..56a5e10 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -1,29 +1,29 @@ -from core.main import * +import main def event(name, eventType): - if not "local" in counters.keys(): - counters["local"] = {} - if not "global" in counters.keys(): - counters["global"] = {} - if not name in counters["local"].keys(): - counters["local"][name] = {} - if eventType not in counters["local"][name].keys(): - counters["local"][name][eventType] = 0 + if not "local" in main.counters.keys(): + main.counters["local"] = {} + if not "global" in main.counters.keys(): + main.counters["global"] = {} + if not name in main.counters["local"].keys(): + main.counters["local"][name] = {} + if eventType not in main.counters["local"][name].keys(): + main.counters["local"][name][eventType] = 0 - if eventType not in counters["global"]: - counters["global"][eventType] = 0 + if eventType not in main.counters["global"]: + main.counters["global"][eventType] = 0 - counters["local"][name][eventType] += 1 - counters["global"][eventType] += 1 + main.counters["local"][name][eventType] += 1 + main.counters["global"][eventType] += 1 def getEvents(name=None): if name == None: - if "global" in counters.keys(): - return counters["global"] + if "global" in main.counters.keys(): + return main.counters["global"] else: return None else: - if name in counters["local"].keys(): - return counters["local"][name] + if name in main.counters["local"].keys(): + return main.counters["local"][name] else: return None diff --git a/modules/keyword.py b/modules/keyword.py index 25207bc..83205a6 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -1,38 +1,38 @@ -from core.main import * +import main from utils.logging.log import * import modules.counters as count def sendMaster(data): - if not len(MonitorPool) == 0: + if not len(main.MonitorPool) == 0: hasMonitors = True else: hasMonitors = False - if config["Master"] == [None, None]: + if main.config["Master"] == [None, None]: if hasMonitors: - for i in MonitorPool: - connections[i].send(data) + for i in main.MonitorPool: + main.connections[i].send(data) return else: - masterbuf.append(data) - saveConf("masterbuf") + main.masterbuf.append(data) + main.saveConf("masterbuf") return - if config["Master"][0] in IRCPool.keys(): - if config["Master"][1] in IRCPool[config["Master"][0]].channels: - IRCPool[config["Master"][0]].msg(config["Master"][1], data) + if main.config["Master"][0] in main.IRCPool.keys(): + if main.config["Master"][1] in main.IRCPool[main.config["Master"][0]].channels: + main.IRCPool[main.config["Master"][0]].msg(main.config["Master"][1], data) else: if not hasMonitors: - masterbuf.append(data) - saveConf("masterbuf") + main.masterbuf.append(data) + main.saveConf("masterbuf") else: if not hasMonitors: - masterbuf.append(data) - saveConf("masterbuf") + main.masterbuf.append(data) + main.saveConf("masterbuf") warn("Master with no IRC instance defined") - for i in MonitorPool: - connections[i].send(data) + for i in main.MonitorPool: + main.connections[i].send(data) def isKeyword(msg): message = msg.lower() @@ -40,10 +40,10 @@ def isKeyword(msg): toUndo = False uniqueNum = 0 totalNum = 0 - for i in keyconf["Keywords"]: + for i in main.keyconf["Keywords"]: if i in message: - if i in keyconf["KeywordsExcept"].keys(): - for x in keyconf["KeywordsExcept"][i]: + if i in main.keyconf["KeywordsExcept"].keys(): + for x in main.keyconf["KeywordsExcept"][i]: if x in message: toUndo = True messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") @@ -68,10 +68,10 @@ def isKeyword(msg): def actKeyword(user, channel, message, nickname, actType, name): toSend = isKeyword(message) - if name == config["Master"][0] and channel == config["Master"][1]: + if name == main.config["Master"][0] and channel == main.config["Master"][1]: pass else: - if config["HighlightNotifications"]: + if main.config["HighlightNotifications"]: msgLower = message.lower() nickLower = nickname.lower() if nickLower in msgLower: @@ -83,19 +83,19 @@ def actKeyword(user, channel, message, nickname, actType, name): count.event(name, "keymatch") def addKeyword(keyword): - if keyword in keyconf["Keywords"]: + if keyword in main.keyconf["Keywords"]: return "EXISTS" else: - for i in keyconf["Keywords"]: + for i in main.keyconf["Keywords"]: if i in keyword or keyword in i: return "ISIN" - keyconf["Keywords"].append(keyword) - saveConf("keyconf") + main.keyconf["Keywords"].append(keyword) + main.saveConf("keyconf") return True def delKeyword(keyword): - if not keyword in keyconf["Keywords"]: + if not keyword in main.keyconf["Keywords"]: return "NOKEY" - keyconf["Keywords"].remove(keyword) - saveConf("keyconf") + main.keyconf["Keywords"].remove(keyword) + main.saveConf("keyconf") return True diff --git a/modules/userinfo.py b/modules/userinfo.py index ad7fe04..d4acba2 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,32 +1,34 @@ -from core.main import * +import main from string import digits #from utils.logging.log import * def setWho(network, newObjects): network = "".join([x for x in network if not x in digits]) - if not network in wholist.keys(): - wholist[network] = {} + if not network in main.wholist.keys(): + main.wholist[network] = {} for i in newObjects.keys(): - wholist[network][i] = newObjects[i] + main.wholist[network][i] = newObjects[i] return def setWhoSingle(network, nick, ident, host): network = "".join([x for x in network if not x in digits]) - if network in wholist.keys(): - if nick in wholist[network].keys(): - wholist[network][nick][1] = ident - wholist[network][nick][2] = host + if network in main.wholist.keys(): + if nick in main.wholist[network].keys(): + main.wholist[network][nick][1] = ident + main.wholist[network][nick][2] = host else: - wholist[network][nick] = [nick, ident, host, None, None, None] + main.wholist[network][nick] = [nick, ident, host, None, None, None] + else: + main.wholist[network] = {nick: [nick, ident, host, None, None, None]} def getWho(nick): result = {} - for i in wholist.keys(): - for x in wholist[i].keys(): + for i in main.wholist.keys(): + for x in main.wholist[i].keys(): if nick.lower() == x.lower(): if not i in result.keys(): result[i] = [] - result[i].append(wholist[i][x]) + result[i].append(main.wholist[i][x]) return result diff --git a/threshold b/threshold index f83795b..71872a5 100755 --- a/threshold +++ b/threshold @@ -6,29 +6,9 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory #from sys import stdout #log.startLogging(stdout) -from core.main import ( - configPath, - certPath, - connections, - IRCPool, - ReactorPool, - FactoryPool, - MonitorPool, - saveConf, - loadConf, - initMain, - ) +import main -initMain() - -from core.main import ( - config, - keyconf, - pool, - help, - wholist, - counters, - ) +main.initMain() from utils.logging.log import * import modules.userinfo as userinfo @@ -37,14 +17,14 @@ from core.server import Server, ServerFactory if __name__ == "__main__": listener = ServerFactory() - if config["Listener"]["UseSSL"] == True: - reactor.listenSSL(config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(certPath+config["Listener"]["Key"], certPath+config["Listener"]["Certificate"]), interface=config["Listener"]["Address"]) - log("Threshold running with SSL on %s:%s" % (config["Listener"]["Address"], config["Listener"]["Port"])) + if main.config["Listener"]["UseSSL"] == True: + reactor.listenSSL(main.config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(main.certPath+main.config["Listener"]["Key"], main.certPath+main.config["Listener"]["Certificate"]), interface=main.config["Listener"]["Address"]) + log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) else: - reactor.listenTCP(config["Listener"]["Port"], listener, interface=config["Listener"]["Address"]) - log("Threshold running on %s:%s" % (config["Listener"]["Address"], config["Listener"]["Port"])) - for i in pool.keys(): - if pool[i]["enabled"] == True: + reactor.listenTCP(main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"]) + log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) + for i in main.pool.keys(): + if main.pool[i]["enabled"] == True: helper.addBot(i) reactor.run() diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 61eb93e..62bc734 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,5 +1,4 @@ from os import listdir -from core.main import * from utils.logging.log import * import commands diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index 11de8ef..543a6b5 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -1,12 +1,12 @@ from os import listdir -from core.main import * +import main from utils.logging.log import * import commands def loadSingle(command, func): if command+".py" in listdir("commands"): try: - if command in CommandMap.keys(): + if command in main.CommandMap.keys(): return "Cannot reload modules" else: className = command.capitalize() diff --git a/utils/logging/send.py b/utils/logging/send.py index 2f894f1..dd6bfab 100644 --- a/utils/logging/send.py +++ b/utils/logging/send.py @@ -1,7 +1,7 @@ -from core.main import connections, help +import main def sendData(addr, data): - connections[addr].send(data) + main.connections[addr].send(data) def sendSuccess(addr, data): sendData(addr, "[y] " + data) @@ -13,14 +13,14 @@ def sendInfo(addr, data): sendData(addr, "[i] " + data) def sendAll(data): - for i in connections: - connections[i].send(data) + for i in main.connections: + main.connections[i].send(data) return def incorrectUsage(addr, mode): if mode == None: sendFailure(addr, "Incorrect usage") return - if mode in help.keys(): - sendFailure(addr, "Usage: " + help[mode]) + if mode in main.help.keys(): + sendFailure(addr, "Usage: " + main.help[mode]) return From 369ddbe9399053a1ccf8499c7bf3f599b7a226cf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 14 Mar 2018 20:20:53 +0000 Subject: [PATCH 076/394] Additional checks on keyword matches to prevent errors when no part message is specified --- conf/keyword.json | 7 ------- modules/keyword.py | 4 ++++ 2 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 conf/keyword.json diff --git a/conf/keyword.json b/conf/keyword.json deleted file mode 100644 index b5e497e..0000000 --- a/conf/keyword.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "Keywords": [ - "example", - "s" - ], - "KeywordsExcept": {} -} \ No newline at end of file diff --git a/modules/keyword.py b/modules/keyword.py index 83205a6..36e9f9b 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -35,6 +35,8 @@ def sendMaster(data): main.connections[i].send(data) def isKeyword(msg): + if msg == None: + return message = msg.lower() messageDuplicate = message toUndo = False @@ -67,6 +69,8 @@ def isKeyword(msg): return [message, uniqueNum, totalNum] def actKeyword(user, channel, message, nickname, actType, name): + if message == None: + return toSend = isKeyword(message) if name == main.config["Master"][0] and channel == main.config["Master"][1]: pass From 7de25e44f40d5303398e52b58e6561b86c1141f8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 21 Apr 2018 14:38:07 +0100 Subject: [PATCH 077/394] Fix typo --- commands/add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/add.py b/commands/add.py index 1351fd1..946bd8a 100644 --- a/commands/add.py +++ b/commands/add.py @@ -101,7 +101,7 @@ class Add: "certificate": main.config["Default"]["certificate"], "enabled": main.config["ConnectOnCreate"], } - if config["ConnectOnCreate"] == True: + if main.config["ConnectOnCreate"] == True: helper.addBot(name) success("Successfully created bot") main.saveConf("pool") From 66d450130315399d8960e45d61ba3b098e9e93f0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 21 Apr 2018 14:44:07 +0100 Subject: [PATCH 078/394] Fix another typo --- commands/enable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/enable.py b/commands/enable.py index 9078815..3fc7a2d 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -8,7 +8,7 @@ class Enable: def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - if not spl[1] in pool.keys(): + if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return main.pool[spl[1]]["enabled"] = True From 2e2e35cad71a402b385c4814f48255069be24c58 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 21 Apr 2018 15:05:32 +0100 Subject: [PATCH 079/394] Fix yet another typo --- core/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index 60eeb8c..68250a8 100644 --- a/core/bot.py +++ b/core/bot.py @@ -328,7 +328,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) sendAll("%s: connection failed: %s" % (self.name, error)) - if config["ConnectionNotifications"]: + if main.config["ConnectionNotifications"]: keyword.sendMaster("CONNFAIL %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) From 66e7785f6f2e7bcffaa6d62dcd6140906b935f29 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 7 May 2018 19:58:19 +0100 Subject: [PATCH 080/394] Implement query notifications and recognise ZNC's messages --- commands/add.py | 7 ++++--- commands/delete.py | 2 +- commands/key.py | 2 +- commands/msg.py | 25 +++++++++++++++++++++++++ conf/example/config.json | 15 +++++++++++++-- conf/help.json | 7 ++++--- core/bot.py | 6 +++--- modules/keyword.py | 29 +++++++++++++++++++++++------ 8 files changed, 74 insertions(+), 19 deletions(-) create mode 100644 commands/msg.py diff --git a/commands/add.py b/commands/add.py index 946bd8a..e2eac5d 100644 --- a/commands/add.py +++ b/commands/add.py @@ -13,6 +13,9 @@ class Add: if length > 1: name = spl[1] + if name.isdigit(): + failure("Network name is all numbers. This will break things.") + return else: incUsage("add") return @@ -71,9 +74,7 @@ class Add: failure("Protocol must be ssl or plain, not %s" % protocol) return - try: - int(port) - except: + if not port.isdigit(): failure("Port must be an integer, not %s" % port) return diff --git a/commands/delete.py b/commands/delete.py index 18d2189..fb57b71 100644 --- a/commands/delete.py +++ b/commands/delete.py @@ -10,7 +10,7 @@ class Delete: if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - del pool[spl[1]] + del main.pool[spl[1]] if spl[1] in main.ReactorPool.keys(): if spl[1] in main.FactoryPool.keys(): main.FactoryPool[spl[1]].stopTrying() diff --git a/commands/key.py b/commands/key.py index 943d03e..20f1b2a 100644 --- a/commands/key.py +++ b/commands/key.py @@ -48,7 +48,7 @@ class Key: return else: if not spl[2] in spl[3]: - failure("Keyword %s not in exception %s. This won't work" % (spl[2], spl[3])) + failure("Keyword %s not in exception %s. This won't work." % (spl[2], spl[3])) return main.keyconf["KeywordsExcept"][spl[2]] = [] diff --git a/commands/msg.py b/commands/msg.py new file mode 100644 index 0000000..a3bad7e --- /dev/null +++ b/commands/msg.py @@ -0,0 +1,25 @@ +import main + +class Msg: + def __init__(self, register): + register("msg", self.msg) + + def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length >= 4: + if not spl[1] in main.pool.keys(): + failure("Name does not exist: %s" % spl[1]) + return + if not spl[1] in main.IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]) + return + if not spl[2] in main.IRCPool[spl[1]].channels: + info("Bot not on channel: %s" % spl[2]) + main.IRCPool[spl[1]].msg(spl[2], " ".join(spl[3:])) + success("Sent %s to %s on %s" % (" ".join(spl[3:]), spl[2], spl[1])) + return + else: + incUsage("msg") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index f7bc320..ecd781d 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -8,14 +8,25 @@ }, "UsePassword": true, "ConnectOnCreate": false, - "HighlightNotifications": true, - "ConnectionNotifications": true, + "Notifications": { + "Highlight": true, + "Connection": true, + "Query": true + }, + "Compat": { + "ZNC": true + }, "Dist": { "Enabled": true, "SendOutput": false, "File": "conf/dist.sh" }, "Password": "s", + "Tweaks": { + "ZNC": { + "Prefix": "*" + } + }, "Default": { "host": null, "port": null, diff --git a/conf/help.json b/conf/help.json index e251599..f0b512a 100644 --- a/conf/help.json +++ b/conf/help.json @@ -6,7 +6,7 @@ "mod": "mod [] []", "default": "default [] []", "get": "get ", - "key": "key [] [] [] []", + "key": "key [] [] [] []", "who": "who ", "join": "join []", "enable": "enable ", @@ -16,5 +16,6 @@ "save": "save ", "load": "load ", "dist": "dist", - "loadmod": "loadmod " -} \ No newline at end of file + "loadmod": "loadmod ", + "msg": "msg " +} diff --git a/core/bot.py b/core/bot.py index 68250a8..4c9cbc7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -220,7 +220,7 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True log("signed on: %s" % self.name) - if main.config["ConnectionNotifications"]: + if main.config["Notifications"]["Connection"]: keyword.sendMaster("SIGNON: %s" % self.name) if self.authtype == "ns": self.msg(self.authentity, "IDENTIFY %s" % self.nspass) @@ -316,7 +316,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) sendAll("%s: connection lost: %s" % (self.name, error)) - if main.config["ConnectionNotifications"]: + if main.config["Notifications"]["Connection"]: keyword.sendMaster("CONNLOST %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -328,7 +328,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) sendAll("%s: connection failed: %s" % (self.name, error)) - if main.config["ConnectionNotifications"]: + if main.config["Notifications"]["Connection"]: keyword.sendMaster("CONNFAIL %s: %s" % (self.name, error)) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/modules/keyword.py b/modules/keyword.py index 36e9f9b..4d7cd9c 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -75,13 +75,30 @@ def actKeyword(user, channel, message, nickname, actType, name): if name == main.config["Master"][0] and channel == main.config["Master"][1]: pass else: - if main.config["HighlightNotifications"]: - msgLower = message.lower() - nickLower = nickname.lower() - if nickLower in msgLower: - msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + msgLower = message.lower() + nickLower = nickname.lower() + if nickLower in msgLower: # Someone has said the bot's nickname + msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") + count.event(name, "nickhighlight") + if main.config["Notifications"]["Highlight"]: sendMaster("NICK %s %s (T:%s): (%s/%s) %s" % (actType, name, msgLower.count(nickLower), user, channel, msgLower)) - count.event(name, "nickhighlight") + if not channel == None: + chanLower = channel.lower() + if nickLower == chanLower: # I received a message directed only at me + ZNCAlreadySent = False + if main.config["Notifications"]["Query"]: + if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in": + if main.config["Compat"]["ZNC"]: + sendMaster("ZNC %s %s: (%s/%s) %s" % (actType, name, user, channel, msgLower)) + ZNCAlreadySent = True + else: + sendMaster("QUERY %s %s: (%s) %s" % (actType, name, user, msgLower)) + else: + sendMaster("QUERY %s %s: (%s) %s" % (actType, name, user, msgLower)) + if not ZNCAlreadySent == True: + if main.config["Compat"]["ZNC"]: + if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in": + sendMaster("ZNC %s %s: (%s/%s) %s" % (actType, name, user, channel, msgLower)) if toSend: sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) count.event(name, "keymatch") From bc87ffddf714b90f2a6ca97b84a791a832087c8b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 27 Jul 2018 22:58:37 +0100 Subject: [PATCH 081/394] Implement monitoring system for flexible metadata matching --- commands/delete.py | 2 +- commands/key.py | 4 +- commands/mon.py | 236 +++++++++++++++++++++++++++++++++++ conf/example/mon.json | 1 + conf/example/monitor.json | 1 + conf/help.json | 10 +- core/bot.py | 19 ++- main.py | 3 +- modules/keyword.py | 4 +- modules/monitor.py | 36 ++++++ utils/fileutil/read_line.py | 0 utils/fileutil/write_line.py | 0 12 files changed, 305 insertions(+), 11 deletions(-) create mode 100644 commands/mon.py create mode 100644 conf/example/mon.json create mode 100644 conf/example/monitor.json create mode 100644 modules/monitor.py delete mode 100644 utils/fileutil/read_line.py delete mode 100644 utils/fileutil/write_line.py diff --git a/commands/delete.py b/commands/delete.py index fb57b71..2476363 100644 --- a/commands/delete.py +++ b/commands/delete.py @@ -19,7 +19,7 @@ class Delete: del main.IRCPool[spl[1]] del main.ReactorPool[spl[1]] del main.FactoryPool[spl[1]] - success("Successfully removed bot") + success("Successfully removed bot: %s" % spl[1]) main.saveConf("pool") return else: diff --git a/commands/key.py b/commands/key.py index 20f1b2a..e8fb4db 100644 --- a/commands/key.py +++ b/commands/key.py @@ -123,11 +123,11 @@ class Key: incUsage("key") return elif length == 2: - if spl[1] == "show": + if spl[1] == "list": info(",".join(main.keyconf["Keywords"])) return - elif spl[1] == "showexcept": + elif spl[1] == "listexcept": exceptMap = [] for i in main.keyconf["KeywordsExcept"].keys(): exceptMap.append("Key: %s" % i) diff --git a/commands/mon.py b/commands/mon.py new file mode 100644 index 0000000..9be3997 --- /dev/null +++ b/commands/mon.py @@ -0,0 +1,236 @@ +import main +import argparse +import sys +from io import StringIO +from pprint import pformat + +class Mon: + def __init__(self, register): + register("mon", self.mon) + + def setup_arguments(self, ArgumentParser): + self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.") + group1 = self.parser.add_mutually_exclusive_group(required=True) + group1.add_argument("--add", metavar="entry", dest="addEntry", help="Add an entry") + group1.add_argument("--del", metavar="entry", dest="delEntry", help="Delete an entry") + group1.add_argument("--list", action="store_true", dest="listEntry", help="List all entries") + group1.add_argument("--mod", metavar="entry", dest="modEntry", help="Modify an entry") + + group2 = self.parser.add_mutually_exclusive_group() + group2.add_argument("--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") + group2.add_argument("--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") + + self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, any") + self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") + self.parser.add_argument("--exact", nargs="*", metavar="query", dest="exact", help="Use exact matching") + self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching") + self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching") + self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching") + self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching (only works on WHO)") + + self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)") + self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action") + self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick") + self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode") + self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)") + + def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + # We need to override the ArgumentParser class because + # it's only meant for CLI applications and exits + # after running, so we override the exit function + # to do nothing. We also need to do it here in order + # to catch the error messages and show them to the user. + # It sucks. I know. + if authed: + class ArgumentParser(argparse.ArgumentParser): + def exit(self, status=0, message=None): + if message: + failure(message) + self.setup_arguments(ArgumentParser) + if length == 1: + info("Incorrect usage, use mon -h for help") + return + old_stdout = sys.stdout + old_stderr = sys.stderr + my_stdout = sys.stdout = StringIO() + my_stderr = sys.stderr = StringIO() + try: + parsed = self.parser.parse_args(spl[1:]) + except: + return + sys.stdout = old_stdout + sys.stderr = old_stderr + stdout = my_stdout.getvalue() + stderr = my_stdout.getvalue() + if not stdout == "": + info(stdout) + elif not stderr == "": + failure(my_stdout.getvalue()) + return + my_stdout.close() + my_stderr.close() + + if parsed.addEntry: + if parsed.addEntry in main.monitor.keys(): + failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry) + return + cast = self.makeCast(parsed, failure, info) + if cast == False: + return + main.monitor[parsed.addEntry] = cast + main.saveConf("monitor") + success("Successfully created monitor group %s" % parsed.addEntry) + return + elif parsed.delEntry: + if not parsed.delEntry in main.monitor.keys(): + failure("No such monitor group: %s" % parsed.delEntry) + return + del main.monitor[parsed.delEntry] + main.saveConf("monitor") + success("Successfully removed monitor group: %s" % parsed.delEntry) + return + elif parsed.modEntry: + if not parsed.modEntry in main.monitor.keys(): + failure("No such monitor group: %s" % parsed.modEntry) + return + cast = self.makeCast(parsed, failure, info) + if cast == False: + return + if parsed.doAppend: + merged = self.addCast(main.monitor[parsed.modEntry], cast, info) + main.monitor[parsed.modEntry] = merged + elif parsed.doRemove: + merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info) + main.monitor[parsed.modEntry] = merged + else: + failure("Specify --append or --remove with --mod") + return + main.saveConf("monitor") + success("Successfully updated entry %s" % parsed.modEntry) + return + elif parsed.listEntry: + info(pformat(main.monitor)) + return + + else: + incUsage(None) + + def parseNetworkFormat(self, lst, failure, info): + dedup = lambda x: list(set(x)) + cast = {} + if lst == None: + return "nil" + if len(lst) == 0: + failure("List has no entries") + return False + elif len(lst) > 0: + if len(lst[0]) == 0: + failure("Nested list has no entries") + return False + for i in lst: + if not i[0] in main.pool.keys(): + failure("Name does not exist: %s" % i[0]) + return False + if i[0] in main.IRCPool.keys(): + for x in i[1:]: + if not x in main.IRCPool[i[0]].channels: + info("%s: Bot not on channel: %s" % (i[0], x)) + if len(i) == 1: + cast[i[0]] = True + else: + if i[0] in cast.keys(): + if not cast[i[0]] == True: + for x in dedup(i[1:]): + cast[i[0]].append(x) + else: + cast[i[0]] = dedup(i[1:]) + else: + cast[i[0]] = dedup(i[1:]) + for i in cast.keys(): + deduped = dedup(cast[i]) + cast[i] = deduped + return cast + + # Create or modify a monitor group magically + def makeCast(self, obj, failure, info): + dedup = lambda x: list(set(x)) + validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "any"] + cast = {} + if not obj.specType == None: + types = dedup(obj.specType) + for i in types: + if not i in validTypes: + failure("Invalid type: %s" % i) + info("Available types: %s" % ", ".join(validTypes)) + return False + cast["type"] = types + if not obj.source == None: + sourceParse = self.parseNetworkFormat(obj.source, failure, info) + if not sourceParse: + return False + if not sourceParse == "nil": + cast["sources"] = sourceParse + + if not obj.send == None: + sendParse = self.parseNetworkFormat(obj.send, failure, info) + if not sendParse: + return False + if not sendParse == "nil": + cast["send"] = sendParse + + if not obj.message == None: + cast["message"] = [] + for i in obj.message: + cast["message"].append(" ".join(i)) + + if not obj.user == None: + cast["user"] = obj.user + if not obj.modes == None: + cast["modes"] = obj.modes + if not obj.free == None: + cast["free"] = obj.free + if not obj.exact == None: + cast["exact"] = obj.exact + if not obj.nick == None: + cast["nick"] = obj.nick + if not obj.ident == None: + cast["ident"] = obj.ident + if not obj.host == None: + cast["host"] = obj.host + if not obj.real == None: + cast["real"] = [] + for i in obj.real: + cast["real"].append(" ".join(i)) + return cast + + def subtractCast(self, source, patch, info): + for i in patch.keys(): + if i in source.keys(): + if isinstance(source[i], dict): + result = self.subtractCast(source[i], patch[i], info) + source[i] = result + continue + for x in patch[i]: + if x in source[i]: + source[i].remove(x) + else: + info("Element %s not in source %s" % (x, i)) + else: + info("Non-matched key: %s" % i) + return source + + def addCast(self, source, patch, info): + for i in patch.keys(): + if i in source.keys(): + if isinstance(source[i], dict): + result = self.addCast(source[i], patch[i], info) + source[i] = result + continue + for x in patch[i]: + if x in source[i]: + info("Element %s already in source %s" % (x, i)) + else: + source[i].append(x) + else: + source[i] = patch[i] + return source diff --git a/conf/example/mon.json b/conf/example/mon.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/mon.json @@ -0,0 +1 @@ +{} diff --git a/conf/example/monitor.json b/conf/example/monitor.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/monitor.json @@ -0,0 +1 @@ +{} diff --git a/conf/help.json b/conf/help.json index f0b512a..a69df80 100644 --- a/conf/help.json +++ b/conf/help.json @@ -6,16 +6,18 @@ "mod": "mod [] []", "default": "default [] []", "get": "get ", - "key": "key [] [] [] []", + "key": "key [] [] [] []", "who": "who ", "join": "join []", + "part": "part ", "enable": "enable ", "disable": "disable ", "list": "list", "stats": "stats []", - "save": "save ", - "load": "load ", + "save": "save ", + "load": "load ", "dist": "dist", "loadmod": "loadmod ", - "msg": "msg " + "msg": "msg ", + "mon": "mon -h" } diff --git a/core/bot.py b/core/bot.py index 4c9cbc7..633d2b8 100644 --- a/core/bot.py +++ b/core/bot.py @@ -5,6 +5,7 @@ from twisted.internet.defer import Deferred import modules.keyword as keyword import modules.userinfo as userinfo import modules.counters as count +import modules.monitor as monitor import main from utils.logging.log import * @@ -70,6 +71,7 @@ class IRCBot(IRCClient): count.event(self.name, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) + monitor.event(self.name, channel, {"type": "msg", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) @@ -77,6 +79,7 @@ class IRCBot(IRCClient): count.event(self.name, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) + monitor.event(self.name, channel, {"type": "notice", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) def action(self, user, channel, msg): nick, ident, host = self.parsen(user) @@ -84,6 +87,7 @@ class IRCBot(IRCClient): count.event(self.name, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) + monitor.event(self.name, channel, {"type": "action", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) def get(self, var): try: @@ -241,25 +245,32 @@ class IRCBot(IRCClient): def left(self, channel, message): if channel in self.channels: self.channels.remove(channel) - keyword.actKeyword(user, channel, message, self.nickname, "SELFPART", self.name) + keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) count.event(self.name, "selfpart") + monitor.event(self.name, channel, {"type": "part", "message": message}) def kickedFrom(self, channel, kicker, message): + nick, ident, host = self.parsen(kicker) + print(kicker) + userinfo.setWhoSingle(self.name, nick, ident, host) if channel in self.channels: self.channels.remove(channel) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.name, "selfkick") + monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "join") + monitor.event(self.name, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) count.event(self.name, "part") + monitor.event(self.name, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) @@ -267,6 +278,7 @@ class IRCBot(IRCClient): count.event(self.name, "quit") keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) + monitor.event(self.name, None, {"type": "quit", "exact": user, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) @@ -274,12 +286,14 @@ class IRCBot(IRCClient): count.event(self.name, "kick") keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) + monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message, "user": kickee}) def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) userinfo.setWhoSingle(self.name, nick, ident, host) userinfo.setWhoSingle(self.name, newname, ident, host) count.event(self.name, "nick") + monitor.event(self.name, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) @@ -287,11 +301,14 @@ class IRCBot(IRCClient): count.event(self.name, "topic") keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) + monitor.event(self.name, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "mode") + print("%s // %s // %s" % (toset, modes, args)) + #monitor.event(self.name, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): diff --git a/main.py b/main.py index 17cfaef..6423266 100644 --- a/main.py +++ b/main.py @@ -14,7 +14,8 @@ filemap = { "wholist": ["wholist.json", "WHO lists"], "counters": ["counters.json", "counters file"], "masterbuf": ["masterbuf.json", "master buffer"], -} + "monitor": ["monitor.json", "monitoring database"], + } connections = {} IRCPool = {} diff --git a/modules/keyword.py b/modules/keyword.py index 4d7cd9c..40638a4 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -89,7 +89,7 @@ def actKeyword(user, channel, message, nickname, actType, name): if main.config["Notifications"]["Query"]: if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in": if main.config["Compat"]["ZNC"]: - sendMaster("ZNC %s %s: (%s/%s) %s" % (actType, name, user, channel, msgLower)) + sendMaster("ZNC %s %s: %s" % (actType, name, msgLower)) ZNCAlreadySent = True else: sendMaster("QUERY %s %s: (%s) %s" % (actType, name, user, msgLower)) @@ -98,7 +98,7 @@ def actKeyword(user, channel, message, nickname, actType, name): if not ZNCAlreadySent == True: if main.config["Compat"]["ZNC"]: if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in": - sendMaster("ZNC %s %s: (%s/%s) %s" % (actType, name, user, channel, msgLower)) + sendMaster("ZNC %s %s: %s" % (actType, name, msgLower)) if toSend: sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) count.event(name, "keymatch") diff --git a/modules/monitor.py b/modules/monitor.py new file mode 100644 index 0000000..0e5c843 --- /dev/null +++ b/modules/monitor.py @@ -0,0 +1,36 @@ +import main +import modules.keyword as keyword +from pprint import pformat + +def testNetTarget(name, target): + for i in main.monitor.keys(): + if "sources" in main.monitor[i].keys(): + if name in main.monitor[i]["sources"]: + if main.monitor[i]["sources"][name] == True: + return i + elif target in main.monitor[i]["sources"][name]: + return i + else: + return i + return False + +def magicFunction(A, B): + return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) + +def event(name, target, cast): + monitorGroup = testNetTarget(name, target) + if monitorGroup == False: + return + matcher = magicFunction(cast, main.monitor[monitorGroup]) + + if matcher == True: + if "send" in main.monitor[monitorGroup]: + for i in main.monitor[monitorGroup]["send"].keys(): + if not i in main.pool.keys(): + keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i)) + if not i in main.IRCPool.keys(): + keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i)) + for x in main.monitor[monitorGroup]["send"][i]: + main.IRCPool[i].msg(x, "MONITOR [%s] %s" % (monitorGroup, pformat(cast))) + else: + keyword.sendMaster("MONITOR [%s] %s " % (monitorGroup, pformat(cast))) diff --git a/utils/fileutil/read_line.py b/utils/fileutil/read_line.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/fileutil/write_line.py b/utils/fileutil/write_line.py deleted file mode 100644 index e69de29..0000000 From edea19222dc1311a2e9254c6dde9800f9258dcf3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Jul 2018 21:30:50 +0100 Subject: [PATCH 082/394] Implement support for modes and get WHO data on a loop --- conf/example/mon.json | 1 - core/bot.py | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) delete mode 100644 conf/example/mon.json diff --git a/conf/example/mon.json b/conf/example/mon.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/mon.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/core/bot.py b/core/bot.py index 633d2b8..e75f776 100644 --- a/core/bot.py +++ b/core/bot.py @@ -1,6 +1,7 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred +from twisted.internet.task import LoopingCall import modules.keyword as keyword import modules.userinfo as userinfo @@ -139,6 +140,7 @@ class IRCBot(IRCClient): n = self._who[channel][1] n[nick] = [nick, user, host, server, status, realname] count.event(self.name, "whoreply") + monitor.event(self.name, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -152,7 +154,6 @@ class IRCBot(IRCClient): def got_who(self, whoinfo): userinfo.setWho(self.name, whoinfo[1]) - #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): """ @@ -232,6 +233,9 @@ class IRCBot(IRCClient): self.join(i) count.event(self.name, "signedon") + def getWho(self, channel): + self.who(channel).addCallback(self.got_who) + def joined(self, channel): if not channel in self.channels: self.channels.append(channel) @@ -241,6 +245,9 @@ class IRCBot(IRCClient): for i in range(len(main.masterbuf)): self.msg(channel, main.masterbuf.pop(0)) main.saveConf("masterbuf") + lc = LoopingCall(self.getWho, channel) + self._getWho[channel] = lc + lc.start(60) def left(self, channel, message): if channel in self.channels: @@ -248,10 +255,11 @@ class IRCBot(IRCClient): keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) count.event(self.name, "selfpart") monitor.event(self.name, channel, {"type": "part", "message": message}) + lc = self._getWho[channel] + lc.stop() def kickedFrom(self, channel, kicker, message): nick, ident, host = self.parsen(kicker) - print(kicker) userinfo.setWhoSingle(self.name, nick, ident, host) if channel in self.channels: self.channels.remove(channel) @@ -307,8 +315,10 @@ class IRCBot(IRCClient): nick, ident, host = self.parsen(user) userinfo.setWhoSingle(self.name, nick, ident, host) count.event(self.name, "mode") - print("%s // %s // %s" % (toset, modes, args)) - #monitor.event(self.name, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) + argList = list(args) + modeList = [i for i in modes] + for a, m in zip(argList, modeList): + monitor.event(self.name, channel, {"type": "mode", "exact": user, "nick": nick, "ident": ident, "host": host, "modes": m, "modeargs": a}) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): From 00985d29c525f0f91cb6440639afc901ca903fb9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Jul 2018 21:33:50 +0100 Subject: [PATCH 083/394] Implement support for more attributes and handle data type collisions when merging monitor group definitions --- commands/list.py | 9 ++---- commands/mod.py | 6 ++-- commands/mon.py | 79 ++++++++++++++++++++++++++++++++++------------ modules/monitor.py | 17 +++++++--- 4 files changed, 74 insertions(+), 37 deletions(-) diff --git a/commands/list.py b/commands/list.py index 000660e..13be3eb 100644 --- a/commands/list.py +++ b/commands/list.py @@ -1,4 +1,5 @@ import main +from yaml import dump class List: def __init__(self, register): @@ -6,13 +7,7 @@ class List: def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - poolMap = [] - for i in main.pool.keys(): - poolMap.append("Server: %s" % i) - for x in main.pool[i].keys(): - poolMap.append(" %s: %s" % (x, main.pool[i][x])) - poolMap.append("\n") - info("\n".join(poolMap)) + info(dump(main.pool)) return else: incUsage(None) diff --git a/commands/mod.py b/commands/mod.py index 95af432..0606c40 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,4 +1,5 @@ import main +from yaml import dump class Mod: def __init__(self, register): @@ -11,10 +12,7 @@ class Mod: if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) return - optionMap = ["Viewing options for %s" % spl[1]] - for i in main.pool[spl[1]].keys(): - optionMap.append(" %s: %s" % (i, main.pool[spl[1]][i])) - info("\n".join(optionMap)) + info(dump({spl[1]: main.pool[spl[1]]})) return elif length == 3: diff --git a/commands/mon.py b/commands/mon.py index 9be3997..9b9ed4c 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -2,7 +2,7 @@ import main import argparse import sys from io import StringIO -from pprint import pformat +from yaml import dump class Mon: def __init__(self, register): @@ -11,27 +11,30 @@ class Mon: def setup_arguments(self, ArgumentParser): self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.") group1 = self.parser.add_mutually_exclusive_group(required=True) - group1.add_argument("--add", metavar="entry", dest="addEntry", help="Add an entry") - group1.add_argument("--del", metavar="entry", dest="delEntry", help="Delete an entry") - group1.add_argument("--list", action="store_true", dest="listEntry", help="List all entries") - group1.add_argument("--mod", metavar="entry", dest="modEntry", help="Modify an entry") + group1.add_argument("-a", "--add", metavar="entry", dest="addEntry", help="Add an entry") + group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry") + group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries") + group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry") group2 = self.parser.add_mutually_exclusive_group() - group2.add_argument("--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") - group2.add_argument("--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") + group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") + group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") - self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, any") + self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who") self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") self.parser.add_argument("--exact", nargs="*", metavar="query", dest="exact", help="Use exact matching") self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching") self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching") self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching") - self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching (only works on WHO)") + self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who") self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)") self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action") self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick") self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode") + self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode") + self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who") + self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who") self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)") def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): @@ -90,6 +93,9 @@ class Mon: success("Successfully removed monitor group: %s" % parsed.delEntry) return elif parsed.modEntry: + if not parsed.doAppend and not parsed.doRemove: + failure("Specify --append or --remove with --mod") + return if not parsed.modEntry in main.monitor.keys(): failure("No such monitor group: %s" % parsed.modEntry) return @@ -101,22 +107,28 @@ class Mon: main.monitor[parsed.modEntry] = merged elif parsed.doRemove: merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info) + if merged == {}: + del main.monitor[parsed.modEntry] + info("Group %s deleted due to having no attributes" % parsed.modEntry) + main.saveConf("monitor") + return main.monitor[parsed.modEntry] = merged - else: - failure("Specify --append or --remove with --mod") - return main.saveConf("monitor") success("Successfully updated entry %s" % parsed.modEntry) return elif parsed.listEntry: - info(pformat(main.monitor)) + info(dump(main.monitor)) return else: incUsage(None) + def dedup(self, data): + if not isinstance(data, bool): + return list(set(data)) + return data + def parseNetworkFormat(self, lst, failure, info): - dedup = lambda x: list(set(x)) cast = {} if lst == None: return "nil" @@ -140,24 +152,23 @@ class Mon: else: if i[0] in cast.keys(): if not cast[i[0]] == True: - for x in dedup(i[1:]): + for x in self.dedup(i[1:]): cast[i[0]].append(x) else: - cast[i[0]] = dedup(i[1:]) + cast[i[0]] = self.dedup(i[1:]) else: - cast[i[0]] = dedup(i[1:]) + cast[i[0]] = self.dedup(i[1:]) for i in cast.keys(): - deduped = dedup(cast[i]) + deduped = self.dedup(cast[i]) cast[i] = deduped return cast # Create or modify a monitor group magically def makeCast(self, obj, failure, info): - dedup = lambda x: list(set(x)) - validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "any"] + validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"] cast = {} if not obj.specType == None: - types = dedup(obj.specType) + types = self.dedup(obj.specType) for i in types: if not i in validTypes: failure("Invalid type: %s" % i) @@ -187,6 +198,12 @@ class Mon: cast["user"] = obj.user if not obj.modes == None: cast["modes"] = obj.modes + if not obj.modeargs == None: + cast["modeargs"] = obj.modeargs + if not obj.server == None: + cast["server"] = obj.server + if not obj.status == None: + cast["status"] = obj.status if not obj.free == None: cast["free"] = obj.free if not obj.exact == None: @@ -208,11 +225,24 @@ class Mon: if i in source.keys(): if isinstance(source[i], dict): result = self.subtractCast(source[i], patch[i], info) + if result == {}: + info("Removing upper element: %s" % i) + del source[i] + continue source[i] = result continue + if isinstance(patch[i], bool): + del source[i] + info("Removing entire element: %s" % i) + continue for x in patch[i]: + if isinstance(source[i], bool): + info("Attempt to remove %s from network-wide definition" % x) + continue if x in source[i]: source[i].remove(x) + if source[i] == []: + del source[i] else: info("Element %s not in source %s" % (x, i)) else: @@ -226,7 +256,14 @@ class Mon: result = self.addCast(source[i], patch[i], info) source[i] = result continue + if isinstance(patch[i], bool): + source[i] = patch[i] + info("Overriding local element %s with network-wide definition" % i) + continue for x in patch[i]: + if isinstance(source[i], bool): + source[i] = [] + info("Overriding element %s, previously set network-wide" % i) if x in source[i]: info("Element %s already in source %s" % (x, i)) else: diff --git a/modules/monitor.py b/modules/monitor.py index 0e5c843..ef5a2e1 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,6 +1,7 @@ import main import modules.keyword as keyword from pprint import pformat +from copy import deepcopy def testNetTarget(name, target): for i in main.monitor.keys(): @@ -15,22 +16,28 @@ def testNetTarget(name, target): return False def magicFunction(A, B): + if "send" in B.keys(): + del B["send"] + if "sources" in B.keys(): + del B["sources"] return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) def event(name, target, cast): monitorGroup = testNetTarget(name, target) if monitorGroup == False: return - matcher = magicFunction(cast, main.monitor[monitorGroup]) - + matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup])) if matcher == True: - if "send" in main.monitor[monitorGroup]: + if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): + if isinstance(main.monitor[monitorGroup]["send"][i], bool): + keyword.sendMaster("ERRDELIV MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, pformat(cast))) + continue if not i in main.pool.keys(): keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i)) if not i in main.IRCPool.keys(): keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i)) for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "MONITOR [%s] %s" % (monitorGroup, pformat(cast))) + main.IRCPool[i].msg(x, "MONITOR [%s] (%s/%s) %s" % (monitorGroup, name, target, pformat(cast))) else: - keyword.sendMaster("MONITOR [%s] %s " % (monitorGroup, pformat(cast))) + keyword.sendMaster("MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, pformat(cast))) From 8300b86b322b28d6f821d6f51496fd3c9b284027 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Jul 2018 21:35:16 +0100 Subject: [PATCH 084/394] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f3d7146..1b77df6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ conf/wholist.json conf/keyword.json conf/counters.json conf/masterbuf.json +conf/monitor.json conf/dist.sh env/ From 23617cae535262d1a1e7515466266d6a50b3afcd Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Jul 2018 21:44:13 +0100 Subject: [PATCH 085/394] Indicate online servers as well as the total --- commands/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commands/stats.py b/commands/stats.py index 342f389..0bc7a09 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -15,7 +15,8 @@ class Stats: numChannels += len(main.IRCPool[i].channels) for i in main.wholist.keys(): numWhoEntries += len(main.wholist[i].keys()) - stats.append("Servers: %s" % len(main.IRCPool.keys())) + stats.append("Servers: %s" % len(main.pool.keys())) + stats.append("Online ervers: %s" % len(main.IRCPool.keys())) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) counterEvents = count.getEvents() From 9e3ae677e1d56bcfcd94ca77837f9f2034522474 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Jul 2018 22:02:40 +0100 Subject: [PATCH 086/394] Add config option for WHO loop interval --- conf/example/config.json | 5 ++++- core/bot.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index ecd781d..6974770 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -25,12 +25,15 @@ "Tweaks": { "ZNC": { "Prefix": "*" + }, + "Delays": { + "WhoLoop": 60 } }, "Default": { "host": null, "port": null, - "protocol": null, + "protocol": "ssl", "bind": null, "timeout": 30, "maxdelay": 360, diff --git a/core/bot.py b/core/bot.py index e75f776..c2ebca5 100644 --- a/core/bot.py +++ b/core/bot.py @@ -247,7 +247,7 @@ class IRCBot(IRCClient): main.saveConf("masterbuf") lc = LoopingCall(self.getWho, channel) self._getWho[channel] = lc - lc.start(60) + lc.start(main.config["Tweaks"]["Delays"]["WhoLoop"]) def left(self, channel, message): if channel in self.channels: From ae017eab36d96e29b5f1a0e9cd76d4cef065c730 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Jul 2018 22:32:24 +0100 Subject: [PATCH 087/394] Show the message on one line in monitor notifications --- modules/monitor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/monitor.py b/modules/monitor.py index ef5a2e1..513c97c 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,6 +1,5 @@ import main import modules.keyword as keyword -from pprint import pformat from copy import deepcopy def testNetTarget(name, target): @@ -31,13 +30,13 @@ def event(name, target, cast): if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): if isinstance(main.monitor[monitorGroup]["send"][i], bool): - keyword.sendMaster("ERRDELIV MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, pformat(cast))) + keyword.sendMaster("ERRDELIV MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) continue if not i in main.pool.keys(): keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i)) if not i in main.IRCPool.keys(): keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i)) for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "MONITOR [%s] (%s/%s) %s" % (monitorGroup, name, target, pformat(cast))) + main.IRCPool[i].msg(x, "MONITOR [%s] (%s/%s) %s" % (monitorGroup, name, target, cast)) else: - keyword.sendMaster("MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, pformat(cast))) + keyword.sendMaster("MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) From a61f74e578139d95d7f135d59f8fcf12f9c8d62f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 29 Jul 2018 13:04:47 +0100 Subject: [PATCH 088/394] Implement staggered WHO information collection to avoid detection --- conf/example/config.json | 3 ++- core/bot.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 6974770..8918523 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -27,7 +27,8 @@ "Prefix": "*" }, "Delays": { - "WhoLoop": 60 + "WhoLoop": 600, + "WhoRange": 1800 } }, "Default": { diff --git a/core/bot.py b/core/bot.py index c2ebca5..4fe36a5 100644 --- a/core/bot.py +++ b/core/bot.py @@ -3,6 +3,8 @@ from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall +from random import randint + import modules.keyword as keyword import modules.userinfo as userinfo import modules.counters as count @@ -247,7 +249,10 @@ class IRCBot(IRCClient): main.saveConf("masterbuf") lc = LoopingCall(self.getWho, channel) self._getWho[channel] = lc - lc.start(main.config["Tweaks"]["Delays"]["WhoLoop"]) + intrange = main.config["Tweaks"]["Delays"]["WhoRange"] + minint = main.config["Tweaks"]["Delays"]["WhoLoop"] + interval = randint(minint, minint+intrange) + lc.start(interval) def left(self, channel, message): if channel in self.channels: @@ -266,6 +271,8 @@ class IRCBot(IRCClient): keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.name, "selfkick") monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) + lc = self._getWho[channel] + lc.stop() def userJoined(self, user, channel): nick, ident, host = self.parsen(user) From 4b7c9f83e75036d1f3443ccdb638204951e0729c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 30 Jul 2018 21:09:34 +0100 Subject: [PATCH 089/394] Fix typo --- commands/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/stats.py b/commands/stats.py index 0bc7a09..6d032f3 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -16,7 +16,7 @@ class Stats: for i in main.wholist.keys(): numWhoEntries += len(main.wholist[i].keys()) stats.append("Servers: %s" % len(main.pool.keys())) - stats.append("Online ervers: %s" % len(main.IRCPool.keys())) + stats.append("Online servers: %s" % len(main.IRCPool.keys())) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) counterEvents = count.getEvents() From de5baf562b192c2b9a69261bef59dae950aa3058 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 19 Aug 2018 16:02:14 +0100 Subject: [PATCH 090/394] Fix monitors to work properly with multiple groups --- modules/monitor.py | 48 ++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/modules/monitor.py b/modules/monitor.py index 513c97c..48b01c3 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -3,16 +3,21 @@ import modules.keyword as keyword from copy import deepcopy def testNetTarget(name, target): + called = False for i in main.monitor.keys(): if "sources" in main.monitor[i].keys(): if name in main.monitor[i]["sources"]: if main.monitor[i]["sources"][name] == True: - return i + yield i + called = True elif target in main.monitor[i]["sources"][name]: - return i + yield i + called = True else: - return i - return False + yield i + called = True + if not called: + return False def magicFunction(A, B): if "send" in B.keys(): @@ -22,21 +27,22 @@ def magicFunction(A, B): return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) def event(name, target, cast): - monitorGroup = testNetTarget(name, target) - if monitorGroup == False: + monitorGroups = testNetTarget(name, target) + if monitorGroups == False: return - matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup])) - if matcher == True: - if "send" in main.monitor[monitorGroup].keys(): - for i in main.monitor[monitorGroup]["send"].keys(): - if isinstance(main.monitor[monitorGroup]["send"][i], bool): - keyword.sendMaster("ERRDELIV MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) - continue - if not i in main.pool.keys(): - keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i)) - if not i in main.IRCPool.keys(): - keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i)) - for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "MONITOR [%s] (%s/%s) %s" % (monitorGroup, name, target, cast)) - else: - keyword.sendMaster("MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) + for monitorGroup in monitorGroups: + matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup])) + if matcher == True: + if "send" in main.monitor[monitorGroup].keys(): + for i in main.monitor[monitorGroup]["send"].keys(): + if isinstance(main.monitor[monitorGroup]["send"][i], bool): + keyword.sendMaster("ERRDELIV MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) + continue + if not i in main.pool.keys(): + keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i)) + if not i in main.IRCPool.keys(): + keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i)) + for x in main.monitor[monitorGroup]["send"][i]: + main.IRCPool[i].msg(x, "MONITOR [%s] (%s/%s) %s" % (monitorGroup, name, target, cast)) + else: + keyword.sendMaster("MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) From b31b5d40e814f78b9dc9f475e4f84d16695e70f5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 26 Aug 2018 19:08:27 +0100 Subject: [PATCH 091/394] Make monitor notifications ignore numbers to support multiple networks in only one reference --- core/bot.py | 53 +++++++++++++++++++++++---------------------- modules/userinfo.py | 3 --- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/core/bot.py b/core/bot.py index 4fe36a5..04e881d 100644 --- a/core/bot.py +++ b/core/bot.py @@ -2,7 +2,7 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall - +from string import digits from random import randint import modules.keyword as keyword @@ -18,6 +18,7 @@ class IRCBot(IRCClient): def __init__(self, name): self.connected = False self.channels = [] + self.net = "".join([x for x in name if not x in digits]) self.buffer = "" self.name = name instance = main.pool[name] @@ -70,27 +71,27 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) - monitor.event(self.name, channel, {"type": "msg", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) + monitor.event(self.net, channel, {"type": "msg", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) - monitor.event(self.name, channel, {"type": "notice", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) + monitor.event(self.net, channel, {"type": "notice", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) def action(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) - monitor.event(self.name, channel, {"type": "action", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) + monitor.event(self.net, channel, {"type": "action", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) def get(self, var): try: @@ -142,7 +143,7 @@ class IRCBot(IRCClient): n = self._who[channel][1] n[nick] = [nick, user, host, server, status, realname] count.event(self.name, "whoreply") - monitor.event(self.name, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) + monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -259,73 +260,73 @@ class IRCBot(IRCClient): self.channels.remove(channel) keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) count.event(self.name, "selfpart") - monitor.event(self.name, channel, {"type": "part", "message": message}) + monitor.event(self.net, channel, {"type": "part", "message": message}) lc = self._getWho[channel] lc.stop() def kickedFrom(self, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) if channel in self.channels: self.channels.remove(channel) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.name, "selfkick") - monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) + monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) lc = self._getWho[channel] lc.stop() def userJoined(self, user, channel): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "join") - monitor.event(self.name, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) + monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) count.event(self.name, "part") - monitor.event(self.name, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) + monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "quit") keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) - monitor.event(self.name, None, {"type": "quit", "exact": user, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + monitor.event(self.net, None, {"type": "quit", "exact": user, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "kick") keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) - monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message, "user": kickee}) + monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message, "user": kickee}) def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) - userinfo.setWhoSingle(self.name, nick, ident, host) - userinfo.setWhoSingle(self.name, newname, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.setWhoSingle(self.net, newname, ident, host) count.event(self.name, "nick") - monitor.event(self.name, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) + monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "topic") keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) - monitor.event(self.name, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) + monitor.event(self.net, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.name, nick, ident, host) + userinfo.setWhoSingle(self.net, nick, ident, host) count.event(self.name, "mode") argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - monitor.event(self.name, channel, {"type": "mode", "exact": user, "nick": nick, "ident": ident, "host": host, "modes": m, "modeargs": a}) + monitor.event(self.net, channel, {"type": "mode", "exact": user, "nick": nick, "ident": ident, "host": host, "modes": m, "modeargs": a}) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): diff --git a/modules/userinfo.py b/modules/userinfo.py index d4acba2..952075d 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -3,7 +3,6 @@ from string import digits #from utils.logging.log import * def setWho(network, newObjects): - network = "".join([x for x in network if not x in digits]) if not network in main.wholist.keys(): main.wholist[network] = {} for i in newObjects.keys(): @@ -12,8 +11,6 @@ def setWho(network, newObjects): return def setWhoSingle(network, nick, ident, host): - network = "".join([x for x in network if not x in digits]) - if network in main.wholist.keys(): if nick in main.wholist[network].keys(): main.wholist[network][nick][1] = ident From 36105c7e9a78d4a1832463c5eeb13d4612e0eb3f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 27 Aug 2018 20:42:49 +0100 Subject: [PATCH 092/394] Move user metadata info into redis --- commands/stats.py | 15 +---- commands/who.py | 4 +- conf/help.json | 4 +- core/bot.py | 107 ++++++++++++++++++++++--------- main.py | 6 +- modules/keyword.py | 1 - modules/userinfo.py | 150 ++++++++++++++++++++++++++++++++++++-------- 7 files changed, 215 insertions(+), 72 deletions(-) diff --git a/commands/stats.py b/commands/stats.py index 6d032f3..35d2d40 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,5 +1,6 @@ import main import modules.counters as count +import modules.userinfo as userinfo class Stats: def __init__(self, register): @@ -13,8 +14,7 @@ class Stats: numWhoEntries = 0 for i in main.IRCPool.keys(): numChannels += len(main.IRCPool[i].channels) - for i in main.wholist.keys(): - numWhoEntries += len(main.wholist[i].keys()) + numWhoEntries += userinfo.getNumTotalWhoEntries() stats.append("Servers: %s" % len(main.pool.keys())) stats.append("Online servers: %s" % len(main.IRCPool.keys())) stats.append("Channels: %s" % numChannels) @@ -34,20 +34,11 @@ class Stats: numChannels = 0 numWhoEntries = 0 - failures = 0 if spl[1] in main.IRCPool.keys(): numChannels += len(main.IRCPool[spl[1]].channels) else: failure("Name does not exist: %s" % spl[1]) - failures += 1 - if spl[1] in main.wholist.keys(): - numWhoEntries += len(main.wholist[spl[1]].keys()) - else: - failure("Who entry does not exist: %s" % spl[1]) - failures += 1 - if failures == 2: - failure("No information found, aborting") - return + numWhoEntries += userinfo.getNumWhoEntries(spl[1]) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) counterEvents = count.getEvents(spl[1]) diff --git a/commands/who.py b/commands/who.py index a2b4cfb..9898cfc 100644 --- a/commands/who.py +++ b/commands/who.py @@ -14,9 +14,9 @@ class Who: rtrn += "Matches from: %s" % i rtrn += "\n" for x in result[i]: - x = [y for y in x if not y == None] - rtrn += str((", ".join(x))) + rtrn += (x) rtrn += "\n" + rtrn += "\n" info(rtrn) return else: diff --git a/conf/help.json b/conf/help.json index a69df80..e2bed17 100644 --- a/conf/help.json +++ b/conf/help.json @@ -14,8 +14,8 @@ "disable": "disable ", "list": "list", "stats": "stats []", - "save": "save ", - "load": "load ", + "save": "save ", + "load": "load ", "dist": "dist", "loadmod": "loadmod ", "msg": "msg ", diff --git a/core/bot.py b/core/bot.py index 04e881d..98ecf31 100644 --- a/core/bot.py +++ b/core/bot.py @@ -2,6 +2,7 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall + from string import digits from random import randint @@ -19,6 +20,8 @@ class IRCBot(IRCClient): self.connected = False self.channels = [] self.net = "".join([x for x in name if not x in digits]) + if self.net == "": + error("Network with all numbers: %s" % name) self.buffer = "" self.name = name instance = main.pool[name] @@ -37,6 +40,8 @@ class IRCBot(IRCClient): self._who = {} self._getWho = {} + self._names = {} + self.authtype = instance["authtype"] if self.authtype == "ns": self.authpass = instance["password"] @@ -71,7 +76,7 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) @@ -79,7 +84,7 @@ class IRCBot(IRCClient): def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) @@ -87,7 +92,7 @@ class IRCBot(IRCClient): def action(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) @@ -109,8 +114,10 @@ class IRCBot(IRCClient): newnick = nickname + "_" return newnick - def nickChanged(self, nick): - self.nickname = nick + def nickChanged(self, olduser, newnick): + oldnick, ident, host = self.parsen(olduser) + userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host) + self.nickname = newnick count.event(self.name, "selfnick") def irc_ERR_NICKNAMEINUSE(self, prefix, params): @@ -122,10 +129,9 @@ class IRCBot(IRCClient): sendAll("%s: password mismatch" % self.name) def who(self, channel): - channel = channel d = Deferred() if channel not in self._who: - self._who[channel] = ([], {}) + self._who[channel] = ([], []) self._who[channel][0].append(d) self.sendLine("WHO %s" % channel) return d @@ -141,7 +147,7 @@ class IRCBot(IRCClient): if channel not in self._who: return n = self._who[channel][1] - n[nick] = [nick, user, host, server, status, realname] + n.append([nick, user, host, server, status, realname]) count.event(self.name, "whoreply") monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) @@ -155,7 +161,50 @@ class IRCBot(IRCClient): del self._who[channel] def got_who(self, whoinfo): - userinfo.setWho(self.name, whoinfo[1]) + userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1]) + + def sanit(self, data): + if len(data) >= 1: + if data[0] in ["!", "~", "&", "@", "%", "+"]: + data = data[1:] + return data + return data + else: + return False + + def names(self, channel): + d = Deferred() + if channel not in self._names: + self._names[channel] = ([], []) + self._names[channel][0].append(d) + self.sendLine("NAMES %s" % channel) + return d + + def irc_RPL_NAMREPLY(self, prefix, params): + channel = params[2] + nicklist = params[3].split(' ') + if channel not in self._names: + return + n = self._names[channel][1] + n.append(nicklist) + + def irc_RPL_ENDOFNAMES(self, prefix, params): + channel = params[1] + if channel not in self._names: + return + callbacks, namelist = self._names[channel] + for cb in callbacks: + cb.callback((channel, namelist)) + del self._names[channel] + + def got_names(self, nicklist): + newNicklist = [] + for i in nicklist[1]: + for x in i: + f = self.sanit(x) + if f: + newNicklist.append(f) + userinfo.initialNames(self.net, nicklist[0], newNicklist) #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): @@ -197,7 +246,7 @@ class IRCBot(IRCClient): """ nick = prefix.split('!', 1)[0] if nick == self.nickname: - self.nickChanged(params[0]) + self.nickChanged(prefix, params[0]) else: self.userRenamed(prefix, params[0]) @@ -236,61 +285,63 @@ class IRCBot(IRCClient): self.join(i) count.event(self.name, "signedon") - def getWho(self, channel): - self.who(channel).addCallback(self.got_who) - def joined(self, channel): if not channel in self.channels: self.channels.append(channel) + self.names(channel).addCallback(self.got_names) self.who(channel).addCallback(self.got_who) count.event(self.name, "selfjoin") if self.name == main.config["Master"][0] and channel == main.config["Master"][1]: for i in range(len(main.masterbuf)): self.msg(channel, main.masterbuf.pop(0)) main.saveConf("masterbuf") - lc = LoopingCall(self.getWho, channel) + lc = LoopingCall(self.who, channel) self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] minint = main.config["Tweaks"]["Delays"]["WhoLoop"] interval = randint(minint, minint+intrange) lc.start(interval) - def left(self, channel, message): + def botLeft(self, channel): if channel in self.channels: self.channels.remove(channel) + if channel in self._getWho.keys(): + lc = self._getWho[channel] + lc.stop() + del self._getWho[channel] + userinfo.delChannel(self.net, channel) + + def left(self, channel, message): keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) count.event(self.name, "selfpart") monitor.event(self.net, channel, {"type": "part", "message": message}) - lc = self._getWho[channel] - lc.stop() + self.botLeft(channel) def kickedFrom(self, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.setWhoSingle(self.net, nick, ident, host) if channel in self.channels: self.channels.remove(channel) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.name, "selfkick") monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) - lc = self._getWho[channel] - lc.stop() + self.botLeft(channel) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.addUser(self.net, channel, nick, user) count.event(self.name, "join") monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.delUser(self.net, channel, nick, user) keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) count.event(self.name, "part") monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.delUserByNetwork(self.net, nick, user) count.event(self.name, "quit") keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) @@ -298,7 +349,8 @@ class IRCBot(IRCClient): def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, kicker) + userinfo.delUserByNick(self.net, channel, kickee) count.event(self.name, "kick") keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) @@ -306,14 +358,13 @@ class IRCBot(IRCClient): def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) - userinfo.setWhoSingle(self.net, nick, ident, host) - userinfo.setWhoSingle(self.net, newname, ident, host) + userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) count.event(self.name, "nick") monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "topic") keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) @@ -321,7 +372,7 @@ class IRCBot(IRCClient): def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "mode") argList = list(args) modeList = [i for i in modes] diff --git a/main.py b/main.py index 6423266..985bbea 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,10 @@ from json import load, dump, loads - +import redis from utils.loaders.command_loader import loadCommands from utils.logging.log import * +r = redis.StrictRedis(unix_socket_path='/tmp/redis.sock', db=0) + configPath = "conf/" certPath = "cert/" @@ -11,7 +13,7 @@ filemap = { "keyconf": ["keyword.json", "keyword lists"], "pool": ["pool.json", "pool"], "help": ["help.json", "command help"], - "wholist": ["wholist.json", "WHO lists"], +# "wholist": ["wholist.json", "WHO lists"], "counters": ["counters.json", "counters file"], "masterbuf": ["masterbuf.json", "master buffer"], "monitor": ["monitor.json", "monitoring database"], diff --git a/modules/keyword.py b/modules/keyword.py index 40638a4..66a588b 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -29,7 +29,6 @@ def sendMaster(data): if not hasMonitors: main.masterbuf.append(data) main.saveConf("masterbuf") - warn("Master with no IRC instance defined") for i in main.MonitorPool: main.connections[i].send(data) diff --git a/modules/userinfo.py b/modules/userinfo.py index 952075d..ebb9dda 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,31 +1,131 @@ import main from string import digits -#from utils.logging.log import * +from utils.logging.log import * -def setWho(network, newObjects): - if not network in main.wholist.keys(): - main.wholist[network] = {} - for i in newObjects.keys(): - main.wholist[network][i] = newObjects[i] +def getWhoSingle(name, query): + result = main.r.sscan("live.who."+name, 0, query) + if result[1] == []: + return None + return [i.decode() for i in result[1]] - return - -def setWhoSingle(network, nick, ident, host): - if network in main.wholist.keys(): - if nick in main.wholist[network].keys(): - main.wholist[network][nick][1] = ident - main.wholist[network][nick][2] = host - else: - main.wholist[network][nick] = [nick, ident, host, None, None, None] - else: - main.wholist[network] = {nick: [nick, ident, host, None, None, None]} - -def getWho(nick): +def getWho(query): result = {} - for i in main.wholist.keys(): - for x in main.wholist[i].keys(): - if nick.lower() == x.lower(): - if not i in result.keys(): - result[i] = [] - result[i].append(main.wholist[i][x]) + for i in main.pool.keys(): + f = getWhoSingle(i, query) + if f: + result[i] = f + print(result) return result + +def getNumWhoEntries(name): + return main.r.scard("live.who."+name) + +def getNumTotalWhoEntries(): + total = 0 + for i in main.pool.keys(): + total += getNumWhoEntries(i) + return total + +def getNamespace(name, channel, nick): + gnamespace = "live.who.%s" % name + namespace = "live.who.%s.%s" % (name, channel) + chanspace = "live.chan.%s.%s" % (name, nick) + return [gnamespace, namespace, chanspace] + +def initialUsers(name, channel, users): + gnamespace = "live.who.%s" % name + p = main.r.pipeline() + for i in users: + p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2]) + p.execute() + +def initialNames(name, channel, names): + namespace = "live.who.%s.%s" % (name, channel) + p = main.r.pipeline() + for i in names: + p.sadd(namespace, i) + p.sadd("live.chan."+name+"."+i, channel) + p.execute() + +def editUser(name, channel, nick, user): + gnamespace = "live.who.%s" % name + main.r.sadd(gnamespace, user) + +def addUser(name, channel, nick, user): + gnamespace, namespace, chanspace = getNamespace(name, channel, nick) + p = main.r.pipeline() + p.sadd(gnamespace, user) + p.sadd(namespace, nick) + p.sadd(chanspace, channel) + p.execute() + +def delUser(name, channel, nick, user): + gnamespace, namespace, chanspace = getNamespace(name, channel, nick) + p = main.r.pipeline() + channels = main.r.smembers(chanspace) + p.srem(namespace, nick) + if channels == {channel.encode()}: + p.delete(chanspace) + p.srem(gnamespace, user) + else: + p.srem(chanspace, channel) + p.execute() + +def getUserByNick(name, nick): + gnamespace = "live.who.%s" % name + usermatch = main.r.sscan(gnamespace, match=nick+"!*") + if usermatch[1] == []: + return False + else: + if len(usermatch[1]) == 1: + user = usermatch[1][0] + return user + else: + return False + +def renameUser(name, oldnick, olduser, newnick, newuser): + gnamespace = "live.who.%s" % name + chanspace = "live.chan.%s.%s" % (name, oldnick) + newchanspace = "live.chan.%s.%s" % (name, newnick) + p = main.r.pipeline() + p.srem(gnamespace, olduser) + p.sadd(gnamespace, newuser) + for i in main.r.smembers(chanspace): + i = i.decode() + p.srem("live.who."+name+"."+i, oldnick) + p.sadd("live.who."+name+"."+i, newnick) + if main.r.exists(chanspace): + p.rename(chanspace, newchanspace) + else: + warn("Key doesn't exist: %s" % chanspace) + p.execute() + +def delUserByNick(name, channel, nick): + gnamespace = "live.who.%s" % name + user = getUserByNick(name, nick) + delUser(name, channel, nick, user) + +def delUserByNetwork(name, nick, user): + gnamespace = "live.who.%s" % name + chanspace = "live.chan.%s.%s" % (name, nick) + p = main.r.pipeline() + p.srem(gnamespace, user) + for i in main.r.smembers(chanspace): + p.srem("live.who."+name+"."+i.decode(), nick) + p.delete(chanspace) + p.execute() + +def delChannel(name, channel): + gnamespace = "live.who.%s" % name + namespace = "live.who.%s.%s" % (name, channel) + p = main.r.pipeline() + for i in main.r.smembers(namespace): + user = getUserByNick(name, i.decode()) + if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}: + if user: + p.srem(gnamespace, user) + p.delete("live.chan."+name+"."+i.decode()) + else: + p.srem("live.chan."+name+"."+i.decode(), channel) + p.delete(namespace) + p.execute() From ad00e0c07dda29015fbe8deb64c420e15ff0d384 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 27 Aug 2018 20:52:39 +0100 Subject: [PATCH 093/394] Make the Redis socket configurable --- conf/example/config.json | 1 + main.py | 6 ++++-- modules/userinfo.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 8918523..93a7527 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -6,6 +6,7 @@ "Key": "key.pem", "Certificate": "cert.pem" }, + "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, "Notifications": { diff --git a/main.py b/main.py index 985bbea..78c11b5 100644 --- a/main.py +++ b/main.py @@ -3,8 +3,6 @@ import redis from utils.loaders.command_loader import loadCommands from utils.logging.log import * -r = redis.StrictRedis(unix_socket_path='/tmp/redis.sock', db=0) - configPath = "conf/" certPath = "cert/" @@ -49,5 +47,9 @@ def initConf(): loadConf(i) def initMain(): + global r initConf() loadCommands(register) + r = redis.StrictRedis(unix_socket_path=config["RedisSocket"], db=0) + + diff --git a/modules/userinfo.py b/modules/userinfo.py index ebb9dda..9814ef0 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -14,7 +14,6 @@ def getWho(query): f = getWhoSingle(i, query) if f: result[i] = f - print(result) return result def getNumWhoEntries(name): From 2cb0b5d4a6f8f86e3b4dcd913d78b9d23cf0a65f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 27 Aug 2018 21:10:49 +0100 Subject: [PATCH 094/394] Add a count parameter to ensure all entries are searched --- modules/userinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index 9814ef0..053edb4 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -3,7 +3,7 @@ from string import digits from utils.logging.log import * def getWhoSingle(name, query): - result = main.r.sscan("live.who."+name, 0, query) + result = main.r.sscan("live.who."+name, 0, query, count=9999999) if result[1] == []: return None return [i.decode() for i in result[1]] From 1de3f17d45f0bc76a186552733c8ba3412d643ef Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 31 Aug 2018 22:20:42 +0100 Subject: [PATCH 095/394] Change help text for who command --- conf/help.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/help.json b/conf/help.json index e2bed17..831c834 100644 --- a/conf/help.json +++ b/conf/help.json @@ -7,7 +7,7 @@ "default": "default [] []", "get": "get ", "key": "key [] [] [] []", - "who": "who ", + "who": "who ", "join": "join []", "part": "part ", "enable": "enable ", From 249e99805aeb894941b77c5556bdb202118b066d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 1 Sep 2018 00:25:51 +0100 Subject: [PATCH 096/394] Implement optional x in y matching for attributes in the monitor system --- commands/mon.py | 5 +++++ modules/monitor.py | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/commands/mon.py b/commands/mon.py index 9b9ed4c..5231abe 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -20,6 +20,9 @@ class Mon: group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") + self.parser.add_argument("--inside", action="store_true", dest="inside", help="Use x in y logic for matching instead of comparing exact values") + self.parser.add_argument("--outside", action="store_false", dest="inside", help="Don't use x in y logic for matching instead of comparing exact values") + self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who") self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") self.parser.add_argument("--exact", nargs="*", metavar="query", dest="exact", help="Use exact matching") @@ -218,6 +221,8 @@ class Mon: cast["real"] = [] for i in obj.real: cast["real"].append(" ".join(i)) + if not obj.inside == None: + cast["inside"] = obj.inside return cast def subtractCast(self, source, patch, info): diff --git a/modules/monitor.py b/modules/monitor.py index 48b01c3..7552648 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -19,12 +19,34 @@ def testNetTarget(name, target): if not called: return False +def contained_in(x, y): + if x is None or y is None: + return False + elif x == y: + return True + else: + return y in x + +def is_in(k, vs, A): + return any(contained_in(A.get(k), vp) for vp in vs) + +def matches(A, B): + return all(is_in(k, vs, A) for (k, vs) in B.items()) + def magicFunction(A, B): + isInside = False if "send" in B.keys(): del B["send"] if "sources" in B.keys(): del B["sources"] - return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) + if "inside" in B.keys(): + if B["inside"] == True: + isInside = True + del B["inside"] + if isInside: + return matches(A, B) + else: + return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) def event(name, target, cast): monitorGroups = testNetTarget(name, target) From 7baa81aa30f7065fc03216475b1690e29d9cd000 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 1 Sep 2018 00:48:15 +0100 Subject: [PATCH 097/394] Fix bug with the inside parameter being specified by default --- commands/mon.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/commands/mon.py b/commands/mon.py index 5231abe..bfcf5bf 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -20,8 +20,7 @@ class Mon: group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") - self.parser.add_argument("--inside", action="store_true", dest="inside", help="Use x in y logic for matching instead of comparing exact values") - self.parser.add_argument("--outside", action="store_false", dest="inside", help="Don't use x in y logic for matching instead of comparing exact values") + self.parser.add_argument("--inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values") self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who") self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") @@ -222,7 +221,13 @@ class Mon: for i in obj.real: cast["real"].append(" ".join(i)) if not obj.inside == None: - cast["inside"] = obj.inside + if obj.inside.lower() in ["yes", "true", "on"]: + cast["inside"] = True + elif obj.inside.lower() in ["no", "false", "off"]: + cast["inside"] = False + else: + failure("inside: Unknown operand: %s" % obj.inside) + return return cast def subtractCast(self, source, patch, info): From eaebab0cf498ca2cc0e76877031270bb956cea30 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 7 Oct 2018 19:50:11 +0100 Subject: [PATCH 098/394] Add a help page for the chans command --- conf/help.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/help.json b/conf/help.json index 831c834..7d75d40 100644 --- a/conf/help.json +++ b/conf/help.json @@ -19,5 +19,6 @@ "dist": "dist", "loadmod": "loadmod ", "msg": "msg ", - "mon": "mon -h" + "mon": "mon -h", + "chans": "chans " } From cfefa1d6274563148badaab58dca05f695980c57 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 7 Oct 2018 20:48:39 +0100 Subject: [PATCH 099/394] Implement a command to get the channels common to one or more users --- commands/chans.py | 22 ++++++++++++++++++++++ modules/userinfo.py | 15 +++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 commands/chans.py diff --git a/commands/chans.py b/commands/chans.py new file mode 100644 index 0000000..b3579a7 --- /dev/null +++ b/commands/chans.py @@ -0,0 +1,22 @@ +import main +import modules.userinfo as userinfo + +class Chans: + def __init__(self, register): + register("chans", self.chans) + + def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + result = userinfo.getChans(spl[1:]) + rtrn = "" + for i in result.keys(): + rtrn += "Matches from: %s" % i + rtrn += "\n" + for x in result[i]: + rtrn += (x) + rtrn += "\n" + rtrn += "\n" + info(rtrn) + return + else: + incUsage(None) diff --git a/modules/userinfo.py b/modules/userinfo.py index 053edb4..e3ad7f0 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -16,6 +16,21 @@ def getWho(query): result[i] = f return result +def getChansSingle(name, query): + query = ["live.chan."+name+"."+i for i in query] + result = main.r.sinter(*query) + if len(result) == 0: + return None + return [i.decode() for i in result] + +def getChans(query): + result = {} + for i in main.pool.keys(): + f = getChansSingle(i, query) + if f: + result[i] = f + return result + def getNumWhoEntries(name): return main.r.scard("live.who."+name) From a98ed4e4d04c04838329548473fa79639d2c93e5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 7 Oct 2018 20:52:58 +0100 Subject: [PATCH 100/394] Fix bug in the keyword module --- modules/keyword.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/keyword.py b/modules/keyword.py index 66a588b..9b6b926 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -48,7 +48,7 @@ def isKeyword(msg): if x in message: toUndo = True messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") - for y in keyconf["Keywords"]: + for y in main.keyconf["Keywords"]: if i in messageDuplicate: totalNum += messageDuplicate.count(i) message = messageDuplicate.replace(i, "{"+i+"}") From 44aa0f1727add2439bcb586c6743a1c192865f89 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 8 Oct 2018 20:08:10 +0100 Subject: [PATCH 101/394] Implement users command to see the mutual users of one or more channels and squash some bugs --- commands/chans.py | 3 +++ commands/users.py | 25 +++++++++++++++++++++++++ conf/help.json | 3 ++- modules/userinfo.py | 25 ++++++++++++++++++++----- 4 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 commands/users.py diff --git a/commands/chans.py b/commands/chans.py index b3579a7..851b9e8 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -7,6 +7,9 @@ class Chans: def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: + if len(spl) < 2: + incUsage("chans") + return result = userinfo.getChans(spl[1:]) rtrn = "" for i in result.keys(): diff --git a/commands/users.py b/commands/users.py new file mode 100644 index 0000000..a76e20e --- /dev/null +++ b/commands/users.py @@ -0,0 +1,25 @@ +import main +import modules.userinfo as userinfo + +class Users: + def __init__(self, register): + register("users", self.users) + + def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if len(spl) < 2: + incUsage("users") + return + result = userinfo.getUsers(spl[1:]) + rtrn = "" + for i in result.keys(): + rtrn += "Matches from: %s" % i + rtrn += "\n" + for x in result[i]: + rtrn += (x) + rtrn += "\n" + rtrn += "\n" + info(rtrn) + return + else: + incUsage(None) diff --git a/conf/help.json b/conf/help.json index 7d75d40..47a0ff3 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,5 +20,6 @@ "loadmod": "loadmod ", "msg": "msg ", "mon": "mon -h", - "chans": "chans " + "chans": "chans ", + "users": "users " } diff --git a/modules/userinfo.py b/modules/userinfo.py index e3ad7f0..b6f8da9 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -16,17 +16,32 @@ def getWho(query): result[i] = f return result -def getChansSingle(name, query): - query = ["live.chan."+name+"."+i for i in query] - result = main.r.sinter(*query) +def getChansSingle(name, nick): + nick = ["live.chan."+name+"."+i for i in nick] + result = main.r.sinter(*nick) if len(result) == 0: return None return [i.decode() for i in result] -def getChans(query): +def getChans(nick): result = {} for i in main.pool.keys(): - f = getChansSingle(i, query) + f = getChansSingle(i, nick) + if f: + result[i] = f + return result + +def getUsersSingle(name, nick): + nick = ["live.who."+name+"."+i for i in nick] + result = main.r.sinter(*nick) + if len(result) == 0: + return None + return [i.decode() for i in result] + +def getUsers(nick): + result = {} + for i in main.pool.keys(): + f = getUsersSingle(i, nick) if f: result[i] = f return result From 9dc202fd537e8cdbd0142702fd5935ddc927786d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Oct 2018 00:42:17 +0100 Subject: [PATCH 102/394] Fix typo in the default command --- commands/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/default.py b/commands/default.py index 82387af..d432cf1 100644 --- a/commands/default.py +++ b/commands/default.py @@ -67,7 +67,7 @@ class Default: spl[2] = [] main.config["Default"][spl[1]] = spl[2] - main.saveConf("main.config") + main.saveConf("config") if toUnset: success("Successfully unset key %s" % spl[1]) else: From 7d7ef69d98c992a6f9d01ee7159e9681ec6d73c7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Oct 2018 14:47:08 +0100 Subject: [PATCH 103/394] Avoid doing pointless lookups against numbered networks --- main.py | 9 +++++++++ modules/userinfo.py | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 78c11b5..24b78ef 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ from json import load, dump, loads import redis +from string import digits from utils.loaders.command_loader import loadCommands from utils.logging.log import * @@ -26,6 +27,14 @@ MonitorPool = [] CommandMap = {} +def nets(): + if not "pool" in globals(): + return + networks = set() + for i in pool: + networks.add("".join([x for x in i if not i in digits])) + return networks + def register(command, function): if not command in CommandMap: CommandMap[command] = function diff --git a/modules/userinfo.py b/modules/userinfo.py index b6f8da9..ac4bd5a 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -10,7 +10,7 @@ def getWhoSingle(name, query): def getWho(query): result = {} - for i in main.pool.keys(): + for i in main.nets(): f = getWhoSingle(i, query) if f: result[i] = f @@ -25,7 +25,7 @@ def getChansSingle(name, nick): def getChans(nick): result = {} - for i in main.pool.keys(): + for i in main.nets(): f = getChansSingle(i, nick) if f: result[i] = f @@ -40,7 +40,7 @@ def getUsersSingle(name, nick): def getUsers(nick): result = {} - for i in main.pool.keys(): + for i in main.nets(): f = getUsersSingle(i, nick) if f: result[i] = f @@ -51,7 +51,7 @@ def getNumWhoEntries(name): def getNumTotalWhoEntries(): total = 0 - for i in main.pool.keys(): + for i in main.nets(): total += getNumWhoEntries(i) return total From c6e9604d6f27a058a4ac3eaabe104e7642b43290 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Oct 2018 17:17:59 +0100 Subject: [PATCH 104/394] Add meta variable for the inside parameter of the mon command --- commands/mon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/mon.py b/commands/mon.py index bfcf5bf..214b363 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -20,7 +20,7 @@ class Mon: group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") - self.parser.add_argument("--inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values") + self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values") self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who") self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") From 3b42f19291aca883c69302d531b632642e5aaed5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Oct 2018 20:16:41 +0100 Subject: [PATCH 105/394] Use the network name instead of the numbered instance name for counting events --- core/bot.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/core/bot.py b/core/bot.py index 98ecf31..d733693 100644 --- a/core/bot.py +++ b/core/bot.py @@ -77,7 +77,7 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) - count.event(self.name, "privmsg") + count.event(self.net, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) monitor.event(self.net, channel, {"type": "msg", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) @@ -85,7 +85,7 @@ class IRCBot(IRCClient): def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) - count.event(self.name, "notice") + count.event(self.net, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) monitor.event(self.net, channel, {"type": "notice", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) @@ -93,7 +93,7 @@ class IRCBot(IRCClient): def action(self, user, channel, msg): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) - count.event(self.name, "action") + count.event(self.net, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) monitor.event(self.net, channel, {"type": "action", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) @@ -118,7 +118,7 @@ class IRCBot(IRCClient): oldnick, ident, host = self.parsen(olduser) userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host) self.nickname = newnick - count.event(self.name, "selfnick") + count.event(self.net, "selfnick") def irc_ERR_NICKNAMEINUSE(self, prefix, params): self._attemptedNick = self.alterCollidedNick(self._attemptedNick) @@ -148,7 +148,7 @@ class IRCBot(IRCClient): return n = self._who[channel][1] n.append([nick, user, host, server, status, realname]) - count.event(self.name, "whoreply") + count.event(self.net, "whoreply") monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) def irc_RPL_ENDOFWHO(self, prefix, params): @@ -283,14 +283,14 @@ class IRCBot(IRCClient): self.msg(self.authentity, "IDENTIFY %s" % self.nspass) for i in self.autojoin: self.join(i) - count.event(self.name, "signedon") + count.event(self.net, "signedon") def joined(self, channel): if not channel in self.channels: self.channels.append(channel) self.names(channel).addCallback(self.got_names) self.who(channel).addCallback(self.got_who) - count.event(self.name, "selfjoin") + count.event(self.net, "selfjoin") if self.name == main.config["Master"][0] and channel == main.config["Master"][1]: for i in range(len(main.masterbuf)): self.msg(channel, main.masterbuf.pop(0)) @@ -313,7 +313,7 @@ class IRCBot(IRCClient): def left(self, channel, message): keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) - count.event(self.name, "selfpart") + count.event(self.net, "selfpart") monitor.event(self.net, channel, {"type": "part", "message": message}) self.botLeft(channel) @@ -322,27 +322,27 @@ class IRCBot(IRCClient): if channel in self.channels: self.channels.remove(channel) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) - count.event(self.name, "selfkick") + count.event(self.net, "selfkick") monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) self.botLeft(channel) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) userinfo.addUser(self.net, channel, nick, user) - count.event(self.name, "join") + count.event(self.net, "join") monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) userinfo.delUser(self.net, channel, nick, user) keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) - count.event(self.name, "part") + count.event(self.net, "part") monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.delUserByNetwork(self.net, nick, user) - count.event(self.name, "quit") + count.event(self.net, "quit") keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) monitor.event(self.net, None, {"type": "quit", "exact": user, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) @@ -351,7 +351,7 @@ class IRCBot(IRCClient): nick, ident, host = self.parsen(kicker) userinfo.editUser(self.net, channel, nick, kicker) userinfo.delUserByNick(self.net, channel, kickee) - count.event(self.name, "kick") + count.event(self.net, "kick") keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message, "user": kickee}) @@ -359,13 +359,13 @@ class IRCBot(IRCClient): def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) - count.event(self.name, "nick") + count.event(self.net, "nick") monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) - count.event(self.name, "topic") + count.event(self.net, "topic") keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) monitor.event(self.net, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) @@ -373,7 +373,7 @@ class IRCBot(IRCClient): def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) - count.event(self.name, "mode") + count.event(self.net, "mode") argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): From 7cd6bc361604fed4844e97f0e54e53b4a0413002 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 21 Oct 2018 00:49:15 +0100 Subject: [PATCH 106/394] Purge metadata entries on quit and escape glob characters --- core/bot.py | 2 ++ modules/userinfo.py | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index d733693..18e9956 100644 --- a/core/bot.py +++ b/core/bot.py @@ -383,6 +383,7 @@ class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name): self.instance = main.pool[name] self.name = name + self.net = "".join([x for x in self.name if not x in digits]) self.client = None self.maxDelay = self.instance["maxdelay"] self.initialDelay = self.instance["initialdelay"] @@ -396,6 +397,7 @@ class IRCBotFactory(ReconnectingClientFactory): return entry def clientConnectionLost(self, connector, reason): + userinfo.delNetwork(self.net, self.client.channels) if not self.client == None: self.client.connected = False self.client.channels = [] diff --git a/modules/userinfo.py b/modules/userinfo.py index ac4bd5a..42f2d48 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -100,9 +100,16 @@ def delUser(name, channel, nick, user): p.srem(chanspace, channel) p.execute() +def escape(text): + chars = ["[", "]", "^", "-", "*", "?"] + text = text.replace("\\", "\\\\") + for i in chars: + text = text.replace(i, "\\"+i) + return text + def getUserByNick(name, nick): gnamespace = "live.who.%s" % name - usermatch = main.r.sscan(gnamespace, match=nick+"!*") + usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=9999999) if usermatch[1] == []: return False else: @@ -110,6 +117,7 @@ def getUserByNick(name, nick): user = usermatch[1][0] return user else: + warn("Entry doesn't exist: %s on %s" % (nick, gnamespace)) return False def renameUser(name, oldnick, olduser, newnick, newuser): @@ -130,7 +138,6 @@ def renameUser(name, oldnick, olduser, newnick, newuser): p.execute() def delUserByNick(name, channel, nick): - gnamespace = "live.who.%s" % name user = getUserByNick(name, nick) delUser(name, channel, nick, user) @@ -153,8 +160,15 @@ def delChannel(name, channel): if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}: if user: p.srem(gnamespace, user) + p.delete("live.chan."+name+"."+i.decode()) else: p.srem("live.chan."+name+"."+i.decode(), channel) p.delete(namespace) p.execute() + +def delNetwork(name, channels): + log("Purging channels for %s" % name) + for i in channels: + delChannel(name, i) + return From 3bf361134fce6e1c003ad233c48e516c1daa3ddd Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 21 Oct 2018 17:14:50 +0100 Subject: [PATCH 107/394] Make the stats command aware of duplicate networks --- commands/stats.py | 21 ++++++++++++++++----- core/bot.py | 2 +- main.py | 10 ++++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/commands/stats.py b/commands/stats.py index 35d2d40..eab9a01 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,6 +1,7 @@ import main import modules.counters as count import modules.userinfo as userinfo +from string import digits class Stats: def __init__(self, register): @@ -15,8 +16,12 @@ class Stats: for i in main.IRCPool.keys(): numChannels += len(main.IRCPool[i].channels) numWhoEntries += userinfo.getNumTotalWhoEntries() - stats.append("Servers: %s" % len(main.pool.keys())) - stats.append("Online servers: %s" % len(main.IRCPool.keys())) + stats.append("Registered servers:") + stats.append(" Total: %s" % len(main.pool.keys())) + stats.append(" Unique: %s" % len(main.nets())) + stats.append("Online servers:") + stats.append(" Total: %s" % len(main.IRCPool.keys())) + stats.append(" Unique: %s" % len(main.liveNets())) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) counterEvents = count.getEvents() @@ -33,14 +38,20 @@ class Stats: stats = [] numChannels = 0 numWhoEntries = 0 + found = False + numNodes = 0 - if spl[1] in main.IRCPool.keys(): - numChannels += len(main.IRCPool[spl[1]].channels) - else: + for i in main.IRCPool.keys(): + if "".join([x for x in i if not x in digits]) == spl[1]: + numChannels += len(main.IRCPool[spl[1]].channels) + found = True + numNodes += 1 + if not found: failure("Name does not exist: %s" % spl[1]) numWhoEntries += userinfo.getNumWhoEntries(spl[1]) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) + stats.append("Endpoints: %s" % numNodes) counterEvents = count.getEvents(spl[1]) if counterEvents == None: stats.append("No counters records") diff --git a/core/bot.py b/core/bot.py index 18e9956..6bc0921 100644 --- a/core/bot.py +++ b/core/bot.py @@ -76,7 +76,7 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, channel, nick, user) + #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) diff --git a/main.py b/main.py index 24b78ef..cc08e94 100644 --- a/main.py +++ b/main.py @@ -31,8 +31,14 @@ def nets(): if not "pool" in globals(): return networks = set() - for i in pool: - networks.add("".join([x for x in i if not i in digits])) + for i in pool.keys(): + networks.add("".join([x for x in i if not x in digits])) + return networks + +def liveNets(): + networks = set() + for i in IRCPool.keys(): + networks.add("".join([x for x in i if not x in digits])) return networks def register(command, function): From 1c3435d0d7ffe01c7a60304a6de8ffc2849deb72 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 21 Oct 2018 19:21:53 +0100 Subject: [PATCH 108/394] Don't provision new user information on messages --- core/bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index 6bc0921..b001818 100644 --- a/core/bot.py +++ b/core/bot.py @@ -84,7 +84,7 @@ class IRCBot(IRCClient): def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, channel, nick, user) + #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) @@ -92,7 +92,7 @@ class IRCBot(IRCClient): def action(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, channel, nick, user) + #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) From e97792c460372e6a2e1ed1d15d2deddca5cd9dad Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 2 Nov 2018 22:58:20 +0000 Subject: [PATCH 109/394] Tweak help output for users and chans to indicate multiple arguments are possible --- conf/help.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/help.json b/conf/help.json index 47a0ff3..995abdb 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,6 +20,6 @@ "loadmod": "loadmod ", "msg": "msg ", "mon": "mon -h", - "chans": "chans ", - "users": "users " + "chans": "chans [ ...]", + "users": "users [ ...]" } From 6046329a83c3bb55833e52087cdc66895afc0e99 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 20 Jan 2019 19:56:54 +0000 Subject: [PATCH 110/394] Start implementing relay abstractions for smarter network handling and minor cosmetic changes --- .gitignore | 2 + commands/alias.py | 50 +++++++++++++++++++++ commands/load.py | 3 ++ commands/relay.py | 54 +++++++++++++++++++++++ commands/save.py | 2 + conf/example/{wholist.json => alias.json} | 0 conf/example/relay.json | 1 + conf/help.json | 8 ++-- main.py | 3 +- utils/logging/send.py | 12 +++-- 10 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 commands/alias.py create mode 100644 commands/relay.py rename conf/example/{wholist.json => alias.json} (100%) create mode 100644 conf/example/relay.json diff --git a/.gitignore b/.gitignore index 1b77df6..23ae7fa 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,7 @@ conf/keyword.json conf/counters.json conf/masterbuf.json conf/monitor.json +conf/alias.json +conf/relay.json conf/dist.sh env/ diff --git a/commands/alias.py b/commands/alias.py new file mode 100644 index 0000000..a466de8 --- /dev/null +++ b/commands/alias.py @@ -0,0 +1,50 @@ +import main +from yaml import dump + +class Alias: + def __init__(self, register): + register("alias", self.alias) + + def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 6: + if spl[1] == "add": + if spl[2] in main.alias.keys(): + failure("Alias already exists: %s" % spl[2]) + return + else: + main.alias[spl[2]] = {"nick": spl[3], + "ident": spl[4], + "realname": spl[5]} + success("Successfully created alias: %s" % spl[2]) + main.saveConf("alias") + return + else: + incUsage("alias") + return + + elif length == 3: + if spl[1] == "del": + if spl[2] in main.alias.keys(): + del main.alias[spl[2]] + success("Successfully removed alias: %s" % spl[2]) + main.saveConf("alias") + return + else: + failure("No such alias: %s" % spl[2]) + return + else: + incUsage("alias") + return + elif length == 2: + if spl[1] == "list": + info(dump(main.alias)) + return + else: + incUsage("alias") + return + else: + incUsage("alias") + return + else: + incUsage(None) diff --git a/commands/load.py b/commands/load.py index 91e168a..dc35292 100644 --- a/commands/load.py +++ b/commands/load.py @@ -16,6 +16,9 @@ class Load: main.loadConf(i) success("Loaded %s from %s" % (i, main.filemap[i][0])) return + elif spl[1] == "list": + info(", ".join(main.filemap.keys())) + return else: incUsage("load") return diff --git a/commands/relay.py b/commands/relay.py new file mode 100644 index 0000000..d611fc2 --- /dev/null +++ b/commands/relay.py @@ -0,0 +1,54 @@ +import main +from yaml import dump + +class Relay: + def __init__(self, register): + register("relay", self.relay) + + def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 7: + if spl[1] == "add": + if spl[2] in main.relay.keys(): + failure("Relay already exists: %s" % spl[2]) + return + else: + if not spl[4].isdigit(): + failure("Port must be an integer, not %s" % spl[4]) + return + main.relay[spl[2]] = {"host": spl[3], + "port": spl[4], + "user": spl[5], + "password": spl[6]} + success("Successfully created relay: %s" % spl[2]) + main.saveConf("relay") + return + else: + incUsage("relay") + return + + elif length == 3: + if spl[1] == "del": + if spl[2] in main.relay.keys(): + del main.relay[spl[2]] + success("Successfully removed relay: %s" % spl[2]) + main.saveConf("relay") + return + else: + failure("No such relay: %s" % spl[2]) + return + else: + incUsage("relay") + return + elif length == 2: + if spl[1] == "list": + info(dump(main.relay)) + return + else: + incUsage("relay") + return + else: + incUsage("relay") + return + else: + incUsage(None) diff --git a/commands/save.py b/commands/save.py index aebec15..492896b 100644 --- a/commands/save.py +++ b/commands/save.py @@ -16,6 +16,8 @@ class Save: main.saveConf(i) success("Saved %s to %s" % (i, main.filemap[i][0])) return + elif spl[1] == "list": + info(", ".join(main.filemap.keys())) else: incUsage("save") return diff --git a/conf/example/wholist.json b/conf/example/alias.json similarity index 100% rename from conf/example/wholist.json rename to conf/example/alias.json diff --git a/conf/example/relay.json b/conf/example/relay.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/relay.json @@ -0,0 +1 @@ +{} diff --git a/conf/help.json b/conf/help.json index 995abdb..d4d8c7b 100644 --- a/conf/help.json +++ b/conf/help.json @@ -14,12 +14,14 @@ "disable": "disable ", "list": "list", "stats": "stats []", - "save": "save ", - "load": "load ", + "save": "save <(file)|list|all>", + "load": "load <(file)|list|all>", "dist": "dist", "loadmod": "loadmod ", "msg": "msg ", "mon": "mon -h", "chans": "chans [ ...]", - "users": "users [ ...]" + "users": "users [ ...]", + "alias": "alias [ ]", + "relay": "relay [ " } diff --git a/main.py b/main.py index cc08e94..aa54dca 100644 --- a/main.py +++ b/main.py @@ -12,10 +12,11 @@ filemap = { "keyconf": ["keyword.json", "keyword lists"], "pool": ["pool.json", "pool"], "help": ["help.json", "command help"], -# "wholist": ["wholist.json", "WHO lists"], "counters": ["counters.json", "counters file"], "masterbuf": ["masterbuf.json", "master buffer"], "monitor": ["monitor.json", "monitoring database"], + "alias": ["alias.json", "alias details"], + "relay": ["relay.json", "relay list"], } connections = {} diff --git a/utils/logging/send.py b/utils/logging/send.py index dd6bfab..50b858a 100644 --- a/utils/logging/send.py +++ b/utils/logging/send.py @@ -3,14 +3,20 @@ import main def sendData(addr, data): main.connections[addr].send(data) +def sendWithPrefix(addr, data, prefix): + toSend = "" + for i in data.split("\n"): + toSend += prefix + " " + i + "\n" + sendData(addr, toSend) + def sendSuccess(addr, data): - sendData(addr, "[y] " + data) + sendWithPrefix(addr, data, "[y]") def sendFailure(addr, data): - sendData(addr, "[n] " + data) + sendWithPrefix(addr, data, "[n]") def sendInfo(addr, data): - sendData(addr, "[i] " + data) + sendWithPrefix(addr, data, "[i]") def sendAll(data): for i in main.connections: From 4efea3f5353a277e65d3b5171e7181a6fb394f55 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 26 Jan 2019 01:57:24 +0000 Subject: [PATCH 111/394] Implement the backend for automatically provisioning relays --- .gitignore | 1 + commands/alias.py | 8 ++- commands/default.py | 80 -------------------------- commands/network.py | 60 ++++++++++++++++++++ commands/provision.py | 59 +++++++++++++++++++ commands/relay.py | 6 +- conf/example/config.json | 38 ++++--------- conf/example/network.json | 1 + conf/help.json | 8 +-- core/bot.py | 115 +++++++++++++++++++++++++++++++++----- core/helper.py | 43 +++++++------- main.py | 3 +- modules/provision.py | 87 ++++++++++++++++++++++++++++ threshold | 4 +- utils/logging/send.py | 3 +- 15 files changed, 356 insertions(+), 160 deletions(-) delete mode 100644 commands/default.py create mode 100644 commands/network.py create mode 100644 commands/provision.py create mode 100644 conf/example/network.json create mode 100644 modules/provision.py diff --git a/.gitignore b/.gitignore index 23ae7fa..76cbb56 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ conf/masterbuf.json conf/monitor.json conf/alias.json conf/relay.json +conf/network.json conf/dist.sh env/ diff --git a/commands/alias.py b/commands/alias.py index a466de8..8baa85b 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -7,15 +7,17 @@ class Alias: def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 6: + if length == 8: if spl[1] == "add": if spl[2] in main.alias.keys(): failure("Alias already exists: %s" % spl[2]) return else: main.alias[spl[2]] = {"nick": spl[3], - "ident": spl[4], - "realname": spl[5]} + "altnick": spl[4], + "ident": spl[5], + "realname": spl[6], + "password": spl[7]} success("Successfully created alias: %s" % spl[2]) main.saveConf("alias") return diff --git a/commands/default.py b/commands/default.py deleted file mode 100644 index d432cf1..0000000 --- a/commands/default.py +++ /dev/null @@ -1,80 +0,0 @@ -import main - -class Default: - def __init__(self, register): - register("default", self.default) - - def default(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - toUnset = False - if length == 1: - optionMap = ["Viewing defaults"] - for i in main.config["Default"].keys(): - optionMap.append("%s: %s" % (i, main.config["Default"][i])) - info("\n".join(optionMap)) - return - elif length == 2: - if not spl[1] in main.config["Default"].keys(): - failure("No such key: %s" % spl[1]) - return - info("%s: %s" % (spl[1], main.config["Default"][spl[1]])) - return - elif length == 3: - if not spl[1] in main.config["Default"].keys(): - failure("No such key: %s" % spl[1]) - return - - if spl[2].lower() in ["none", "nil"]: - spl[2] = None - toUnset = True - - if spl[1] in ["port", "timeout", "maxdelay"]: - try: - spl[2] = int(spl[2]) - except: - failure("Value must be an integer, not %s" % spl[2]) - return - - if spl[2] in ["initialdelay", "factor", "jitter"]: - try: - spl[3] = float(spl[3]) - except: - failure("Value must be a floating point integer, not %s" % spl[3]) - return - - if spl[1] == "protocol": - if not toUnset: - if not spl[2] in ["ssl", "plain"]: - failure("Protocol must be ssl or plain, not %s" % spl[2]) - return - - if spl[2] == main.config["Default"][spl[1]]: - failure("Value already exists: %s" % spl[2]) - return - - if spl[1] == "authtype": - if not toUnset: - if not spl[2] in ["sp", "ns"]: - failure("Authtype must be sp or ns, not %s" % spl[2]) - return - if spl[1] == "enabled": - failure("Use the ConnectOnCreate main.config parameter to set this") - return - if spl[1] == "autojoin": - if not toUnset: - spl[2] = spl[2].split(",") - else: - spl[2] = [] - - main.config["Default"][spl[1]] = spl[2] - main.saveConf("config") - if toUnset: - success("Successfully unset key %s" % spl[1]) - else: - success("Successfully set key %s to %s" % (spl[1], spl[2])) - return - else: - incUsage("default") - return - else: - incUsage(None) diff --git a/commands/network.py b/commands/network.py new file mode 100644 index 0000000..e78d87d --- /dev/null +++ b/commands/network.py @@ -0,0 +1,60 @@ +import main +from yaml import dump + +class Network: + def __init__(self, register): + register("network", self.network) + + def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 7: + if spl[1] == "add": + if spl[2] in main.network.keys(): + failure("Network already exists: %s" % spl[2]) + return + if not spl[4].isdigit(): + failure("Port must be an integer, not %s" % spl[4]) + return + if not spl[5].lower() in ["ssl", "plain"]: + failure("Security must be ssl or plain, not %s" % spl[5]) + return + if not spl[6].lower() in ["sasl", "ns", "none"]: + failure("Auth must be sasl, ns or none, not %s" % spl[5]) + return + else: + main.network[spl[2]] = {"host": spl[3], + "port": spl[4], + "security": spl[5].lower(), + "auth": spl[6].lower()} + success("Successfully created network: %s" % spl[2]) + main.saveConf("network") + return + else: + incUsage("network") + return + + elif length == 3: + if spl[1] == "del": + if spl[2] in main.network.keys(): + del main.network[spl[2]] + success("Successfully removed network: %s" % spl[2]) + main.saveConf("network") + return + else: + failure("No such network: %s" % spl[2]) + return + else: + incUsage("network") + return + elif length == 2: + if spl[1] == "list": + info(dump(main.network)) + return + else: + incUsage("network") + return + else: + incUsage("network") + return + else: + incUsage(None) diff --git a/commands/provision.py b/commands/provision.py new file mode 100644 index 0000000..e1c2556 --- /dev/null +++ b/commands/provision.py @@ -0,0 +1,59 @@ +import main +from modules import provision + +class Provision: + def __init__(self, register): + register("provision", self.provision) + + def provision(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 4 or length == 3: + if not spl[1] in main.relay.keys(): + failure("No such relay: %s" % spl[1]) + return + if not spl[2] in main.alias.keys(): + failure("No such alias: %s" % spl[2]) + return + if length == 4: # provision for relay, alias and network + if not spl[3] in main.network.keys(): + failure("No such network: %s" % spl[3]) + return + + if "users" in main.relay[spl[1]]: + if not spl[2] in main.relay[spl[1]]["users"]: + failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) + return + else: + failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) + return + + rtrn = provision.provisionRelayForNetwork(spl[1], spl[2], spl[3]) + if rtrn == "PROVISIONED": + failure("Relay %s already provisioned for network %s" % (spl[1], spl[3])) + return + elif rtrn == "DUPLICATE": + failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) + elif rtrn: + success("Started provisioning network %s on relay %s for alias %s" % (spl[3], spl[1], spl[2])) + info("Instance name is %s" % rtrn) + return + else: + failure("Failure while provisioning relay %s" % spl[1]) + return + if length == 3: # provision for relay and alias only + rtrn = provision.provisionRelayForAlias(spl[1], spl[2]) + if rtrn == "PROVISIONED": + failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2])) + return + elif rtrn: + success("Started provisioning relay %s for alias %s" % (spl[1], spl[2])) + return + else: + failure("Failure while provisioning relay %s" % spl[1]) + return + + else: + incUsage("provision") + return + else: + incUsage(None) diff --git a/commands/relay.py b/commands/relay.py index d611fc2..0ad2bf4 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -12,10 +12,10 @@ class Relay: if spl[2] in main.relay.keys(): failure("Relay already exists: %s" % spl[2]) return + if not spl[4].isdigit(): + failure("Port must be an integer, not %s" % spl[4]) + return else: - if not spl[4].isdigit(): - failure("Port must be an integer, not %s" % spl[4]) - return main.relay[spl[2]] = {"host": spl[3], "port": spl[4], "user": spl[5], diff --git a/conf/example/config.json b/conf/example/config.json index 93a7527..2b2959d 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -2,13 +2,14 @@ "Listener": { "Port": 13867, "Address": "127.0.0.1", - "UseSSL": true, - "Key": "key.pem", - "Certificate": "cert.pem" + "UseSSL": true }, + "Key": "key.pem", + "Certificate": "cert.pem", "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, + "RelayPassword": "s", "Notifications": { "Highlight": true, "Connection": true, @@ -29,32 +30,13 @@ }, "Delays": { "WhoLoop": 600, - "WhoRange": 1800 + "WhoRange": 1800, + "Timeout": 30, + "MaxDelay": 360, + "InitialDelay": 1.0, + "Factor": 2.718281828459045, + "Jitter": 0.11962656472 } }, - "Default": { - "host": null, - "port": null, - "protocol": "ssl", - "bind": null, - "timeout": 30, - "maxdelay": 360, - "initialdelay": 1.0, - "factor": 2.7182818284590451, - "jitter": 0.11962656472, - "nickname": null, - "username": null, - "realname": null, - "userinfo": null, - "finger": null, - "version": null, - "source": null, - "autojoin": [], - "authtype": null, - "password": null, - "authentity": "NickServ", - "key": "key.pem", - "certificate": "cert.pem" - }, "Master": [null, null] } diff --git a/conf/example/network.json b/conf/example/network.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/network.json @@ -0,0 +1 @@ +{} diff --git a/conf/help.json b/conf/help.json index d4d8c7b..0c77572 100644 --- a/conf/help.json +++ b/conf/help.json @@ -1,10 +1,8 @@ { "pass": "pass ", "logout": "logout", - "add": "add [
] [] [] []", "del": "del ", "mod": "mod [] []", - "default": "default [] []", "get": "get ", "key": "key [] [] [] []", "who": "who ", @@ -22,6 +20,8 @@ "mon": "mon -h", "chans": "chans [ ...]", "users": "users [ ...]", - "alias": "alias [ ]", - "relay": "relay [ " + "alias": "alias [ ]", + "relay": "relay [ ]", + "network": "network [
]", + "provision": "provision []" } diff --git a/core/bot.py b/core/bot.py index b001818..3f063c6 100644 --- a/core/bot.py +++ b/core/bot.py @@ -1,7 +1,8 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater +from twisted.internet import reactor from string import digits from random import randint @@ -15,6 +16,76 @@ import main from utils.logging.log import * from utils.logging.send import * +from twisted.internet.ssl import DefaultOpenSSLContextFactory + +def deliverRelayCommands(relay, relayCommands, user=None, stage2=None): + keyFN = main.certPath+main.config["Key"] + certFN = main.certPath+main.config["Certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), + certFN.encode("utf-8", "replace")) + bot = IRCBotFactory(None, relay, relayCommands, user, stage2) + rct = reactor.connectSSL(main.relay[relay]["host"], + int(main.relay[relay]["port"]), + bot, contextFactory) + +class IRCRelay(IRCClient): + def __init__(self, relay, relayCommands, user, stage2): + self.connected = False + self.buffer = "" + if user == None: + self.user = main.relay[relay]["user"] + else: + self.user = user + password = main.relay[relay]["password"] + self.nickname = self.user + self.realname = self.user + self.username = self.user + self.password = self.user+":"+password + + self.relayCommands = relayCommands + self.relay = relay + self.stage2 = stage2 + + def parsen(self, user): + step = user.split("!") + nick = step[0] + if len(step) == 2: + step2 = step[1].split("@") + ident, host = step2 + else: + ident = nick + host = nick + + return [nick, ident, host] + + def privmsg(self, user, channel, msg): + nick, ident, host = self.parsen(user) + if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]: + nick = nick[1:] + if nick in self.relayCommands.keys(): + sendAll("[%s] %s -> %s" % (self.relay, nick, msg)) + + def irc_ERR_PASSWDMISMATCH(self, prefix, params): + log("%s: relay password mismatch" % self.relay) + sendAll("%s: relay password mismatch" % self.relay) + + def signedOn(self): + self.connected = True + log("signed on as a relay: %s" % self.relay) + if main.config["Notifications"]["Connection"]: + keyword.sendMaster("SIGNON: %s" % self.relay) + for i in self.relayCommands.keys(): + for x in self.relayCommands[i]: + self.msg(main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) + if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []] + if not len(self.stage2) == 0: + user = self.stage2[0].pop(0) + commands = self.stage2[0].pop(0) + del self.stage2[0] + deliverRelayCommands(self.relay, commands, user, self.stage2) + deferLater(reactor, 1, self.transport.loseConnection) + return + class IRCBot(IRCClient): def __init__(self, name): self.connected = False @@ -380,33 +451,46 @@ class IRCBot(IRCClient): monitor.event(self.net, channel, {"type": "mode", "exact": user, "nick": nick, "ident": ident, "host": host, "modes": m, "modeargs": a}) class IRCBotFactory(ReconnectingClientFactory): - def __init__(self, name): - self.instance = main.pool[name] - self.name = name - self.net = "".join([x for x in self.name if not x in digits]) + def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): + if not name == None: + self.name = name + self.instance = main.pool[name] + self.net = "".join([x for x in self.name if not x in digits]) + else: + self.name = "Relay to "+relay self.client = None - self.maxDelay = self.instance["maxdelay"] - self.initialDelay = self.instance["initialdelay"] - self.factor = self.instance["factor"] - self.jitter = self.instance["jitter"] + self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"] + self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"] + self.factor = main.config["Tweaks"]["Delays"]["Factor"] + self.jitter = main.config["Tweaks"]["Delays"]["Jitter"] + + self.relay, self.relayCommands, self.user, self.stage2 = relay, relayCommands, user, stage2 def buildProtocol(self, addr): - entry = IRCBot(self.name) - main.IRCPool[self.name] = entry + + if self.relay == None: + entry = IRCBot(self.name) + main.IRCPool[self.name] = entry + else: + entry = IRCRelay(self.relay, self.relayCommands, self.user, self.stage2) + self.client = entry return entry def clientConnectionLost(self, connector, reason): - userinfo.delNetwork(self.net, self.client.channels) + if not self.relay: + userinfo.delNetwork(self.net, self.client.channels) if not self.client == None: self.client.connected = False self.client.channels = [] error = reason.getErrorMessage() log("%s: connection lost: %s" % (self.name, error)) - sendAll("%s: connection lost: %s" % (self.name, error)) + if not self.relay: + sendAll("%s: connection lost: %s" % (self.name, error)) if main.config["Notifications"]["Connection"]: keyword.sendMaster("CONNLOST %s: %s" % (self.name, error)) - self.retry(connector) + if not self.relay: + self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): @@ -418,6 +502,7 @@ class IRCBotFactory(ReconnectingClientFactory): sendAll("%s: connection failed: %s" % (self.name, error)) if main.config["Notifications"]["Connection"]: keyword.sendMaster("CONNFAIL %s: %s" % (self.name, error)) - self.retry(connector) + if not self.relay: + self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/helper.py b/core/helper.py index 98e6364..570fd2c 100644 --- a/core/helper.py +++ b/core/helper.py @@ -1,35 +1,27 @@ from twisted.internet import reactor -from twisted.internet.ssl import DefaultOpenSSLContextFactory - from core.bot import IRCBot, IRCBotFactory import main from utils.logging.log import * def addBot(name): instance = main.pool[name] - log("Started bot %s to %s:%s protocol %s nickname %s" % (name, instance["host"], instance["port"], instance["protocol"], instance["nickname"])) - if instance["protocol"] == "plain": + log("Started bot %s to %s:%s protocol %s nickname %s" % (name, + instance["host"], + instance["port"], + instance["protocol"], + instance["nickname"])) + + + if instance["protocol"] == "ssl": + keyFN = main.certPath+main.config["Key"] + certFN = main.certPath+main.config["Certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), + certFN.encode("utf-8", "replace")) if instance["bind"] == None: bot = IRCBotFactory(name) - rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"])) - - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return - else: - bot = IRCBotFactory(name) - rct = reactor.connectTCP(instance["host"], instance["port"], bot, timeout=int(instance["timeout"]), bindAddress=instance["bind"]) - - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return - elif instance["protocol"] == "ssl": - keyFN = main.certPath+instance["key"] - certFN = main.certPath+instance["certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - if instance["bind"] == None: - bot = IRCBotFactory(name) - rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory) + rct = reactor.connectSSL(instance["host"], + int(instance["port"]), + bot, contextFactory) main.ReactorPool[name] = rct main.FactoryPool[name] = bot @@ -37,7 +29,10 @@ def addBot(name): else: bot = IRCBotFactory(name) - rct = reactor.connectSSL(instance["host"], int(instance["port"]), bot, contextFactory, bindAddress=instance["bind"]) + rct = reactor.connectSSL(instance["host"], + int(instance["port"]), + bot, contextFactory, + bindAddress=instance["bind"]) main.ReactorPool[name] = rct main.FactoryPool[name] = bot diff --git a/main.py b/main.py index aa54dca..75fe1e7 100644 --- a/main.py +++ b/main.py @@ -10,13 +10,14 @@ certPath = "cert/" filemap = { "config": ["config.json", "configuration"], "keyconf": ["keyword.json", "keyword lists"], - "pool": ["pool.json", "pool"], + "pool": ["pool.json", "network, alias and relay mappings"], "help": ["help.json", "command help"], "counters": ["counters.json", "counters file"], "masterbuf": ["masterbuf.json", "master buffer"], "monitor": ["monitor.json", "monitoring database"], "alias": ["alias.json", "alias details"], "relay": ["relay.json", "relay list"], + "network": ["network.json", "network list"], } connections = {} diff --git a/modules/provision.py b/modules/provision.py new file mode 100644 index 0000000..8d5f96a --- /dev/null +++ b/modules/provision.py @@ -0,0 +1,87 @@ +import main +from core.bot import deliverRelayCommands +from utils.logging.log import * + +def provisionUserData(relay, alias, nick, altnick, ident, realname, password): + commands = {} + commands["controlpanel"] = [] + commands["controlpanel"].append("AddUser %s %s" % (alias, password)) + commands["controlpanel"].append("Set Nick %s %s" % (alias, nick)) + commands["controlpanel"].append("Set Altnick %s %s" % (alias, altnick)) + commands["controlpanel"].append("Set Ident %s %s" % (alias, ident)) + commands["controlpanel"].append("Set RealName %s %s" % (alias, realname)) + deliverRelayCommands(relay, commands) + return + +def provisionNetworkData(relay, alias, network, host, port, security, auth, password): + commands = {} + stage2commands = {} + commands["controlpanel"] = [] + commands["controlpanel"].append("AddNetwork %s %s" % (alias, network)) + if security == "ssl": + commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (alias, network)) # Don't judge me + commands["controlpanel"].append("AddServer %s %s %s +%s" % (alias, network, host, port)) + elif security == "plain": + commands["controlpanel"].append("AddServer %s %s %s %s" % (alias, network, host, port)) + if auth == "sasl": + stage2commands["status"] = [] + stage2commands["sasl"] = [] + stage2commands["status"].append("LoadMod sasl") + stage2commands["sasl"].append("Mechanism plain") + stage2commands["sasl"].append("Set %s %s" % (alias, password)) + elif auth == "ns": + stage2commands["status"] = [] + stage2commands["nickserv"] = [] + stage2commands["status"].append("LoadMod NickServ") + stage2commands["nickserv"].append("Set %s" % password) + deliverRelayCommands(relay, commands, stage2=[[alias+"/"+network, stage2commands]]) + return + +def provisionRelayForAlias(relay, alias): + if "users" in main.relay[relay].keys(): + if alias in main.relay[relay]["users"]: + return "PROVISIONED" + else: + main.relay[relay]["users"] = [] + main.relay[relay]["users"].append(alias) + provisionUserData(relay, alias, main.alias[alias]["nick"], + main.alias[alias]["altnick"], + main.alias[alias]["ident"], + main.alias[alias]["realname"], + main.relay[relay]["password"]) + main.saveConf("relay") + return True + +def provisionRelayForNetwork(relay, alias, network): + if "networks" in main.relay[relay].keys(): + if network in main.relay[relay]["networks"]: + return "PROVISIONED" + else: + main.relay[relay]["networks"] = [] + main.relay[relay]["networks"].append(network) + provisionNetworkData(relay, alias, network, + main.network[network]["host"], + main.network[network]["port"], + main.network[network]["security"], + main.network[network]["auth"], + main.alias[alias]["password"]) + main.saveConf("relay") + storedNetwork = False + num = 1 + while not storedNetwork: + i = str(num) + if num == 1000: + error("Too many iterations in while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network)) + return False + + if network+i in main.pool.keys(): + if main.pool[network+i]["alias"] == alias and main.pool[network+i]["relay"] == relay: + return "DUPLICATE" + num += 1 + else: + main.pool[network+i] = {"relay": relay, + "alias": alias, + "network": network} + main.saveConf("pool") + storedNetwork = True + return network+i diff --git a/threshold b/threshold index 71872a5..adab5cf 100755 --- a/threshold +++ b/threshold @@ -18,12 +18,14 @@ from core.server import Server, ServerFactory if __name__ == "__main__": listener = ServerFactory() if main.config["Listener"]["UseSSL"] == True: - reactor.listenSSL(main.config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(main.certPath+main.config["Listener"]["Key"], main.certPath+main.config["Listener"]["Certificate"]), interface=main.config["Listener"]["Address"]) + reactor.listenSSL(main.config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["Listener"]["Address"]) log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) else: reactor.listenTCP(main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"]) log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) for i in main.pool.keys(): + if not "enabled" in main.pool[i]: + continue if main.pool[i]["enabled"] == True: helper.addBot(i) diff --git a/utils/logging/send.py b/utils/logging/send.py index 50b858a..0f4eeee 100644 --- a/utils/logging/send.py +++ b/utils/logging/send.py @@ -20,7 +20,8 @@ def sendInfo(addr, data): def sendAll(data): for i in main.connections: - main.connections[i].send(data) + if main.connections[i].authed: + main.connections[i].send(data) return def incorrectUsage(addr, mode): From 8926cb76eca6f601b633a3da3b2f7aa5a6a3087f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 26 Jan 2019 18:58:21 +0000 Subject: [PATCH 112/394] Renovate the module system and implement adding and resuming pool instances using the new relay/alias/network system --- commands/add.py | 111 ------------------------------ commands/alias.py | 4 +- commands/chans.py | 4 +- commands/cmd.py | 24 +++++++ commands/{delete.py => del.py} | 6 +- commands/disable.py | 10 ++- commands/dist.py | 4 +- commands/enable.py | 14 ++-- commands/get.py | 4 +- commands/help.py | 4 +- commands/join.py | 4 +- commands/key.py | 4 +- commands/list.py | 4 +- commands/load.py | 4 +- commands/loadmod.py | 11 +-- commands/logout.py | 4 +- commands/mod.py | 41 +---------- commands/mon.py | 4 +- commands/msg.py | 4 +- commands/network.py | 4 +- commands/part.py | 4 +- commands/{password.py => pass.py} | 6 +- commands/provision.py | 4 +- commands/relay.py | 4 +- commands/save.py | 4 +- commands/stats.py | 4 +- commands/users.py | 4 +- commands/who.py | 4 +- conf/help.json | 3 +- core/bot.py | 46 ++++--------- core/helper.py | 57 ++++++++------- core/parser.py | 7 +- main.py | 13 +--- modules/provision.py | 6 +- threshold | 9 +-- utils/loaders/command_loader.py | 13 +++- utils/loaders/single_loader.py | 29 +++++--- 37 files changed, 184 insertions(+), 302 deletions(-) delete mode 100644 commands/add.py create mode 100644 commands/cmd.py rename commands/{delete.py => del.py} (92%) rename commands/{password.py => pass.py} (87%) diff --git a/commands/add.py b/commands/add.py deleted file mode 100644 index e2eac5d..0000000 --- a/commands/add.py +++ /dev/null @@ -1,111 +0,0 @@ -import main -import core.helper as helper - -class Add: - def __init__(self, register): - register("add", self.add) - - def add(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - if length > 6: - failure("Too many arguments") - return - - if length > 1: - name = spl[1] - if name.isdigit(): - failure("Network name is all numbers. This will break things.") - return - else: - incUsage("add") - return - if length > 2: - host = spl[2] - if length > 3: - port = spl[3] - if length > 4: - protocol = spl[4] - if length > 5: - nickname = spl[5] - - toFail = False - if length < 6: - if main.config["Default"]["nickname"] == None: - failure("Choose a nickname, or configure one in defaults") - toFail = True - else: - nickname = main.config["Default"]["nickname"] - - if length < 5: - if main.config["Default"]["protocol"] == None: - failure("Choose a protocol, or configure one in defaults") - toFail = True - else: - protocol = main.config["Default"]["protocol"] - - if length < 4: - if main.config["Default"]["port"] == None: - failure("Choose a port, or configure one in defaults") - toFail = True - else: - port = main.config["Default"]["port"] - - if length < 3: - if main.config["Default"]["host"] == None: - failure("Choose a host, or configure one in defaults") - toFail = True - else: - host = main.config["Default"]["host"] - if toFail: - failure("Stopping due to previous error(s)") - return - - if length < 2: - incUsage("add") - return - - if name in main.pool.keys(): - failure("Name already exists: %s" % name) - return - - protocol = protocol.lower() - - if not protocol in ["ssl", "plain"]: - failure("Protocol must be ssl or plain, not %s" % protocol) - return - - if not port.isdigit(): - failure("Port must be an integer, not %s" % port) - return - - main.pool[name] = { "host": host, - "port": port, - "protocol": protocol, - "bind": main.config["Default"]["bind"], - "timeout": main.config["Default"]["timeout"], - "maxdelay": main.config["Default"]["maxdelay"], - "initialdelay": main.config["Default"]["initialdelay"], - "factor": main.config["Default"]["factor"], - "jitter": main.config["Default"]["jitter"], - "nickname": nickname, - "username": main.config["Default"]["username"], - "realname": main.config["Default"]["realname"], - "userinfo": main.config["Default"]["userinfo"], - "finger": main.config["Default"]["finger"], - "version": main.config["Default"]["version"], - "source": main.config["Default"]["source"], - "autojoin": main.config["Default"]["autojoin"], - "authtype": main.config["Default"]["authtype"], - "password": main.config["Default"]["password"], - "authentity": main.config["Default"]["authentity"], - "key": main.config["Default"]["key"], - "certificate": main.config["Default"]["certificate"], - "enabled": main.config["ConnectOnCreate"], - } - if main.config["ConnectOnCreate"] == True: - helper.addBot(name) - success("Successfully created bot") - main.saveConf("pool") - return - else: - incUsage(None) diff --git a/commands/alias.py b/commands/alias.py index 8baa85b..0b127ed 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -2,8 +2,8 @@ import main from yaml import dump class Alias: - def __init__(self, register): - register("alias", self.alias) + def __init__(self, *args): + self.alias(*args) def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/chans.py b/commands/chans.py index 851b9e8..3f00f43 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -2,8 +2,8 @@ import main import modules.userinfo as userinfo class Chans: - def __init__(self, register): - register("chans", self.chans) + def __init__(self, *args): + self.chans(*args) def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/cmd.py b/commands/cmd.py new file mode 100644 index 0000000..5908bda --- /dev/null +++ b/commands/cmd.py @@ -0,0 +1,24 @@ +import main +from core.bot import deliverRelayCommands + +class Cmd: + def __init__(self, *args): + self.cmd(*args) + + def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 4: + if not spl[1] in main.relay.keys(): + failure("No such relay: %s" % spl[1]) + return + + commands = {spl[3]: [" ".join(spl[4:])]} + print(" ".join(spl[4:])) + success("Sending commands to relay %s as user %s" % (spl[1], spl[2])) + deliverRelayCommands(spl[1], commands, user=spl[2]) + return + else: + incUsage("cmd") + return + else: + incUsage(None) diff --git a/commands/delete.py b/commands/del.py similarity index 92% rename from commands/delete.py rename to commands/del.py index 2476363..9d4868a 100644 --- a/commands/delete.py +++ b/commands/del.py @@ -1,8 +1,8 @@ import main -class Delete: - def __init__(self, register): - register("del", self.delete) +class Del: + def __init__(self, *args): + self.delete(*args) def delete(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/disable.py b/commands/disable.py index 681f155..97c5742 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -1,8 +1,9 @@ import main +from core.bot import deliverRelayCommands class Disable: - def __init__(self, register): - register("disable", self.disable) + def __init__(self, *args): + self.disable(*args) def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: @@ -11,6 +12,11 @@ class Disable: failure("Name does not exist: %s" % spl[1]) return main.pool[spl[1]]["enabled"] = False + user = main.pool[spl[1]]["alias"] + network = main.pool[spl[1]]["network"] + relay = main.pool[spl[1]]["relay"] + commands = {"status": ["Disconnect"]} + deliverRelayCommands(relay, commands, user=user+"/"+network) main.saveConf("pool") if spl[1] in main.ReactorPool.keys(): if spl[1] in main.FactoryPool.keys(): diff --git a/commands/dist.py b/commands/dist.py index 6fd579c..3c94f5d 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -2,8 +2,8 @@ import main from subprocess import run, PIPE class Dist: - def __init__(self, register): - register("dist", self.dist) + def __init__(self, *args): + self.dist(*args) def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/enable.py b/commands/enable.py index 3fc7a2d..d6a29c5 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -1,9 +1,10 @@ import main -import core.helper as helper +from core.helper import startBot +from core.bot import deliverRelayCommands class Enable: - def __init__(self, register): - register("enable", self.enable) + def __init__(self, *args): + self.enable(*args) def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: @@ -12,9 +13,14 @@ class Enable: failure("Name does not exist: %s" % spl[1]) return main.pool[spl[1]]["enabled"] = True + user = main.pool[spl[1]]["alias"] + network = main.pool[spl[1]]["network"] + relay = main.pool[spl[1]]["relay"] + commands = {"status": ["Connect"]} + deliverRelayCommands(relay, commands, user=user+"/"+network) main.saveConf("pool") if not spl[1] in main.IRCPool.keys(): - helper.addBot(spl[1]) + startBot(spl[1]) else: pass success("Successfully enabled bot %s" % spl[1]) diff --git a/commands/get.py b/commands/get.py index 3c302b8..51ccd7f 100644 --- a/commands/get.py +++ b/commands/get.py @@ -1,8 +1,8 @@ import main class Get: - def __init__(self, register): - register("get", self.get) + def __init__(self, *args): + self.get(*args) def get(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/help.py b/commands/help.py index 46a9782..6e3e229 100644 --- a/commands/help.py +++ b/commands/help.py @@ -1,8 +1,8 @@ import main class Help: - def __init__(self, register): - register("help", self.help) + def __init__(self, *args): + self.help(*args) def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/join.py b/commands/join.py index 3be5708..0b1c223 100644 --- a/commands/join.py +++ b/commands/join.py @@ -1,8 +1,8 @@ import main class Join: - def __init__(self, register): - register("join", self.join) + def __init__(self, *args): + self.join(*args) def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/key.py b/commands/key.py index e8fb4db..d6b88a7 100644 --- a/commands/key.py +++ b/commands/key.py @@ -2,8 +2,8 @@ import main import modules.keyword as keyword class Key: - def __init__(self, register): - register("key", self.key) + def __init__(self, *args): + self.key(*args) def key(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/list.py b/commands/list.py index 13be3eb..3edef6d 100644 --- a/commands/list.py +++ b/commands/list.py @@ -2,8 +2,8 @@ import main from yaml import dump class List: - def __init__(self, register): - register("list", self.list) + def __init__(self, *args): + self.list(*args) def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/load.py b/commands/load.py index dc35292..8038c5b 100644 --- a/commands/load.py +++ b/commands/load.py @@ -1,8 +1,8 @@ import main class Load: - def __init__(self, register): - register("load", self.load) + def __init__(self, *args): + self.list(*args) def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/loadmod.py b/commands/loadmod.py index 18a403e..c5c463f 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -2,15 +2,18 @@ import main from utils.loaders.single_loader import loadSingle class Loadmod: - def __init__(self, register): - register("loadmod", self.loadmod) + def __init__(self, *args): + self.loadmod(*args) def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: - rtrn = loadSingle(spl[1], main.register) + rtrn = loadSingle(spl[1]) if rtrn == True: - success("Loaded module %s" % spl[1]) + success("Loaded module: %s" % spl[1]) + return + elif rtrn == "RELOAD": + success("Reloaded module: %s" % spl[1]) return else: failure("Error loading module %s: %s" % (spl[1], rtrn)) diff --git a/commands/logout.py b/commands/logout.py index a50ea02..960ac1e 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -1,8 +1,8 @@ import main class Logout: - def __init__(self, register): - register("logout", self.logout) + def __init__(self, *args): + self.logout(*args) def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/mod.py b/commands/mod.py index 0606c40..d11b33b 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -2,12 +2,11 @@ import main from yaml import dump class Mod: - def __init__(self, register): - register("mod", self.mod) + def __init__(self, *args): + self.mod(*args) def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - toUnset = False if length == 2: if not spl[1] in main.pool.keys(): failure("Name does not exist: %s" % spl[1]) @@ -33,51 +32,17 @@ class Mod: failure("No such key: %s" % spl[2]) return - if spl[2] == "protocol": - if not spl[3] in ["ssl", "plain"]: - failure("Protocol must be ssl or plain, not %s" % spl[3]) - return if spl[3] == main.pool[spl[1]][spl[2]]: failure("Value already exists: %s" % spl[3]) return - if spl[3].lower() in ["none", "nil"]: - spl[3] = None - toUnset = True - - if spl[2] in ["port", "timeout", "maxdelay"]: - try: - spl[3] = int(spl[3]) - except: - failure("Value must be an integer, not %s" % spl[3]) - return - - if spl[2] in ["initialdelay", "factor", "jitter"]: - try: - spl[3] = float(spl[3]) - except: - failure("Value must be a floating point integer, not %s" % spl[3]) - return - - if spl[2] == "authtype": - if not toUnset: - if not spl[3] in ["sp", "ns"]: - failure("Authtype must be sp or ns, not %s" % spl[3]) - return if spl[2] == "enabled": failure("Use the enable and disable commands to manage this") return - if spl[2] == "autojoin": - spl[3] = spl[3].split(",") main.pool[spl[1]][spl[2]] = spl[3] - if spl[1] in main.IRCPool.keys(): - main.IRCPool[spl[1]].refresh() main.saveConf("pool") - if toUnset: - success("Successfully unset key %s on %s" % (spl[2], spl[1])) - else: - success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) + success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return else: diff --git a/commands/mon.py b/commands/mon.py index 214b363..0a05d7d 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -5,8 +5,8 @@ from io import StringIO from yaml import dump class Mon: - def __init__(self, register): - register("mon", self.mon) + def __init__(self, *args): + self.mon(*args) def setup_arguments(self, ArgumentParser): self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.") diff --git a/commands/msg.py b/commands/msg.py index a3bad7e..3638c28 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -1,8 +1,8 @@ import main class Msg: - def __init__(self, register): - register("msg", self.msg) + def __init__(self, *args): + self.msg(*args) def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/network.py b/commands/network.py index e78d87d..024042e 100644 --- a/commands/network.py +++ b/commands/network.py @@ -2,8 +2,8 @@ import main from yaml import dump class Network: - def __init__(self, register): - register("network", self.network) + def __init__(self, *args): + self.network(*args) def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/part.py b/commands/part.py index a1adacb..67bf539 100644 --- a/commands/part.py +++ b/commands/part.py @@ -1,8 +1,8 @@ import main class Part: - def __init__(self, register): - register("part", self.part) + def __init__(self, *args): + self.part(*args) def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/password.py b/commands/pass.py similarity index 87% rename from commands/password.py rename to commands/pass.py index 6f88617..20f4d9a 100644 --- a/commands/password.py +++ b/commands/pass.py @@ -1,8 +1,8 @@ import main -class Password: - def __init__(self, register): - register("pass", self.password) +class Pass: + def __init__(self, *args): + self.password(*args) def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/provision.py b/commands/provision.py index e1c2556..a9df1b3 100644 --- a/commands/provision.py +++ b/commands/provision.py @@ -2,8 +2,8 @@ import main from modules import provision class Provision: - def __init__(self, register): - register("provision", self.provision) + def __init__(self, *args): + self.provision(*args) def provision(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/relay.py b/commands/relay.py index 0ad2bf4..894c80b 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -2,8 +2,8 @@ import main from yaml import dump class Relay: - def __init__(self, register): - register("relay", self.relay) + def __init__(self, *args): + self.relay(*args) def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/save.py b/commands/save.py index 492896b..edd4d1c 100644 --- a/commands/save.py +++ b/commands/save.py @@ -1,8 +1,8 @@ import main class Save: - def __init__(self, register): - register("save", self.save) + def __init__(self, *args): + self.save(*args) def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/stats.py b/commands/stats.py index eab9a01..5b08b7e 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -4,8 +4,8 @@ import modules.userinfo as userinfo from string import digits class Stats: - def __init__(self, register): - register("stats", self.stats) + def __init__(self, *args): + self.stats(*args) def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/users.py b/commands/users.py index a76e20e..9836a41 100644 --- a/commands/users.py +++ b/commands/users.py @@ -2,8 +2,8 @@ import main import modules.userinfo as userinfo class Users: - def __init__(self, register): - register("users", self.users) + def __init__(self, *args): + self.users(*args) def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/commands/who.py b/commands/who.py index 9898cfc..7ae4909 100644 --- a/commands/who.py +++ b/commands/who.py @@ -2,8 +2,8 @@ import main import modules.userinfo as userinfo class Who: - def __init__(self, register): - register("who", self.who) + def __init__(self, *args): + self.who(*args) def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/conf/help.json b/conf/help.json index 0c77572..2ecefc7 100644 --- a/conf/help.json +++ b/conf/help.json @@ -23,5 +23,6 @@ "alias": "alias [ ]", "relay": "relay [ ]", "network": "network [
]", - "provision": "provision []" + "provision": "provision []", + "cmd": "cmd " } diff --git a/core/bot.py b/core/bot.py index 3f063c6..2d54174 100644 --- a/core/bot.py +++ b/core/bot.py @@ -95,44 +95,27 @@ class IRCBot(IRCClient): error("Network with all numbers: %s" % name) self.buffer = "" self.name = name - instance = main.pool[name] + inst = main.pool[name] + alias = main.alias[inst["alias"]] + relay = main.relay[inst["relay"]] + network = main.network[inst["network"]] - self.nickname = instance["nickname"] - self.realname = instance["realname"] - self.username = instance["username"] - self.userinfo = instance["userinfo"] - self.fingerReply = instance["finger"] - self.versionName = instance["version"] + self.nickname = alias["nick"] + self.realname = alias["realname"] + self.username = inst["alias"]+"/"+inst["network"] + self.password = relay["password"] + self.userinfo = None + self.fingerReply = None + self.versionName = None self.versionNum = None self.versionEnv = None - self.sourceURL = instance["source"] - self.autojoin = instance["autojoin"] + self.sourceURL = None self._who = {} self._getWho = {} self._names = {} - self.authtype = instance["authtype"] - if self.authtype == "ns": - self.authpass = instance["password"] - self.authentity = instance["authentity"] - else: - self.password = instance["password"] - - def refresh(self): - instance = main.pool[self.name] - if not instance["nickname"] == self.nickname: - self.nickname = instance["nickname"] - self.setNick(self.nickname) - - self.userinfo = instance["userinfo"] - self.fingerReply = instance["finger"] - self.versionName = instance["version"] - self.versionNum = None - self.versionEnv = None - self.sourceURL = instance["source"] - def parsen(self, user): step = user.split("!") nick = step[0] @@ -350,10 +333,6 @@ class IRCBot(IRCClient): log("signed on: %s" % self.name) if main.config["Notifications"]["Connection"]: keyword.sendMaster("SIGNON: %s" % self.name) - if self.authtype == "ns": - self.msg(self.authentity, "IDENTIFY %s" % self.nspass) - for i in self.autojoin: - self.join(i) count.event(self.net, "signedon") def joined(self, channel): @@ -454,7 +433,6 @@ class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): if not name == None: self.name = name - self.instance = main.pool[name] self.net = "".join([x for x in self.name if not x in digits]) else: self.name = "Relay to "+relay diff --git a/core/helper.py b/core/helper.py index 570fd2c..fa6af95 100644 --- a/core/helper.py +++ b/core/helper.py @@ -1,39 +1,38 @@ from twisted.internet import reactor from core.bot import IRCBot, IRCBotFactory +from twisted.internet.ssl import DefaultOpenSSLContextFactory import main from utils.logging.log import * -def addBot(name): - instance = main.pool[name] - log("Started bot %s to %s:%s protocol %s nickname %s" % (name, - instance["host"], - instance["port"], - instance["protocol"], - instance["nickname"])) +def startBot(name): + inst = main.pool[name] + relay, alias, network = inst["relay"], inst["alias"], inst["network"] + host = main.relay[relay]["host"] + port = int(main.relay[relay]["port"]) + log("Started bot %s to %s network %s" % (name, relay, network)) - if instance["protocol"] == "ssl": - keyFN = main.certPath+main.config["Key"] - certFN = main.certPath+main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), - certFN.encode("utf-8", "replace")) - if instance["bind"] == None: - bot = IRCBotFactory(name) - rct = reactor.connectSSL(instance["host"], - int(instance["port"]), - bot, contextFactory) + keyFN = main.certPath+main.config["Key"] + certFN = main.certPath+main.config["Certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), + certFN.encode("utf-8", "replace")) + if "bind" in main.relay[relay].keys(): + bot = IRCBotFactory(name) + rct = reactor.connectSSL(host, + port, + bot, contextFactory, + bindAddress=main.relay[relay]["bind"]) - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return - else: + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot + return + else: + bot = IRCBotFactory(name) + rct = reactor.connectSSL(host, + port, + bot, contextFactory) - bot = IRCBotFactory(name) - rct = reactor.connectSSL(instance["host"], - int(instance["port"]), - bot, contextFactory, - bindAddress=instance["bind"]) + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot + return - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return diff --git a/core/parser.py b/core/parser.py index a8a001e..64270f8 100644 --- a/core/parser.py +++ b/core/parser.py @@ -22,9 +22,8 @@ def parseCommand(addr, authed, data): else: failure("No text was sent") return - for i in main.CommandMap.keys(): - if spl[0] == i: - main.CommandMap[i](addr, authed, data, obj, spl, success, failure, info, incUsage, length) - return + if spl[0] in main.CommandMap.keys(): + main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length) + return incUsage(None) return diff --git a/main.py b/main.py index 75fe1e7..05c8deb 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,6 @@ from json import load, dump, loads -import redis +from redis import StrictRedis from string import digits -from utils.loaders.command_loader import loadCommands from utils.logging.log import * configPath = "conf/" @@ -43,13 +42,6 @@ def liveNets(): networks.add("".join([x for x in i if not x in digits])) return networks -def register(command, function): - if not command in CommandMap: - CommandMap[command] = function - debug("Registered command: %s" % command) - else: - error("Duplicate command: %s" % (command)) - def saveConf(var): with open(configPath+filemap[var][0], "w") as f: dump(globals()[var], f, indent=4) @@ -66,7 +58,6 @@ def initConf(): def initMain(): global r initConf() - loadCommands(register) - r = redis.StrictRedis(unix_socket_path=config["RedisSocket"], db=0) + r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) diff --git a/modules/provision.py b/modules/provision.py index 8d5f96a..34438c0 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands from utils.logging.log import * +from core.helper import startBot def provisionUserData(relay, alias, nick, altnick, ident, realname, password): commands = {} @@ -81,7 +82,10 @@ def provisionRelayForNetwork(relay, alias, network): else: main.pool[network+i] = {"relay": relay, "alias": alias, - "network": network} + "network": network, + "enabled": main.config["ConnectOnCreate"]} main.saveConf("pool") + if main.config["ConnectOnCreate"]: + startBot(network+i) storedNetwork = True return network+i diff --git a/threshold b/threshold index adab5cf..5ca234a 100755 --- a/threshold +++ b/threshold @@ -1,7 +1,6 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory - #from twisted.python import log #from sys import stdout #log.startLogging(stdout) @@ -11,10 +10,12 @@ import main main.initMain() from utils.logging.log import * -import modules.userinfo as userinfo -import core.helper as helper +from utils.loaders.command_loader import loadCommands +from core.helper import startBot from core.server import Server, ServerFactory +loadCommands() + if __name__ == "__main__": listener = ServerFactory() if main.config["Listener"]["UseSSL"] == True: @@ -27,6 +28,6 @@ if __name__ == "__main__": if not "enabled" in main.pool[i]: continue if main.pool[i]["enabled"] == True: - helper.addBot(i) + startBot(i) reactor.run() diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 62bc734..775d267 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,14 +1,21 @@ from os import listdir + from utils.logging.log import * import commands -def loadCommands(func): +from main import CommandMap + +def loadCommands(): for filename in listdir('commands'): if filename.endswith('.py') and filename != "__init__.py": commandName = filename[0:-3] className = commandName.capitalize() try: - __import__('commands.%s' % commandName) - eval('commands.%s.%s(func)' % (commandName, className)) + module = __import__('commands.%s' % commandName) + if not commandName in CommandMap: + CommandMap[commandName] = getattr(getattr(module, commandName), className) + debug("Registered command: %s" % commandName) + else: + error("Duplicate command: %s" % (commandName)) except Exception as err: error("Exception while loading command %s:\n%s" % (commandName, err)) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index 543a6b5..3d891e2 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -1,17 +1,26 @@ from os import listdir -import main +from importlib import reload +from sys import modules + from utils.logging.log import * import commands -def loadSingle(command, func): - if command+".py" in listdir("commands"): +from main import CommandMap + +def loadSingle(commandName): + if commandName+".py" in listdir("commands"): + className = commandName.capitalize() try: - if command in main.CommandMap.keys(): - return "Cannot reload modules" - else: - className = command.capitalize() - __import__("commands.%s" % command) - eval("commands.%s.%s(func)" % (command, className)) - return True + if commandName in CommandMap.keys(): + reload(modules["commands."+commandName]) + CommandMap[commandName] = getattr(modules["commands."+commandName], className) + debug("Reloaded command: %s" % commandName) + return "RELOAD" + module = __import__('commands.%s' % commandName) + CommandMap[commandName] = getattr(getattr(module, commandName), className) + debug("Registered command: %s" % commandName) + return True + except Exception as err: return err + return False From a4b7bd50b1d28b285e5849740a1ae5cf339f2421 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 1 Feb 2019 23:26:01 +0000 Subject: [PATCH 113/394] Implement additional error checking for provisioning instances and parsing messages, and prevent ZNC from auto-connecting an instance if ConnectOnCreate is off --- commands/provision.py | 3 ++- core/bot.py | 6 +++++- core/server.py | 2 +- modules/provision.py | 7 +++++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/commands/provision.py b/commands/provision.py index a9df1b3..3453171 100644 --- a/commands/provision.py +++ b/commands/provision.py @@ -29,10 +29,11 @@ class Provision: rtrn = provision.provisionRelayForNetwork(spl[1], spl[2], spl[3]) if rtrn == "PROVISIONED": - failure("Relay %s already provisioned for network %s" % (spl[1], spl[3])) + failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3])) return elif rtrn == "DUPLICATE": failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) + return elif rtrn: success("Started provisioning network %s on relay %s for alias %s" % (spl[3], spl[1], spl[2])) info("Instance name is %s" % rtrn) diff --git a/core/bot.py b/core/bot.py index 2d54174..29e4223 100644 --- a/core/bot.py +++ b/core/bot.py @@ -121,7 +121,11 @@ class IRCBot(IRCClient): nick = step[0] if len(step) == 2: step2 = step[1].split("@") - ident, host = step2 + if len(step2) == 2: + ident, host = step2 + else: + ident = nick + host = nick else: ident = nick host = nick diff --git a/core/server.py b/core/server.py index 2f40ccf..30ca24d 100644 --- a/core/server.py +++ b/core/server.py @@ -29,7 +29,7 @@ class Server(Protocol): def connectionMade(self): log("Connection from %s:%s" % (self.addr.host, self.addr.port)) - self.send("Hello.") + self.send("Greetings.") def connectionLost(self, reason): self.authed = False diff --git a/modules/provision.py b/modules/provision.py index 34438c0..63e00a2 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -54,8 +54,8 @@ def provisionRelayForAlias(relay, alias): return True def provisionRelayForNetwork(relay, alias, network): - if "networks" in main.relay[relay].keys(): - if network in main.relay[relay]["networks"]: + if all([x in ["users", "networks"] for x in main.relay[relay].keys() if x in ["users", "networks"]]): + if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: return "PROVISIONED" else: main.relay[relay]["networks"] = [] @@ -87,5 +87,8 @@ def provisionRelayForNetwork(relay, alias, network): main.saveConf("pool") if main.config["ConnectOnCreate"]: startBot(network+i) + else: + deliverRelayCommands(relay, {"status": ["Disconnect"]}, user=alias+"/"+network) + storedNetwork = True return network+i From 488d81dac800d8b892418be8f6880a277b70617d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 16 Mar 2019 17:05:16 +0000 Subject: [PATCH 114/394] Fix a race condition in disabling networks post-creation, remove redundant bindhost code and fix a minor bug in the load command --- commands/load.py | 2 +- core/helper.py | 25 +++++++------------------ modules/provision.py | 12 ++++++++---- utils/loaders/command_loader.py | 4 ++-- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/commands/load.py b/commands/load.py index 8038c5b..3f61fcc 100644 --- a/commands/load.py +++ b/commands/load.py @@ -2,7 +2,7 @@ import main class Load: def __init__(self, *args): - self.list(*args) + self.load(*args) def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: diff --git a/core/helper.py b/core/helper.py index fa6af95..3d4dd55 100644 --- a/core/helper.py +++ b/core/helper.py @@ -16,23 +16,12 @@ def startBot(name): certFN = main.certPath+main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - if "bind" in main.relay[relay].keys(): - bot = IRCBotFactory(name) - rct = reactor.connectSSL(host, - port, - bot, contextFactory, - bindAddress=main.relay[relay]["bind"]) + bot = IRCBotFactory(name) + rct = reactor.connectSSL(host, + port, + bot, contextFactory) - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return - else: - bot = IRCBotFactory(name) - rct = reactor.connectSSL(host, - port, - bot, contextFactory) - - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot + return diff --git a/modules/provision.py b/modules/provision.py index 63e00a2..e1e5bc0 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -17,6 +17,7 @@ def provisionUserData(relay, alias, nick, altnick, ident, realname, password): def provisionNetworkData(relay, alias, network, host, port, security, auth, password): commands = {} stage2commands = {} + stage3commands = {} commands["controlpanel"] = [] commands["controlpanel"].append("AddNetwork %s %s" % (alias, network)) if security == "ssl": @@ -35,7 +36,12 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass stage2commands["nickserv"] = [] stage2commands["status"].append("LoadMod NickServ") stage2commands["nickserv"].append("Set %s" % password) - deliverRelayCommands(relay, commands, stage2=[[alias+"/"+network, stage2commands]]) + if not main.config["ConnectOnCreate"]: + stage3commands["status"] = [] + stage3commands["status"].append("Disconnect") + deliverRelayCommands(relay, commands, + stage2=[[alias+"/"+network, stage2commands], + [alias+"/"+network, stage3commands]]) return def provisionRelayForAlias(relay, alias): @@ -54,7 +60,7 @@ def provisionRelayForAlias(relay, alias): return True def provisionRelayForNetwork(relay, alias, network): - if all([x in ["users", "networks"] for x in main.relay[relay].keys() if x in ["users", "networks"]]): + if set(["users", "networks"]).issubset(main.relay[relay].keys()): if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: return "PROVISIONED" else: @@ -87,8 +93,6 @@ def provisionRelayForNetwork(relay, alias, network): main.saveConf("pool") if main.config["ConnectOnCreate"]: startBot(network+i) - else: - deliverRelayCommands(relay, {"status": ["Disconnect"]}, user=alias+"/"+network) storedNetwork = True return network+i diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 775d267..e3ea253 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -6,8 +6,8 @@ import commands from main import CommandMap def loadCommands(): - for filename in listdir('commands'): - if filename.endswith('.py') and filename != "__init__.py": + for filename in listdir("commands"): + if filename.endswith(".py") and filename != "__init__.py": commandName = filename[0:-3] className = commandName.capitalize() try: From c05eb298ea6dd78216cd5b2865c3a6566e24f824 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 17 Mar 2019 01:22:21 +0000 Subject: [PATCH 115/394] Fix channel number in status command --- commands/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/stats.py b/commands/stats.py index 5b08b7e..6cdc3be 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -43,7 +43,7 @@ class Stats: for i in main.IRCPool.keys(): if "".join([x for x in i if not x in digits]) == spl[1]: - numChannels += len(main.IRCPool[spl[1]].channels) + numChannels += len(main.IRCPool[i].channels) found = True numNodes += 1 if not found: From fc304d4b25b93d6ecc1a5b9bc51a7554f29ff4d3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 18 Mar 2019 21:01:28 +0000 Subject: [PATCH 116/394] Implement the relay channel and command for generating tokens --- commands/token.py | 56 ++++++++++++++++++ conf/example/config.json | 7 ++- conf/example/tokens.json | 1 + conf/help.json | 3 +- conf/tokens.json | 7 +++ core/relay.py | 120 +++++++++++++++++++++++++++++++++++++++ main.py | 2 + threshold | 9 +++ 8 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 commands/token.py create mode 100644 conf/example/tokens.json create mode 100644 conf/tokens.json create mode 100644 core/relay.py diff --git a/commands/token.py b/commands/token.py new file mode 100644 index 0000000..cd00e38 --- /dev/null +++ b/commands/token.py @@ -0,0 +1,56 @@ +import main +from yaml import dump +from uuid import uuid4 + +class Token: + def __init__(self, *args): + self.token(*args) + + def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if spl[1] == "list": + info(dump(main.tokens)) + return + else: + incUsage("token") + return + elif length == 3: + if spl[1] == "del": + if spl[2] in main.tokens.keys(): + del main.tokens[spl[2]] + main.saveConf("tokens") + success("Successfully removed token: %s" % spl[2]) + return + else: + failure("No such token") + return + else: + incUsage("token") + return + elif length == 4: + if spl[1] == "add": + if not spl[2] in main.tokens.keys(): + if spl[3] in ["relay"]: # more to come! + main.tokens[spl[2]] = {"hello": str(uuid4()), + "usage": spl[3], + "counter": str(uuid4()), + } + main.saveConf("tokens") + success("Successfully created token %s:" % spl[2]) + info(dump(main.tokens[spl[2]])) + return + else: + incUsage("token") + return + else: + failure("Token already exists: %s" % spl[2]) + return + else: + incUsage("token") + return + else: + incUsage("token") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index 2b2959d..27e79fb 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -4,12 +4,17 @@ "Address": "127.0.0.1", "UseSSL": true }, + "Relay": { + "Enabled": true, + "Port": 13868, + "Address": "127.0.0.1", + "UseSSL": true + }, "Key": "key.pem", "Certificate": "cert.pem", "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, - "RelayPassword": "s", "Notifications": { "Highlight": true, "Connection": true, diff --git a/conf/example/tokens.json b/conf/example/tokens.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/tokens.json @@ -0,0 +1 @@ +{} diff --git a/conf/help.json b/conf/help.json index 2ecefc7..13ee310 100644 --- a/conf/help.json +++ b/conf/help.json @@ -24,5 +24,6 @@ "relay": "relay [ ]", "network": "network [
]", "provision": "provision []", - "cmd": "cmd " + "cmd": "cmd ", + "token": "token [] []" } diff --git a/conf/tokens.json b/conf/tokens.json new file mode 100644 index 0000000..d93cbde --- /dev/null +++ b/conf/tokens.json @@ -0,0 +1,7 @@ +{ + "a": { + "hello": "9814a659-9241-4389-8569-8bb23c69aea5", + "usage": "relay", + "counter": "98375c39-c421-42a1-ab48-b566587513b9" + } +} \ No newline at end of file diff --git a/core/relay.py b/core/relay.py new file mode 100644 index 0000000..a0d2881 --- /dev/null +++ b/core/relay.py @@ -0,0 +1,120 @@ +from twisted.internet.protocol import Protocol, Factory, ClientFactory +from json import dumps, loads +import main +from utils.logging.log import * + +class Relay(Protocol): + def __init__(self, addr): + self.addr = addr + self.authed = False + self.subscriptions = [] + + def send(self, data): + data += "\r\n" + data = data.encode("utf-8", "replace") + self.transport.write(data) + + def sendErr(self, data): + self.send(dumps({"type": "error", "reason": data})) + return + + def sendMsg(self, data): + self.send(dumps(data)) + + def dataReceived(self, data): + data = data.decode("utf-8", "replace") + try: + parsed = loads(data) + except: + self.sendErr("MALFORMED") + return + if not "type" in parsed.keys(): + self.sendErr("NOTYPE") + return + if parsed["type"] == "hello": + if set(["key", "hello"]).issubset(set(parsed)): + self.handleHello(parsed) + else: + self.sendErr("WRONGFIELDS") + return + return + elif parsed["type"] == "control": + if "subscribe" in parsed.keys(): + self.handleSubscribe(parsed["subscribe"]) + return + elif "unsubscribe" in parsed.keys(): + self.handleUnsubscribe(parsed["unsubscribe"]) + return + else: + self.sendErr("UNCLEAR") + return + else: + self.sendErr("UNCLEAR") + return + + def handleSubscribe(self, lst): + if not isinstance(lst, list): + self.sendErr("NOTLIST") + return + for i in lst: + if not i in ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode"]: + self.sendErr("NONEXISTANT") + return + if i in self.subscriptions: + self.sendErr("SUBSCRIBED") + return + self.subscriptions.append(i) + self.sendMsg({"type": "success"}) + return + + def handleUnubscribe(self, lst): + if not isinstance(lst, list): + self.sendErr("NOTLIST") + return + for i in lst: + if not i in ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode"]: + self.sendErr("NONEXISTANT") + return + if not i in self.subscriptions: + self.sendErr("NOTSUBSCRIBED") + return + del self.subscriptions[i] + self.sendMsg({"type": "success"}) + return + + def handleHello(self, parsed): + if parsed["key"] in main.tokens.keys(): + if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay": + self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}) + self.authed = True + else: + self.transport.loseConnection() + return + else: + self.sendErr("NOKEY") + return + + def connectionMade(self): + log("Connection from %s:%s" % (self.addr.host, self.addr.port)) + #self.send("Greetings.") + + def connectionLost(self, reason): + self.authed = False + log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) + if self.addr in main.relayConnections.keys(): + del main.relayConnections[self.addr] + else: + warn("Tried to remove a non-existant relay connection.") + +class RelayFactory(Factory): + def buildProtocol(self, addr): + entry = Relay(addr) + main.relayConnections[addr] = entry + return entry + + def send(self, addr, data): + if addr in main.relayConnections.keys(): + connection = main.relayConnections[addr] + connection.send(data) + else: + return diff --git a/main.py b/main.py index 05c8deb..8797afa 100644 --- a/main.py +++ b/main.py @@ -17,9 +17,11 @@ filemap = { "alias": ["alias.json", "alias details"], "relay": ["relay.json", "relay list"], "network": ["network.json", "network list"], + "tokens": ["tokens.json", "authentication tokens"], } connections = {} +relayConnections = {} IRCPool = {} ReactorPool = {} FactoryPool = {} diff --git a/threshold b/threshold index 5ca234a..ddba8d5 100755 --- a/threshold +++ b/threshold @@ -13,6 +13,7 @@ from utils.logging.log import * from utils.loaders.command_loader import loadCommands from core.helper import startBot from core.server import Server, ServerFactory +from core.relay import Relay, RelayFactory loadCommands() @@ -24,6 +25,14 @@ if __name__ == "__main__": else: reactor.listenTCP(main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"]) log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) + if main.config["Relay"]["Enabled"]: + relay = RelayFactory() + if main.config["Relay"]["UseSSL"] == True: + reactor.listenSSL(main.config["Relay"]["Port"], relay, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["Relay"]["Address"]) + log("Threshold relay running with SSL on %s:%s" % (main.config["Relay"]["Address"], main.config["Relay"]["Port"])) + else: + reactor.listenTCP(main.config["Relay"]["Port"], relay, interface=main.config["Relay"]["Address"]) + log("Threshold relay running on %s:%s" % (main.config["Relay"]["Address"], main.config["Relay"]["Port"])) for i in main.pool.keys(): if not "enabled" in main.pool[i]: continue From 9e1a6613a4ee5800c2d1eac749003bdd352b822f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 20 Mar 2019 20:22:46 +0000 Subject: [PATCH 117/394] Implement sending of relay messages --- modules/monitor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/monitor.py b/modules/monitor.py index 7552648..6c2ae56 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,6 +1,7 @@ import main import modules.keyword as keyword from copy import deepcopy +from json import dumps def testNetTarget(name, target): called = False @@ -49,6 +50,14 @@ def magicFunction(A, B): return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) def event(name, target, cast): + for i in main.relayConnections.keys(): + if main.relayConnections[i].authed and cast["type"] in main.relayConnections[i].subscriptions: + newCast = deepcopy(cast) + if not target == None: + newCast["target"] = target + if "exact" in newCast.keys(): + del newCast["exact"] + main.relayConnections[i].send(dumps(newCast)) monitorGroups = testNetTarget(name, target) if monitorGroups == False: return From c850984a3be0fbbdaceffebd87bcb851b8042ac8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 22 Mar 2019 23:05:00 +0000 Subject: [PATCH 118/394] Update gitignore and remove token configuration --- .gitignore | 1 + conf/tokens.json | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 conf/tokens.json diff --git a/.gitignore b/.gitignore index 76cbb56..ba45df5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,6 @@ conf/monitor.json conf/alias.json conf/relay.json conf/network.json +conf/tokens.json conf/dist.sh env/ diff --git a/conf/tokens.json b/conf/tokens.json deleted file mode 100644 index d93cbde..0000000 --- a/conf/tokens.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "a": { - "hello": "9814a659-9241-4389-8569-8bb23c69aea5", - "usage": "relay", - "counter": "98375c39-c421-42a1-ab48-b566587513b9" - } -} \ No newline at end of file From 29424383de3e8b1a7bdd9877ec449d249e470329 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 23 Mar 2019 11:39:28 +0000 Subject: [PATCH 119/394] Fix bug in relay subscriptions --- core/relay.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/relay.py b/core/relay.py index a0d2881..f121110 100644 --- a/core/relay.py +++ b/core/relay.py @@ -64,8 +64,8 @@ class Relay(Protocol): self.sendErr("SUBSCRIBED") return self.subscriptions.append(i) - self.sendMsg({"type": "success"}) - return + self.sendMsg({"type": "success"}) + return def handleUnubscribe(self, lst): if not isinstance(lst, list): @@ -78,9 +78,9 @@ class Relay(Protocol): if not i in self.subscriptions: self.sendErr("NOTSUBSCRIBED") return - del self.subscriptions[i] - self.sendMsg({"type": "success"}) - return + del self.subscriptions[i] + self.sendMsg({"type": "success"}) + return def handleHello(self, parsed): if parsed["key"] in main.tokens.keys(): From 38cabc04729419c23e14ddb5280a2e591bd9e4c3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 23 Mar 2019 18:22:46 +0000 Subject: [PATCH 120/394] Fix bug in relay unsubscribing --- core/relay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/relay.py b/core/relay.py index f121110..68aa5c5 100644 --- a/core/relay.py +++ b/core/relay.py @@ -78,7 +78,7 @@ class Relay(Protocol): if not i in self.subscriptions: self.sendErr("NOTSUBSCRIBED") return - del self.subscriptions[i] + del self.subscriptions[i] self.sendMsg({"type": "success"}) return From 4ce093bfbe4490af2cf4681cc5b179be533f5578 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 28 Jul 2019 15:07:46 +0100 Subject: [PATCH 121/394] Implement a running count of the number of events per minute --- commands/stats.py | 1 + main.py | 3 +++ modules/counters.py | 10 ++++++++++ threshold | 4 ++-- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/commands/stats.py b/commands/stats.py index 6cdc3be..178666e 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -24,6 +24,7 @@ class Stats: stats.append(" Unique: %s" % len(main.liveNets())) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) + stats.append("Events/min: %s" % main.lastMinuteSample) counterEvents = count.getEvents() if counterEvents == None: stats.append("No counters records") diff --git a/main.py b/main.py index 8797afa..fb8224d 100644 --- a/main.py +++ b/main.py @@ -30,6 +30,9 @@ MonitorPool = [] CommandMap = {} +runningSample = 0 +lastMinuteSample = 0 + def nets(): if not "pool" in globals(): return diff --git a/modules/counters.py b/modules/counters.py index 56a5e10..bb2b518 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -1,4 +1,5 @@ import main +from twisted.internet.task import LoopingCall def event(name, eventType): if not "local" in main.counters.keys(): @@ -15,6 +16,7 @@ def event(name, eventType): main.counters["local"][name][eventType] += 1 main.counters["global"][eventType] += 1 + main.runningSample += 1 def getEvents(name=None): if name == None: @@ -27,3 +29,11 @@ def getEvents(name=None): return main.counters["local"][name] else: return None + +def takeSample(): + main.lastMinuteSample = main.runningSample + main.runningSample = 0 + +def setupCounterLoop(): + lc = LoopingCall(takeSample) + lc.start(60) diff --git a/threshold b/threshold index ddba8d5..5ff3a3f 100755 --- a/threshold +++ b/threshold @@ -14,7 +14,7 @@ from utils.loaders.command_loader import loadCommands from core.helper import startBot from core.server import Server, ServerFactory from core.relay import Relay, RelayFactory - +import modules.counters loadCommands() if __name__ == "__main__": @@ -38,5 +38,5 @@ if __name__ == "__main__": continue if main.pool[i]["enabled"] == True: startBot(i) - + modules.counters.setupCounterLoop() reactor.run() From 0637f762ea0ef76156b725b0352f806d61eecad4 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 4 Aug 2019 04:22:40 +0100 Subject: [PATCH 122/394] Add network name to fields shown in relay notifications --- modules/monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/monitor.py b/modules/monitor.py index 6c2ae56..3044452 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -57,6 +57,7 @@ def event(name, target, cast): newCast["target"] = target if "exact" in newCast.keys(): del newCast["exact"] + newCast["name"] = name main.relayConnections[i].send(dumps(newCast)) monitorGroups = testNetTarget(name, target) if monitorGroups == False: From 68c6aa969dce95ceecf9e2e14f62d91cad81d972 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 5 Aug 2019 22:51:16 +0100 Subject: [PATCH 123/394] Remove keyword system, implement ZNC notifications to relay, remove exact from cast fields and fix security bug in relay --- commands/key.py | 155 -------------------------------------- commands/mon.py | 3 - conf/example/keyword.json | 6 -- core/bot.py | 85 +++++++++++---------- core/relay.py | 36 +++++++-- main.py | 1 - modules/keyword.py | 121 ----------------------------- modules/monitor.py | 38 +++++----- 8 files changed, 97 insertions(+), 348 deletions(-) delete mode 100644 commands/key.py delete mode 100644 conf/example/keyword.json delete mode 100644 modules/keyword.py diff --git a/commands/key.py b/commands/key.py deleted file mode 100644 index d6b88a7..0000000 --- a/commands/key.py +++ /dev/null @@ -1,155 +0,0 @@ -import main -import modules.keyword as keyword - -class Key: - def __init__(self, *args): - self.key(*args) - - def key(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - if data.startswith("key add"): - if not data in ["key add ", "key add"] and data[3] == " ": - keywordsToAdd = data[8:] - keywords = keywordsToAdd.split(",") - for i in keywords: - rtrn = keyword.addKeyword(i) - if rtrn == "EXISTS": - failure("Keyword already exists: %s" % i) - elif rtrn == "ISIN": - failure("Keyword already matched: %s" % i) - elif rtrn == True: - success("Keyword added: %s" % i) - return - else: - incUsage("key") - return - elif data.startswith("key del"): - if not data in ["key del ", "key del"] and data[3] == " ": - keywordsToDel = data[8:] - keywords = keywordsToDel.split(",") - for i in keywords: - rtrn = keyword.delKeyword(i) - if rtrn == "NOKEY": - failure("Keyword does not exist: %s" % i) - elif rtrn == True: - success("Keyword deleted: %s" % i) - return - else: - incUsage("key") - return - if length == 4: - if spl[1] == "except": - if not spl[2] in main.keyconf["Keywords"]: - failure("No such keyword: %s" % spl[2]) - return - if spl[2] in main.keyconf["KeywordsExcept"].keys(): - if spl[3] in main.keyconf["KeywordsExcept"][spl[2]]: - failure("Exception exists: %s" % spl[3]) - return - else: - if not spl[2] in spl[3]: - failure("Keyword %s not in exception %s. This won't work." % (spl[2], spl[3])) - return - main.keyconf["KeywordsExcept"][spl[2]] = [] - - main.keyconf["KeywordsExcept"][spl[2]].append(spl[3]) - main.saveConf("keyconf") - success("Successfully added exception %s for keyword %s" % (spl[3], spl[2])) - return - elif spl[1] == "master": - if not spl[2] in main.pool.keys(): - failure("Name does not exist: %s" % spl[2]) - return - if spl[2] in main.IRCPool.keys(): - if not spl[3] in main.IRCPool[spl[2]].channels: - info("Bot not on channel: %s" % spl[3]) - main.config["Master"] = [spl[2], spl[3]] - main.saveConf("config") - success("Master set to %s on %s" % (spl[3], spl[2])) - return - elif spl[1] == "unexcept": - if not spl[2] in main.keyconf["KeywordsExcept"].keys(): - failure("No such exception: %s" % spl[2]) - return - if not spl[3] in main.keyconf["KeywordsExcept"][spl[2]]: - failure("Exception %s has no attribute %s" % (spl[2], spl[3])) - return - main.keyconf["KeywordsExcept"][spl[2]].remove(spl[3]) - if main.keyconf["KeywordsExcept"][spl[2]] == []: - del main.keyconf["KeywordsExcept"][spl[2]] - main.saveConf("keyconf") - success("Successfully removed exception %s for keyword %s" % (spl[3], spl[2])) - return - else: - incUsage("key") - return - elif length == 3: - if spl[1] == "unexcept": - if not spl[2] in main.keyconf["KeywordsExcept"].keys(): - failure("No such exception: %s" % spl[2]) - return - del main.keyconf["KeywordsExcept"][spl[2]] - main.saveConf("keyconf") - success("Successfully removed exception list of %s" % spl[2]) - return - elif spl[1] == "monitor": - if spl[2] == "on": - if not obj.addr in main.MonitorPool: - main.MonitorPool.append(obj.addr) - success("Keyword monitoring enabled") - if len(main.masterbuf) == 0: - return - rtrn = [] - for i in range(len(main.masterbuf)): - rtrn.append(main.masterbuf.pop(0)) - main.saveConf("masterbuf") - info("\n".join(rtrn)) - return - else: - failure("Keyword monitoring is already enabled") - return - elif spl[2] == "off": - if obj.addr in main.MonitorPool: - main.MonitorPool.remove(obj.addr) - success("Keyword monitoring disabled") - return - else: - failure("Keyword monitoring is already disabled") - return - else: - incUsage("key") - return - else: - incUsage("key") - return - elif length == 2: - if spl[1] == "list": - info(",".join(main.keyconf["Keywords"])) - return - - elif spl[1] == "listexcept": - exceptMap = [] - for i in main.keyconf["KeywordsExcept"].keys(): - exceptMap.append("Key: %s" % i) - exceptMap.append("%s: %s" % (i, ",".join(main.keyconf["KeywordsExcept"][i]))) - exceptMap.append("\n") - info("\n".join(exceptMap)) - return - elif spl[1] == "master": - info(" - ".join([str(i) for i in main.config["Master"]])) - return - elif spl[1] == "monitor": - if obj.addr in main.MonitorPool: - info("Keyword monitoring is enabled on this connection") - return - else: - info("Keyword monitoring is disabled on this connection") - return - else: - incUsage("key") - return - else: - incUsage("key") - return - else: - incUsage(None) diff --git a/commands/mon.py b/commands/mon.py index 0a05d7d..c5a1ecc 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -24,7 +24,6 @@ class Mon: self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who") self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") - self.parser.add_argument("--exact", nargs="*", metavar="query", dest="exact", help="Use exact matching") self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching") self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching") self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching") @@ -208,8 +207,6 @@ class Mon: cast["status"] = obj.status if not obj.free == None: cast["free"] = obj.free - if not obj.exact == None: - cast["exact"] = obj.exact if not obj.nick == None: cast["nick"] = obj.nick if not obj.ident == None: diff --git a/conf/example/keyword.json b/conf/example/keyword.json deleted file mode 100644 index 3980412..0000000 --- a/conf/example/keyword.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "Keywords": [ - "example" - ], - "KeywordsExcept": {} -} \ No newline at end of file diff --git a/core/bot.py b/core/bot.py index 29e4223..a783dca 100644 --- a/core/bot.py +++ b/core/bot.py @@ -7,10 +7,11 @@ from twisted.internet import reactor from string import digits from random import randint -import modules.keyword as keyword -import modules.userinfo as userinfo -import modules.counters as count -import modules.monitor as monitor +from modules import userinfo +from modules import counters as count +from modules import monitor + +from core.relay import sendRelayNotification import main from utils.logging.log import * @@ -73,7 +74,7 @@ class IRCRelay(IRCClient): self.connected = True log("signed on as a relay: %s" % self.relay) if main.config["Notifications"]["Connection"]: - keyword.sendMaster("SIGNON: %s" % self.relay) + sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) for i in self.relayCommands.keys(): for x in self.relayCommands[i]: self.msg(main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) @@ -134,27 +135,27 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) + # ZNC adds messages when no users are in the channel, messing up tracking #userinfo.editUser(self.net, channel, nick, user) - count.event(self.net, "privmsg") - - keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) - monitor.event(self.net, channel, {"type": "msg", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) + count.event(self.net, "msg") + #event = None + #if self.nickname.lower() in msg.lower(): + # event = "highlight" + monitor.event(self.net, {"type": "msg", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "notice") - keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) - monitor.event(self.net, channel, {"type": "notice", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) + monitor.event(self.net, {"type": "notice", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) def action(self, user, channel, msg): nick, ident, host = self.parsen(user) #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "action") - keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) - monitor.event(self.net, channel, {"type": "action", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg}) + monitor.event(self.net, {"type": "action", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) def get(self, var): try: @@ -207,7 +208,7 @@ class IRCBot(IRCClient): n = self._who[channel][1] n.append([nick, user, host, server, status, realname]) count.event(self.net, "whoreply") - monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) + monitor.event(self.net, {"type": "who", "nick": nick, "ident": user, "host": host, "realname": realname, "target": channel, "server": server, "status": status}) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -287,7 +288,7 @@ class IRCBot(IRCClient): else: message = None if nick == self.nickname: - self.left(channel, message) + self.left(prefix, channel, message) else: self.userLeft(prefix, channel, message) @@ -295,8 +296,11 @@ class IRCBot(IRCClient): """ Called when a user has quit. """ - #nick = prefix.split('!')[0] - self.userQuit(prefix, params[0]) + nick = prefix.split('!')[0] + if nick == self.nickname: + self.botQuit(prefix, params[0]) + else: + self.userQuit(prefix, params[0]) def irc_NICK(self, prefix, params): """ @@ -336,7 +340,7 @@ class IRCBot(IRCClient): self.connected = True log("signed on: %s" % self.name) if main.config["Notifications"]["Connection"]: - keyword.sendMaster("SIGNON: %s" % self.name) + sendRelayNotification(self.name, {"type": "conn", "status": "connected"}) count.event(self.net, "signedon") def joined(self, channel): @@ -365,41 +369,48 @@ class IRCBot(IRCClient): del self._getWho[channel] userinfo.delChannel(self.net, channel) - def left(self, channel, message): - keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) + def left(self, user, channel, message): + nick, ident, host = self.parsen(user) count.event(self.net, "selfpart") - monitor.event(self.net, channel, {"type": "part", "message": message}) + monitor.event(self.net, {"type": "part", "target": channel, "message": message}) + monitor.event(self.net, {"type": "part", "self": True, "target": channel, "message": message}) self.botLeft(channel) def kickedFrom(self, channel, kicker, message): nick, ident, host = self.parsen(kicker) if channel in self.channels: self.channels.remove(channel) - keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.net, "selfkick") - monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) + monitor.event(self.net, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) + monitor.event(self.net, {"type": "kick", "self": True, "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) self.botLeft(channel) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) userinfo.addUser(self.net, channel, nick, user) count.event(self.net, "join") - monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) + monitor.event(self.net, {"type": "join", "nick": nick, "ident": ident, "host": host, "target": channel}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) userinfo.delUser(self.net, channel, nick, user) - keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) count.event(self.net, "part") - monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) + monitor.event(self.net, {"type": "part", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.delUserByNetwork(self.net, nick, user) count.event(self.net, "quit") - keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) - monitor.event(self.net, None, {"type": "quit", "exact": user, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + monitor.event(self.net, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + + def botQuit(self, user, quitMessage): + nick, ident, host = self.parsen(user) + userinfo.delUserByNetwork(self.net, nick, user) + count.event(self.net, "selfquit") + + monitor.event(self.net, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + monitor.event(self.net, {"type": "quit", "self": True, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) @@ -407,22 +418,20 @@ class IRCBot(IRCClient): userinfo.delUserByNick(self.net, channel, kickee) count.event(self.net, "kick") - keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) - monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message, "user": kickee}) + monitor.event(self.net, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message, "user": kickee}) def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) count.event(self.net, "nick") - monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) + monitor.event(self.net, {"type": "nick", "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "topic") - keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) - monitor.event(self.net, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic}) + monitor.event(self.net, {"type": "topic", "nick": nick, "ident": ident, "host": host, "target": channel, "message": newTopic}) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) @@ -431,7 +440,7 @@ class IRCBot(IRCClient): argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - monitor.event(self.net, channel, {"type": "mode", "exact": user, "nick": nick, "ident": ident, "host": host, "modes": m, "modeargs": a}) + monitor.event(self.net, {"type": "mode", "nick": nick, "ident": ident, "host": host, "target": channel, "modes": m, "modeargs": a}) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): @@ -449,7 +458,6 @@ class IRCBotFactory(ReconnectingClientFactory): self.relay, self.relayCommands, self.user, self.stage2 = relay, relayCommands, user, stage2 def buildProtocol(self, addr): - if self.relay == None: entry = IRCBot(self.name) main.IRCPool[self.name] = entry @@ -470,7 +478,7 @@ class IRCBotFactory(ReconnectingClientFactory): if not self.relay: sendAll("%s: connection lost: %s" % (self.name, error)) if main.config["Notifications"]["Connection"]: - keyword.sendMaster("CONNLOST %s: %s" % (self.name, error)) + sendRelayNotification(self.name, {"type": "conn", "status": "lost", "reason": error}) if not self.relay: self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -481,9 +489,10 @@ class IRCBotFactory(ReconnectingClientFactory): self.client.channels = [] error = reason.getErrorMessage() log("%s: connection failed: %s" % (self.name, error)) - sendAll("%s: connection failed: %s" % (self.name, error)) + if not self.relay: + sendAll("%s: connection failed: %s" % (self.name, error)) if main.config["Notifications"]["Connection"]: - keyword.sendMaster("CONNFAIL %s: %s" % (self.name, error)) + sendRelayNotification(self.name, {"type": "conn", "status": "failed", "reason": error}) if not self.relay: self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/relay.py b/core/relay.py index 68aa5c5..ff3eaaa 100644 --- a/core/relay.py +++ b/core/relay.py @@ -1,8 +1,15 @@ from twisted.internet.protocol import Protocol, Factory, ClientFactory from json import dumps, loads +from copy import deepcopy +from datetime import datetime + import main from utils.logging.log import * +validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err"] + +selfTypes = ["query", "self", "highlight"] + class Relay(Protocol): def __init__(self, addr): self.addr = addr @@ -40,11 +47,19 @@ class Relay(Protocol): return elif parsed["type"] == "control": if "subscribe" in parsed.keys(): - self.handleSubscribe(parsed["subscribe"]) - return + if self.authed: + self.handleSubscribe(parsed["subscribe"]) + return + else: + self.sendErr("DENIED") + return elif "unsubscribe" in parsed.keys(): - self.handleUnsubscribe(parsed["unsubscribe"]) - return + if self.authed: + self.handleUnsubscribe(parsed["unsubscribe"]) + return + else: + self.sendErr("DENIED") + return else: self.sendErr("UNCLEAR") return @@ -57,7 +72,7 @@ class Relay(Protocol): self.sendErr("NOTLIST") return for i in lst: - if not i in ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode"]: + if not i in validTypes: self.sendErr("NONEXISTANT") return if i in self.subscriptions: @@ -72,7 +87,7 @@ class Relay(Protocol): self.sendErr("NOTLIST") return for i in lst: - if not i in ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode"]: + if not i in validTypes: self.sendErr("NONEXISTANT") return if not i in self.subscriptions: @@ -118,3 +133,12 @@ class RelayFactory(Factory): connection.send(data) else: return + +def sendRelayNotification(name, cast): + for i in main.relayConnections.keys(): + if main.relayConnections[i].authed: + if cast["type"] in main.relayConnections[i].subscriptions or set(main.relayConnections[i].subscriptions).issubset(cast): + newCast = deepcopy(cast) + newCast["name"] = name + newCast["time"] = datetime.now() + main.relayConnections[i].send(dumps(newCast)) diff --git a/main.py b/main.py index fb8224d..34add10 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,6 @@ certPath = "cert/" filemap = { "config": ["config.json", "configuration"], - "keyconf": ["keyword.json", "keyword lists"], "pool": ["pool.json", "network, alias and relay mappings"], "help": ["help.json", "command help"], "counters": ["counters.json", "counters file"], diff --git a/modules/keyword.py b/modules/keyword.py deleted file mode 100644 index 9b6b926..0000000 --- a/modules/keyword.py +++ /dev/null @@ -1,121 +0,0 @@ -import main -from utils.logging.log import * -import modules.counters as count - -def sendMaster(data): - if not len(main.MonitorPool) == 0: - hasMonitors = True - else: - hasMonitors = False - - if main.config["Master"] == [None, None]: - if hasMonitors: - for i in main.MonitorPool: - main.connections[i].send(data) - return - else: - main.masterbuf.append(data) - main.saveConf("masterbuf") - return - - if main.config["Master"][0] in main.IRCPool.keys(): - if main.config["Master"][1] in main.IRCPool[main.config["Master"][0]].channels: - main.IRCPool[main.config["Master"][0]].msg(main.config["Master"][1], data) - else: - if not hasMonitors: - main.masterbuf.append(data) - main.saveConf("masterbuf") - else: - if not hasMonitors: - main.masterbuf.append(data) - main.saveConf("masterbuf") - - for i in main.MonitorPool: - main.connections[i].send(data) - -def isKeyword(msg): - if msg == None: - return - message = msg.lower() - messageDuplicate = message - toUndo = False - uniqueNum = 0 - totalNum = 0 - for i in main.keyconf["Keywords"]: - if i in message: - if i in main.keyconf["KeywordsExcept"].keys(): - for x in main.keyconf["KeywordsExcept"][i]: - if x in message: - toUndo = True - messageDuplicate = messageDuplicate.replace(x, "\0\r\n\n\0") - for y in main.keyconf["Keywords"]: - if i in messageDuplicate: - totalNum += messageDuplicate.count(i) - message = messageDuplicate.replace(i, "{"+i+"}") - message = message.replace("\0\r\n\n\0", x) - uniqueNum += 1 - - if toUndo == False: - totalNum += message.count(i) - message = message.replace(i, "{"+i+"}") - uniqueNum += 1 - - toUndo = False - - if totalNum == 0: - return False - else: - return [message, uniqueNum, totalNum] - -def actKeyword(user, channel, message, nickname, actType, name): - if message == None: - return - toSend = isKeyword(message) - if name == main.config["Master"][0] and channel == main.config["Master"][1]: - pass - else: - msgLower = message.lower() - nickLower = nickname.lower() - if nickLower in msgLower: # Someone has said the bot's nickname - msgLower = msgLower.replace(nickLower, "{"+nickLower+"}") - count.event(name, "nickhighlight") - if main.config["Notifications"]["Highlight"]: - sendMaster("NICK %s %s (T:%s): (%s/%s) %s" % (actType, name, msgLower.count(nickLower), user, channel, msgLower)) - if not channel == None: - chanLower = channel.lower() - if nickLower == chanLower: # I received a message directed only at me - ZNCAlreadySent = False - if main.config["Notifications"]["Query"]: - if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in": - if main.config["Compat"]["ZNC"]: - sendMaster("ZNC %s %s: %s" % (actType, name, msgLower)) - ZNCAlreadySent = True - else: - sendMaster("QUERY %s %s: (%s) %s" % (actType, name, user, msgLower)) - else: - sendMaster("QUERY %s %s: (%s) %s" % (actType, name, user, msgLower)) - if not ZNCAlreadySent == True: - if main.config["Compat"]["ZNC"]: - if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in": - sendMaster("ZNC %s %s: %s" % (actType, name, msgLower)) - if toSend: - sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0])) - count.event(name, "keymatch") - -def addKeyword(keyword): - if keyword in main.keyconf["Keywords"]: - return "EXISTS" - else: - for i in main.keyconf["Keywords"]: - if i in keyword or keyword in i: - return "ISIN" - main.keyconf["Keywords"].append(keyword) - main.saveConf("keyconf") - return True - -def delKeyword(keyword): - if not keyword in main.keyconf["Keywords"]: - return "NOKEY" - main.keyconf["Keywords"].remove(keyword) - main.saveConf("keyconf") - return True diff --git a/modules/monitor.py b/modules/monitor.py index 3044452..9ed1ca2 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,8 +1,9 @@ -import main -import modules.keyword as keyword from copy import deepcopy from json import dumps +import main +from core.relay import sendRelayNotification + def testNetTarget(name, target): called = False for i in main.monitor.keys(): @@ -49,32 +50,33 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(name, target, cast): - for i in main.relayConnections.keys(): - if main.relayConnections[i].authed and cast["type"] in main.relayConnections[i].subscriptions: - newCast = deepcopy(cast) - if not target == None: - newCast["target"] = target - if "exact" in newCast.keys(): - del newCast["exact"] - newCast["name"] = name - main.relayConnections[i].send(dumps(newCast)) +def event(name, cast, event=None): + if "target" in cast.keys(): + target = cast["target"] + else: + target = None + if set(["nick", "ident", "host", "message"]).issubset(set(cast)): + if main.config["Compat"]["ZNC"] and "message" in cast.keys(): + if cast["nick"][0] == main.config["Tweaks"]["ZNC"]["Prefix"] and cast["ident"] == "znc" and cast["host"] == "znc.in": + sendRelayNotification(name, {"type": "znc", "message": cast["message"]}) + return + + sendRelayNotification(name, cast) monitorGroups = testNetTarget(name, target) if monitorGroups == False: return for monitorGroup in monitorGroups: matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup])) if matcher == True: + cast["monitor"] = True if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): if isinstance(main.monitor[monitorGroup]["send"][i], bool): - keyword.sendMaster("ERRDELIV MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) + sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "errdeliv"}) continue if not i in main.pool.keys(): - keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i)) + sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noname"}) if not i in main.IRCPool.keys(): - keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i)) + sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noinstance"}) for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "MONITOR [%s] (%s/%s) %s" % (monitorGroup, name, target, cast)) - else: - keyword.sendMaster("MONITOR [%s] (%s/%s) %s " % (monitorGroup, name, target, cast)) + main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, name, cast)) From 0ee8ae0ead033bd26d133ad60009e1957bf0defb Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 5 Aug 2019 22:52:06 +0100 Subject: [PATCH 124/394] Remove keyword store from gitignore file --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba45df5..e23d8a6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ __pycache__/ conf/config.json conf/pool.json conf/wholist.json -conf/keyword.json conf/counters.json conf/masterbuf.json conf/monitor.json From c0b45c1db6b9df8b5921e71e03e2828275e5c38b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 5 Aug 2019 23:10:06 +0100 Subject: [PATCH 125/394] Fix adding of time to relay messages --- core/relay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/relay.py b/core/relay.py index ff3eaaa..76dbc87 100644 --- a/core/relay.py +++ b/core/relay.py @@ -140,5 +140,5 @@ def sendRelayNotification(name, cast): if cast["type"] in main.relayConnections[i].subscriptions or set(main.relayConnections[i].subscriptions).issubset(cast): newCast = deepcopy(cast) newCast["name"] = name - newCast["time"] = datetime.now() + newCast["time"] = str(datetime.now()) main.relayConnections[i].send(dumps(newCast)) From 56840e006039ee0915bfb714e40d91fbdcfb435d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 6 Aug 2019 12:49:29 +0100 Subject: [PATCH 126/394] Add the network number in ZNC relay notifications --- core/bot.py | 34 +++++++++++++++++----------------- modules/monitor.py | 4 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/bot.py b/core/bot.py index a783dca..bb91bc7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -141,21 +141,21 @@ class IRCBot(IRCClient): #event = None #if self.nickname.lower() in msg.lower(): # event = "highlight" - monitor.event(self.net, {"type": "msg", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) + monitor.event(self.net, self.name, {"type": "msg", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "notice") - monitor.event(self.net, {"type": "notice", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) + monitor.event(self.net, self.name, {"type": "notice", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) def action(self, user, channel, msg): nick, ident, host = self.parsen(user) #userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "action") - monitor.event(self.net, {"type": "action", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) + monitor.event(self.net, self.name, {"type": "action", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) def get(self, var): try: @@ -208,7 +208,7 @@ class IRCBot(IRCClient): n = self._who[channel][1] n.append([nick, user, host, server, status, realname]) count.event(self.net, "whoreply") - monitor.event(self.net, {"type": "who", "nick": nick, "ident": user, "host": host, "realname": realname, "target": channel, "server": server, "status": status}) + monitor.event(self.net, self.name, {"type": "who", "nick": nick, "ident": user, "host": host, "realname": realname, "target": channel, "server": server, "status": status}) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -372,8 +372,8 @@ class IRCBot(IRCClient): def left(self, user, channel, message): nick, ident, host = self.parsen(user) count.event(self.net, "selfpart") - monitor.event(self.net, {"type": "part", "target": channel, "message": message}) - monitor.event(self.net, {"type": "part", "self": True, "target": channel, "message": message}) + monitor.event(self.net, self.name, {"type": "part", "target": channel, "message": message}) + monitor.event(self.net, self.name, {"type": "part", "self": True, "target": channel, "message": message}) self.botLeft(channel) def kickedFrom(self, channel, kicker, message): @@ -381,36 +381,36 @@ class IRCBot(IRCClient): if channel in self.channels: self.channels.remove(channel) count.event(self.net, "selfkick") - monitor.event(self.net, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) - monitor.event(self.net, {"type": "kick", "self": True, "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) + monitor.event(self.net, self.name, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) + monitor.event(self.net, self.name, {"type": "kick", "self": True, "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) self.botLeft(channel) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) userinfo.addUser(self.net, channel, nick, user) count.event(self.net, "join") - monitor.event(self.net, {"type": "join", "nick": nick, "ident": ident, "host": host, "target": channel}) + monitor.event(self.net, self.name, {"type": "join", "nick": nick, "ident": ident, "host": host, "target": channel}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) userinfo.delUser(self.net, channel, nick, user) count.event(self.net, "part") - monitor.event(self.net, {"type": "part", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) + monitor.event(self.net, self.name, {"type": "part", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.delUserByNetwork(self.net, nick, user) count.event(self.net, "quit") - monitor.event(self.net, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + monitor.event(self.net, self.name, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) def botQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.delUserByNetwork(self.net, nick, user) count.event(self.net, "selfquit") - monitor.event(self.net, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) - monitor.event(self.net, {"type": "quit", "self": True, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + monitor.event(self.net, self.name, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + monitor.event(self.net, self.name, {"type": "quit", "self": True, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) @@ -418,20 +418,20 @@ class IRCBot(IRCClient): userinfo.delUserByNick(self.net, channel, kickee) count.event(self.net, "kick") - monitor.event(self.net, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message, "user": kickee}) + monitor.event(self.net, self.name, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message, "user": kickee}) def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) count.event(self.net, "nick") - monitor.event(self.net, {"type": "nick", "nick": nick, "ident": ident, "host": host, "user": newname}) + monitor.event(self.net, self.name, {"type": "nick", "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) userinfo.editUser(self.net, channel, nick, user) count.event(self.net, "topic") - monitor.event(self.net, {"type": "topic", "nick": nick, "ident": ident, "host": host, "target": channel, "message": newTopic}) + monitor.event(self.net, self.name, {"type": "topic", "nick": nick, "ident": ident, "host": host, "target": channel, "message": newTopic}) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) @@ -440,7 +440,7 @@ class IRCBot(IRCClient): argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - monitor.event(self.net, {"type": "mode", "nick": nick, "ident": ident, "host": host, "target": channel, "modes": m, "modeargs": a}) + monitor.event(self.net, self.name, {"type": "mode", "nick": nick, "ident": ident, "host": host, "target": channel, "modes": m, "modeargs": a}) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): diff --git a/modules/monitor.py b/modules/monitor.py index 9ed1ca2..45f5118 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -50,7 +50,7 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(name, cast, event=None): +def event(name, numberedName, cast, event=None): if "target" in cast.keys(): target = cast["target"] else: @@ -58,7 +58,7 @@ def event(name, cast, event=None): if set(["nick", "ident", "host", "message"]).issubset(set(cast)): if main.config["Compat"]["ZNC"] and "message" in cast.keys(): if cast["nick"][0] == main.config["Tweaks"]["ZNC"]["Prefix"] and cast["ident"] == "znc" and cast["host"] == "znc.in": - sendRelayNotification(name, {"type": "znc", "message": cast["message"]}) + sendRelayNotification(numberedName, {"type": "znc", "message": cast["message"]}) return sendRelayNotification(name, cast) From 88077782f3412bcc4f77470362dd3d79208a0e01 Mon Sep 17 00:00:00 2001 From: Al Beano Date: Fri, 9 Aug 2019 23:06:34 +0100 Subject: [PATCH 127/394] Rename classes representing commands to Command --- commands/alias.py | 2 +- commands/chans.py | 2 +- commands/cmd.py | 2 +- commands/del.py | 2 +- commands/disable.py | 2 +- commands/dist.py | 2 +- commands/enable.py | 2 +- commands/get.py | 2 +- commands/help.py | 2 +- commands/join.py | 2 +- commands/list.py | 2 +- commands/load.py | 2 +- commands/loadmod.py | 2 +- commands/logout.py | 2 +- commands/mod.py | 2 +- commands/mon.py | 2 +- commands/msg.py | 2 +- commands/network.py | 2 +- commands/part.py | 2 +- commands/pass.py | 2 +- commands/provision.py | 2 +- commands/relay.py | 2 +- commands/save.py | 2 +- commands/stats.py | 2 +- commands/token.py | 2 +- commands/users.py | 2 +- commands/who.py | 2 +- utils/loaders/command_loader.py | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/commands/alias.py b/commands/alias.py index 0b127ed..cd9ef7a 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -1,7 +1,7 @@ import main from yaml import dump -class Alias: +class AliasCommand: def __init__(self, *args): self.alias(*args) diff --git a/commands/chans.py b/commands/chans.py index 3f00f43..f92482b 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -1,7 +1,7 @@ import main import modules.userinfo as userinfo -class Chans: +class ChansCommand: def __init__(self, *args): self.chans(*args) diff --git a/commands/cmd.py b/commands/cmd.py index 5908bda..6c75ec5 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -1,7 +1,7 @@ import main from core.bot import deliverRelayCommands -class Cmd: +class CmdCommand: def __init__(self, *args): self.cmd(*args) diff --git a/commands/del.py b/commands/del.py index 9d4868a..ef6f9ad 100644 --- a/commands/del.py +++ b/commands/del.py @@ -1,6 +1,6 @@ import main -class Del: +class DelCommand: def __init__(self, *args): self.delete(*args) diff --git a/commands/disable.py b/commands/disable.py index 97c5742..d9182fd 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -1,7 +1,7 @@ import main from core.bot import deliverRelayCommands -class Disable: +class DisableCommand: def __init__(self, *args): self.disable(*args) diff --git a/commands/dist.py b/commands/dist.py index 3c94f5d..d5f6f53 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -1,7 +1,7 @@ import main from subprocess import run, PIPE -class Dist: +class DistCommand: def __init__(self, *args): self.dist(*args) diff --git a/commands/enable.py b/commands/enable.py index d6a29c5..81f1139 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -2,7 +2,7 @@ import main from core.helper import startBot from core.bot import deliverRelayCommands -class Enable: +class EnableCommand: def __init__(self, *args): self.enable(*args) diff --git a/commands/get.py b/commands/get.py index 51ccd7f..b56f5dc 100644 --- a/commands/get.py +++ b/commands/get.py @@ -1,6 +1,6 @@ import main -class Get: +class GetCommand: def __init__(self, *args): self.get(*args) diff --git a/commands/help.py b/commands/help.py index 6e3e229..bb12aff 100644 --- a/commands/help.py +++ b/commands/help.py @@ -1,6 +1,6 @@ import main -class Help: +class HelpCommand: def __init__(self, *args): self.help(*args) diff --git a/commands/join.py b/commands/join.py index 0b1c223..edf6577 100644 --- a/commands/join.py +++ b/commands/join.py @@ -1,6 +1,6 @@ import main -class Join: +class JoinCommand: def __init__(self, *args): self.join(*args) diff --git a/commands/list.py b/commands/list.py index 3edef6d..55ba053 100644 --- a/commands/list.py +++ b/commands/list.py @@ -1,7 +1,7 @@ import main from yaml import dump -class List: +class ListCommand: def __init__(self, *args): self.list(*args) diff --git a/commands/load.py b/commands/load.py index 3f61fcc..aa6b37d 100644 --- a/commands/load.py +++ b/commands/load.py @@ -1,6 +1,6 @@ import main -class Load: +class LoadCommand: def __init__(self, *args): self.load(*args) diff --git a/commands/loadmod.py b/commands/loadmod.py index c5c463f..fb542b9 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -1,7 +1,7 @@ import main from utils.loaders.single_loader import loadSingle -class Loadmod: +class LoadmodCommand: def __init__(self, *args): self.loadmod(*args) diff --git a/commands/logout.py b/commands/logout.py index 960ac1e..fca76d6 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -1,6 +1,6 @@ import main -class Logout: +class LogoutCommand: def __init__(self, *args): self.logout(*args) diff --git a/commands/mod.py b/commands/mod.py index d11b33b..7f5a822 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,7 +1,7 @@ import main from yaml import dump -class Mod: +class ModCommand: def __init__(self, *args): self.mod(*args) diff --git a/commands/mon.py b/commands/mon.py index c5a1ecc..65e83d1 100644 --- a/commands/mon.py +++ b/commands/mon.py @@ -4,7 +4,7 @@ import sys from io import StringIO from yaml import dump -class Mon: +class MonCommand: def __init__(self, *args): self.mon(*args) diff --git a/commands/msg.py b/commands/msg.py index 3638c28..001cbe8 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -1,6 +1,6 @@ import main -class Msg: +class MsgCommand: def __init__(self, *args): self.msg(*args) diff --git a/commands/network.py b/commands/network.py index 024042e..9f0d67a 100644 --- a/commands/network.py +++ b/commands/network.py @@ -1,7 +1,7 @@ import main from yaml import dump -class Network: +class NetworkCommand: def __init__(self, *args): self.network(*args) diff --git a/commands/part.py b/commands/part.py index 67bf539..a63a20d 100644 --- a/commands/part.py +++ b/commands/part.py @@ -1,6 +1,6 @@ import main -class Part: +class PartCommand: def __init__(self, *args): self.part(*args) diff --git a/commands/pass.py b/commands/pass.py index 20f4d9a..fe3f240 100644 --- a/commands/pass.py +++ b/commands/pass.py @@ -1,6 +1,6 @@ import main -class Pass: +class PassCommand: def __init__(self, *args): self.password(*args) diff --git a/commands/provision.py b/commands/provision.py index 3453171..edc0d07 100644 --- a/commands/provision.py +++ b/commands/provision.py @@ -1,7 +1,7 @@ import main from modules import provision -class Provision: +class ProvisionCommand: def __init__(self, *args): self.provision(*args) diff --git a/commands/relay.py b/commands/relay.py index 894c80b..92ba014 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -1,7 +1,7 @@ import main from yaml import dump -class Relay: +class RelayCommand: def __init__(self, *args): self.relay(*args) diff --git a/commands/save.py b/commands/save.py index edd4d1c..d61d1c1 100644 --- a/commands/save.py +++ b/commands/save.py @@ -1,6 +1,6 @@ import main -class Save: +class SaveCommand: def __init__(self, *args): self.save(*args) diff --git a/commands/stats.py b/commands/stats.py index 178666e..3281a19 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -3,7 +3,7 @@ import modules.counters as count import modules.userinfo as userinfo from string import digits -class Stats: +class StatsCommand: def __init__(self, *args): self.stats(*args) diff --git a/commands/token.py b/commands/token.py index cd00e38..7e20e5b 100644 --- a/commands/token.py +++ b/commands/token.py @@ -2,7 +2,7 @@ import main from yaml import dump from uuid import uuid4 -class Token: +class TokenCommand: def __init__(self, *args): self.token(*args) diff --git a/commands/users.py b/commands/users.py index 9836a41..30da28e 100644 --- a/commands/users.py +++ b/commands/users.py @@ -1,7 +1,7 @@ import main import modules.userinfo as userinfo -class Users: +class UsersCommand: def __init__(self, *args): self.users(*args) diff --git a/commands/who.py b/commands/who.py index 7ae4909..f19e462 100644 --- a/commands/who.py +++ b/commands/who.py @@ -1,7 +1,7 @@ import main import modules.userinfo as userinfo -class Who: +class WhoCommand: def __init__(self, *args): self.who(*args) diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index e3ea253..b22cbe1 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -9,7 +9,7 @@ def loadCommands(): for filename in listdir("commands"): if filename.endswith(".py") and filename != "__init__.py": commandName = filename[0:-3] - className = commandName.capitalize() + className = commandName.capitalize()+"Command" try: module = __import__('commands.%s' % commandName) if not commandName in CommandMap: From 15bc195648f9ea42a89c325e79142c773e5c3678 Mon Sep 17 00:00:00 2001 From: Al Beano Date: Sat, 10 Aug 2019 11:44:31 +0100 Subject: [PATCH 128/394] Add automatic alias generation function --- commands/alias.py | 9 +++++++ conf/aliasdata.json | 1 + conf/help.json | 2 +- main.py | 1 + modules/alias.py | 65 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 conf/aliasdata.json create mode 100644 modules/alias.py diff --git a/commands/alias.py b/commands/alias.py index 0b127ed..f5457ef 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -1,5 +1,6 @@ import main from yaml import dump +import modules.alias as alias class Alias: def __init__(self, *args): @@ -35,6 +36,14 @@ class Alias: else: failure("No such alias: %s" % spl[2]) return + elif spl[1] == "add" and spl[2] == "auto": + newalias = alias.generate_alias() + while newalias["nick"] in main.alias.keys(): + newalias = alias.generate_alias() + main.alias[newalias["nick"]] = newalias + success("Successfully created alias: %s" % newalias["nick"]) + main.saveConf("alias") + return else: incUsage("alias") return diff --git a/conf/aliasdata.json b/conf/aliasdata.json new file mode 100644 index 0000000..9fbc997 --- /dev/null +++ b/conf/aliasdata.json @@ -0,0 +1 @@ +{"realnames":["Roslyn Capps","Vasiliki Mor","Tiffani Harlin","Jarrod Bobo","Ellamae Lenser","Jamal Coil","Kristan Baugher","Barton Bucklin","Francesco So","Shemeka Pickerel","Enola Running","Takisha Voegele","Yuko Stern","Lorraine Li","Deanna Marano","Pamula Wallis","Glory Hammock","Shenita Hajduk","Adrienne Pflum","Sherill Canton","Maxima Marsee","Oliva Kutcher","Digna Gugino","Liz Astudillo","Rico Mayorga","Dominga Fewell","Hannelore Serra","Carlene Rosario","Clifton Pasha","Adell Leffingwell","Jenise Koziel","Sharla Marlatt","Harvey Daoust","Keri Badger","Tyson Lev","Cathrine Otero","Kristofer Truman","Lauretta Bleau","Saundra Kimberly","Bobette Winford","Lawerence Tepper","Cordell Lagrone","Saran Koh","Christal Simmers","Leda Mendosa","Armand Simmon","Kara Macaulay","Pamala Gabrielson","Pearline Pierre","Erline Aguilar","Hilario Mcclusky","Donnette Stapleton","Jazmin Oberholtzer","Nicolle Tufts","Pam Kogut","Jaunita Wilkenson","Mozell Battaglia","Elke Viands","Erinn Siple","Jacklyn Rhodus","Harmony Obey","Malissa Chagolla","Betsey Broce","Nadene Cundiff","Natalia Kealey","Edmond Bowie","Sadie Koopman","Robbin Blunk","Afton Fling","Scot Wei","Creola Lauber","Hazel Stong","Fredricka Auxier","Cherilyn Terrazas","Kenna Wedding","Anthony Jacobo","Claudine Freels","Michelina Linz","Jonathon Lipari","Brendon Raposo","Inga Fullam","Valentin Freitas","Marhta Kime","Mikaela Hilty","Vania Viars","Sidney Schuyler","Yang Carlin","Candance Sandin","Felipe Engh","Tamara Everman","Zita Mershon","Eugena Chagoya","Willodean Yoo","Sydney Hulbert","Tatiana Tristan","Clarissa Smedley","Arline Struck","Kourtney Schulenberg","Van Gomez","Ellsworth Lozada","Ina Spires","Teisha Benedetti","Daniela Lauzon","Devora Dugan","Clifton Heinze","Charmaine Maxton","Delaine Pullum","Cindy Herrman","Catalina Sabine","Tomoko Morra","Edgar Flinchbaugh","Kelly Pleasants","David Munsell","Jeanette Obert","Shaunte Breault","Abby Maillet","Chad Tafolla","Lakiesha Wilber","Jim Cardillo","Robbie Milton","Mac Tibbets","Nancy Kirst","Kiley Masone","Cathy Ohearn","Minh Ahlstrom","Nannie Schillaci","Lane Leishman","King Beech","Agnes Reising","Katelynn Alton","Trudie Shelley","Johnsie Duda","Rosette Ruffo","Loreta Mor","Winston Johnson","Marilu Fabian","Romeo Reinhard","Johnnie Minder","Tasia Burgett","Fumiko Fuentez","Lesha Vowels","Savanna Toothman","Darwin Smithson","Rona Blazek","Gigi Wollard","Denny Gee","Norman Fellows","Noelle Palmieri","Lesli Ronan","Celestine Lor","Evan Briere","Dolly Kline","Cody Artman","Lakia Smothers","Luke Wagar","Larraine Hillin","Teresia Sedor","Luanne Broderick","Johnny Cable","Shirley Shorey","Colin Mancilla","Yevette Cisco","Barbra Schwenk","Dawn Heavrin","Lieselotte Stall","Yasmin Combs","Alla Ambler","Sebastian Shevlin","Stephan Kennington","Lorena Dardar","Gayla Zeledon","Etha Winslow","Marilynn Lazzaro","Julian Shiver","Emmie Criswell","Mae Demeritt","Buddy Nathaniel","Cruz Loring","Marita Callihan","Dorathy Mckennon","Xochitl Chappelle","Rebecka Pfarr","Branden Rafferty","Ozell Boutte","Ruben Madruga","Dwayne Delzell","Adolph Reinecke","Valery Kugel","Carolina Mcglown","Kelly Culver","Esmeralda Demo","Maricela Tandy","Micheline Oathout","Alec Newhouse","Leeanna Meggs","Kamilah Lundgren","Evette Nardi","Hattie Gallien","Isa Hagens","Jesus Matousek","Newton Primrose","Lazaro Morley","Racheal Gerrity","Cecile Brumley","Susanna Clausing","Barbar Taranto","Honey Hentges","Daisy Rapp","Irmgard Rotunno","Coletta Bothe","Genevive Phillips","Reynaldo Cameron","Francisca Digby","Sherrie Joachim","Camelia Williamson","Chandra Rojas","Heidi Agtarap","Marlena Martir","Enid Issac","Chelsey Paulino","Latasha Hostetter","Warren Humphries","Maegan Eatmon","Gertrud Heyward","Emmie Metts","Rossie Neugebauer","Barb Haberle","Kortney Leamon","Alene Peele","Jaqueline Elizondo","Giovanni Moose","Cathryn Ake","Tammi Scott","Elsa Capote","Craig Hoadley","Ferdinand Sweatman","Janeen Iwamoto","Harvey Gullick","Deena Eickhoff","Wava Zook","Karla Seats","Maureen Arocha","Myra Kirkpatrick","Gertrude Causby","Marya Fonville","Amparo Loveland","Janeth Hylton","Denis Dickinson","Charise Grunden","Colby Meshell","Clemente Griese","Genevieve Dougal","Jenell Tieu","Loree Monte","Sally Batton","Salvatore Teague","Lyla Cassinelli","Leighann Lamberti","Myrl Delaughter","Tracie Alverez","Raymonde Lundstrom","Everette Woodham","Rickey Flanary","See Crete","Young Monsour","Maya Osterman","Darrell Feiler","Christia Oatman","Mimi Forbes","Stanton Ackerley","Ines Farquharson","Geraldo Santee","Estella Charron","Kara Tift","Rodger Giambrone","Jospeh Remick","Corrine Gaumer","Mao Meggs","Luisa Jaimes","Kena Linsey","Cherilyn Vrooman","Tamra Dexter","Rossie Sweatt","Reuben Gabaldon","Taunya Pulice","Enriqueta Sieber","Sarah Ochoa","Vannessa Knickerbocker","Renea Canaday","Bailey Witmer","Thanh Nobile","Cristobal Ma","Colby Alanis","Abe Housand","Russ Whitmore","Sharolyn Cabrales","Fletcher Reynold","Denna Siefert","Mary Gaynor","Latoyia Romero","Barbra Chalk","Ted Frisk","Lore Guardiola","Summer Sine","Hisako Petrella","Jaime Despain","Coreen Greenstein","Joelle Massi","Rosia Hintz","Loralee Moone","Ciera Wheelwright","Tosha Mcduffee","Christina Mcglynn","Abdul Colgan","Charlott Packer","Trinh Haecker","Armandina Kimbro","Houston Justiniano","Perla Erazo","Elinore Mera","Dona Loveland","Deanne Holte","Grazyna Ravenell","Jenell Furry","Alexander Jeanpierre","Nobuko Ange","Sanjuana Mcdougle","Minnie Shankle","Earlene Wescott","Lucila Mcnally","Chanell Windham","Kathyrn Puff","Felicitas Lisenby","Hanna Dahms","Markita Reisner","Soledad Cron","Corey Haslem","Augustina Chien","Silvia Capuano","Kate Stice","Leisa Whitfield","Kimberlee Tartaglia","Janelle Breese","Suzi Ellard","Buffy Deschaine","Glendora Reuss","Kiera Sisk","Alina Neace","Suk See","Graham Ballard"],"stubs":["aaren","aarika","abagael","abagail","abbe","abbey","abbi","abbie","abby","abbye","abigael","abigail","abigale","abra","ada","adah","adaline","adan","adara","adda","addi","addia","addie","addy","adel","adela","adelaida","adelaide","adele","adelheid","adelice","adelina","adelind","adeline","adella","adelle","adena","adey","adi","adiana","adina","adora","adore","adoree","adorne","adrea","adria","adriaens","adrian","adriana","adriane","adrianna","adrianne","adriena","adrienne","aeriel","aeriela","aeriell","afton","ag","agace","agata","agatha","agathe","aggi","aggie","aggy","agna","agnella","agnes","agnese","agnesse","agneta","agnola","agretha","aida","aidan","aigneis","aila","aile","ailee","aileen","ailene","ailey","aili","ailina","ailis","ailsun","ailyn","aime","aimee","aimil","aindrea","ainslee","ainsley","ainslie","ajay","alaine","alameda","alana","alanah","alane","alanna","alayne","alberta","albertina","albertine","albina","alecia","aleda","aleece","aleen","alejandra","alejandrina","alena","alene","alessandra","aleta","alethea","alex","alexa","alexandra","alexandrina","alexi","alexia","alexina","alexine","alexis","alfi","alfie","alfreda","alfy","ali","alia","alica","alice","alicea","alicia","alida","alidia","alie","alika","alikee","alina","aline","alis","alisa","alisha","alison","alissa","alisun","alix","aliza","alla","alleen","allegra","allene","alli","allianora","allie","allina","allis","allison","allissa","allix","allsun","allx","ally","allyce","allyn","allys","allyson","alma","almeda","almeria","almeta","almira","almire","aloise","aloisia","aloysia","alta","althea","alvera","alverta","alvina","alvinia","alvira","alyce","alyda","alys","alysa","alyse","alysia","alyson","alyss","alyssa","amabel","amabelle","amalea","amalee","amaleta","amalia","amalie","amalita","amalle","amanda","amandi","amandie","amandy","amara","amargo","amata","amber","amberly","ambur","ame","amelia","amelie","amelina","ameline","amelita","ami","amie","amii","amil","amitie","amity","ammamaria","amy","amye","ana","anabal","anabel","anabella","anabelle","analiese","analise","anallese","anallise","anastasia","anastasie","anastassia","anatola","andee","andeee","anderea","andi","andie","andra","andrea","andreana","andree","andrei","andria","andriana","andriette","andromache","andy","anestassia","anet","anett","anetta","anette","ange","angel","angela","angele","angelia","angelica","angelika","angelina","angeline","angelique","angelita","angelle","angie","angil","angy","ania","anica","anissa","anita","anitra","anjanette","anjela","ann","ann-marie","anna","anna-diana","anna-diane","anna-maria","annabal","annabel","annabela","annabell","annabella","annabelle","annadiana","annadiane","annalee","annaliese","annalise","annamaria","annamarie","anne","anne-corinne","anne-marie","annecorinne","anneliese","annelise","annemarie","annetta","annette","anni","annice","annie","annis","annissa","annmaria","annmarie","annnora","annora","anny","anselma","ansley","anstice","anthe","anthea","anthia","anthiathia","antoinette","antonella","antonetta","antonia","antonie","antonietta","antonina","anya","appolonia","april","aprilette","ara","arabel","arabela","arabele","arabella","arabelle","arda","ardath","ardeen","ardelia","ardelis","ardella","ardelle","arden","ardene","ardenia","ardine","ardis","ardisj","ardith","ardra","ardyce","ardys","ardyth","aretha","ariadne","ariana","aridatha","ariel","ariela","ariella","arielle","arlana","arlee","arleen","arlen","arlena","arlene","arleta","arlette","arleyne","arlie","arliene","arlina","arlinda","arline","arluene","arly","arlyn","arlyne","aryn","ashely","ashia","ashien","ashil","ashla","ashlan","ashlee","ashleigh","ashlen","ashley","ashli","ashlie","ashly","asia","astra","astrid","astrix","atalanta","athena","athene","atlanta","atlante","auberta","aubine","aubree","aubrette","aubrey","aubrie","aubry","audi","audie","audra","audre","audrey","audrie","audry","audrye","audy","augusta","auguste","augustina","augustine","aundrea","aura","aurea","aurel","aurelea","aurelia","aurelie","auria","aurie","aurilia","aurlie","auroora","aurora","aurore","austin","austina","austine","ava","aveline","averil","averyl","avie","avis","aviva","avivah","avril","avrit","ayn","bab","babara","babb","babbette","babbie","babette","babita","babs","bambi","bambie","bamby","barb","barbabra","barbara","barbara-anne","barbaraanne","barbe","barbee","barbette","barbey","barbi","barbie","barbra","barby","bari","barrie","barry","basia","bathsheba","batsheva","bea","beatrice","beatrisa","beatrix","beatriz","bebe","becca","becka","becki","beckie","becky","bee","beilul","beitris","bekki","bel","belia","belicia","belinda","belita","bell","bella","bellanca","belle","bellina","belva","belvia","bendite","benedetta","benedicta","benedikta","benetta","benita","benni","bennie","benny","benoite","berenice","beret","berget","berna","bernadene","bernadette","bernadina","bernadine","bernardina","bernardine","bernelle","bernete","bernetta","bernette","berni","bernice","bernie","bernita","berny","berri","berrie","berry","bert","berta","berte","bertha","berthe","berti","bertie","bertina","bertine","berty","beryl","beryle","bess","bessie","bessy","beth","bethanne","bethany","bethena","bethina","betsey","betsy","betta","bette","bette-ann","betteann","betteanne","betti","bettina","bettine","betty","bettye","beulah","bev","beverie","beverlee","beverley","beverlie","beverly","bevvy","bianca","bianka","bibbie","bibby","bibbye","bibi","biddie","biddy","bidget","bili","bill","billi","billie","billy","billye","binni","binnie","binny","bird","birdie","birgit","birgitta","blair","blaire","blake","blakelee","blakeley","blanca","blanch","blancha","blanche","blinni","blinnie","blinny","bliss","blisse","blithe","blondell","blondelle","blondie","blondy","blythe","bobbe","bobbee","bobbette","bobbi","bobbie","bobby","bobbye","bobette","bobina","bobine","bobinette","bonita","bonnee","bonni","bonnibelle","bonnie","bonny","brana","brandais","brande","brandea","brandi","brandice","brandie","brandise","brandy","breanne","brear","bree","breena","bren","brena","brenda","brenn","brenna","brett","bria","briana","brianna","brianne","bride","bridget","bridgette","bridie","brier","brietta","brigid","brigida","brigit","brigitta","brigitte","brina","briney","brinn","brinna","briny","brit","brita","britney","britni","britt","britta","brittan","brittaney","brittani","brittany","britte","britteny","brittne","brittney","brittni","brook","brooke","brooks","brunhilda","brunhilde","bryana","bryn","bryna","brynn","brynna","brynne","buffy","bunni","bunnie","bunny","cacilia","cacilie","cahra","cairistiona","caitlin","caitrin","cal","calida","calla","calley","calli","callida","callie","cally","calypso","cam","camala","camel","camella","camellia","cami","camila","camile","camilla","camille","cammi","cammie","cammy","candace","candi","candice","candida","candide","candie","candis","candra","candy","caprice","cara","caralie","caren","carena","caresa","caressa","caresse","carey","cari","caria","carie","caril","carilyn","carin","carina","carine","cariotta","carissa","carita","caritta","carla","carlee","carleen","carlen","carlene","carley","carlie","carlin","carlina","carline","carlita","carlota","carlotta","carly","carlye","carlyn","carlynn","carlynne","carma","carmel","carmela","carmelia","carmelina","carmelita","carmella","carmelle","carmen","carmencita","carmina","carmine","carmita","carmon","caro","carol","carol-jean","carola","carolan","carolann","carole","carolee","carolin","carolina","caroline","caroljean","carolyn","carolyne","carolynn","caron","carree","carri","carrie","carrissa","carroll","carry","cary","caryl","caryn","casandra","casey","casi","casie","cass","cassandra","cassandre","cassandry","cassaundra","cassey","cassi","cassie","cassondra","cassy","catarina","cate","caterina","catha","catharina","catharine","cathe","cathee","catherin","catherina","catherine","cathi","cathie","cathleen","cathlene","cathrin","cathrine","cathryn","cathy","cathyleen","cati","catie","catina","catlaina","catlee","catlin","catrina","catriona","caty","caye","cayla","cecelia","cecil","cecile","ceciley","cecilia","cecilla","cecily","ceil","cele","celene","celesta","celeste","celestia","celestina","celestine","celestyn","celestyna","celia","celie","celina","celinda","celine","celinka","celisse","celka","celle","cesya","chad","chanda","chandal","chandra","channa","chantal","chantalle","charil","charin","charis","charissa","charisse","charita","charity","charla","charlean","charleen","charlena","charlene","charline","charlot","charlotta","charlotte","charmain","charmaine","charmane","charmian","charmine","charmion","charo","charyl","chastity","chelsae","chelsea","chelsey","chelsie","chelsy","cher","chere","cherey","cheri","cherianne","cherice","cherida","cherie","cherilyn","cherilynn","cherin","cherise","cherish","cherlyn","cherri","cherrita","cherry","chery","cherye","cheryl","cheslie","chiarra","chickie","chicky","chiquia","chiquita","chlo","chloe","chloette","chloris","chris","chrissie","chrissy","christa","christabel","christabella","christal","christalle","christan","christean","christel","christen","christi","christian","christiana","christiane","christie","christin","christina","christine","christy","christye","christyna","chrysa","chrysler","chrystal","chryste","chrystel","cicely","cicily","ciel","cilka","cinda","cindee","cindelyn","cinderella","cindi","cindie","cindra","cindy","cinnamon","cissiee","cissy","clair","claire","clara","clarabelle","clare","claresta","clareta","claretta","clarette","clarey","clari","claribel","clarice","clarie","clarinda","clarine","clarissa","clarisse","clarita","clary","claude","claudelle","claudetta","claudette","claudia","claudie","claudina","claudine","clea","clem","clemence","clementia","clementina","clementine","clemmie","clemmy","cleo","cleopatra","clerissa","clio","clo","cloe","cloris","clotilda","clovis","codee","codi","codie","cody","coleen","colene","coletta","colette","colleen","collen","collete","collette","collie","colline","colly","con","concettina","conchita","concordia","conni","connie","conny","consolata","constance","constancia","constancy","constanta","constantia","constantina","constantine","consuela","consuelo","cookie","cora","corabel","corabella","corabelle","coral","coralie","coraline","coralyn","cordelia","cordelie","cordey","cordi","cordie","cordula","cordy","coreen","corella","corenda","corene","coretta","corette","corey","cori","corie","corilla","corina","corine","corinna","corinne","coriss","corissa","corliss","corly","cornela","cornelia","cornelle","cornie","corny","correna","correy","corri","corrianne","corrie","corrina","corrine","corrinne","corry","cortney","cory","cosetta","cosette","costanza","courtenay","courtnay","courtney","crin","cris","crissie","crissy","crista","cristabel","cristal","cristen","cristi","cristie","cristin","cristina","cristine","cristionna","cristy","crysta","crystal","crystie","cthrine","cyb","cybil","cybill","cymbre","cynde","cyndi","cyndia","cyndie","cyndy","cynthea","cynthia","cynthie","cynthy","dacey","dacia","dacie","dacy","dael","daffi","daffie","daffy","dagmar","dahlia","daile","daisey","daisi","daisie","daisy","dale","dalenna","dalia","dalila","dallas","daloris","damara","damaris","damita","dana","danell","danella","danette","dani","dania","danica","danice","daniela","daniele","daniella","danielle","danika","danila","danit","danita","danna","danni","dannie","danny","dannye","danya","danyelle","danyette","daphene","daphna","daphne","dara","darb","darbie","darby","darcee","darcey","darci","darcie","darcy","darda","dareen","darell","darelle","dari","daria","darice","darla","darleen","darlene","darline","darlleen","daron","darrelle","darryl","darsey","darsie","darya","daryl","daryn","dasha","dasi","dasie","dasya","datha","daune","daveen","daveta","davida","davina","davine","davita","dawn","dawna","dayle","dayna","ddene","de","deana","deane","deanna","deanne","deb","debbi","debbie","debby","debee","debera","debi","debor","debora","deborah","debra","dede","dedie","dedra","dee","dee dee","deeann","deeanne","deedee","deena","deerdre","deeyn","dehlia","deidre","deina","deirdre","del","dela","delcina","delcine","delia","delila","delilah","delinda","dell","della","delly","delora","delores","deloria","deloris","delphine","delphinia","demeter","demetra","demetria","demetris","dena","deni","denice","denise","denna","denni","dennie","denny","deny","denys","denyse","deonne","desdemona","desirae","desiree","desiri","deva","devan","devi","devin","devina","devinne","devon","devondra","devonna","devonne","devora","di","diahann","dian","diana","diandra","diane","diane-marie","dianemarie","diann","dianna","dianne","diannne","didi","dido","diena","dierdre","dina","dinah","dinnie","dinny","dion","dione","dionis","dionne","dita","dix","dixie","dniren","dode","dodi","dodie","dody","doe","doll","dolley","dolli","dollie","dolly","dolores","dolorita","doloritas","domeniga","dominga","domini","dominica","dominique","dona","donella","donelle","donetta","donia","donica","donielle","donna","donnamarie","donni","donnie","donny","dora","doralia","doralin","doralyn","doralynn","doralynne","dore","doreen","dorelia","dorella","dorelle","dorena","dorene","doretta","dorette","dorey","dori","doria","dorian","dorice","dorie","dorine","doris","dorisa","dorise","dorita","doro","dorolice","dorolisa","dorotea","doroteya","dorothea","dorothee","dorothy","dorree","dorri","dorrie","dorris","dorry","dorthea","dorthy","dory","dosi","dot","doti","dotti","dottie","dotty","dre","dreddy","dredi","drona","dru","druci","drucie","drucill","drucy","drusi","drusie","drusilla","drusy","dulce","dulcea","dulci","dulcia","dulciana","dulcie","dulcine","dulcinea","dulcy","dulsea","dusty","dyan","dyana","dyane","dyann","dyanna","dyanne","dyna","dynah","eachelle","eada","eadie","eadith","ealasaid","eartha","easter","eba","ebba","ebonee","ebony","eda","eddi","eddie","eddy","ede","edee","edeline","eden","edi","edie","edin","edita","edith","editha","edithe","ediva","edna","edwina","edy","edyth","edythe","effie","eileen","eilis","eimile","eirena","ekaterina","elaina","elaine","elana","elane","elayne","elberta","elbertina","elbertine","eleanor","eleanora","eleanore","electra","eleen","elena","elene","eleni","elenore","eleonora","eleonore","elfie","elfreda","elfrida","elfrieda","elga","elianora","elianore","elicia","elie","elinor","elinore","elisa","elisabet","elisabeth","elisabetta","elise","elisha","elissa","elita","eliza","elizabet","elizabeth","elka","elke","ella","elladine","elle","ellen","ellene","ellette","elli","ellie","ellissa","elly","ellyn","ellynn","elmira","elna","elnora","elnore","eloisa","eloise","elonore","elora","elsa","elsbeth","else","elset","elsey","elsi","elsie","elsinore","elspeth","elsy","elva","elvera","elvina","elvira","elwira","elyn","elyse","elysee","elysha","elysia","elyssa","em","ema","emalee","emalia","emelda","emelia","emelina","emeline","emelita","emelyne","emera","emilee","emili","emilia","emilie","emiline","emily","emlyn","emlynn","emlynne","emma","emmalee","emmaline","emmalyn","emmalynn","emmalynne","emmeline","emmey","emmi","emmie","emmy","emmye","emogene","emyle","emylee","engracia","enid","enrica","enrichetta","enrika","enriqueta","eolanda","eolande","eran","erda","erena","erica","ericha","ericka","erika","erin","erina","erinn","erinna","erma","ermengarde","ermentrude","ermina","erminia","erminie","erna","ernaline","ernesta","ernestine","ertha","eryn","esma","esmaria","esme","esmeralda","essa","essie","essy","esta","estel","estele","estell","estella","estelle","ester","esther","estrella","estrellita","ethel","ethelda","ethelin","ethelind","etheline","ethelyn","ethyl","etta","etti","ettie","etty","eudora","eugenia","eugenie","eugine","eula","eulalie","eunice","euphemia","eustacia","eva","evaleen","evangelia","evangelin","evangelina","evangeline","evania","evanne","eve","eveleen","evelina","eveline","evelyn","evey","evie","evita","evonne","evvie","evvy","evy","eyde","eydie","ezmeralda","fae","faina","faith","fallon","fan","fanchette","fanchon","fancie","fancy","fanechka","fania","fanni","fannie","fanny","fanya","fara","farah","farand","farica","farra","farrah","farrand","faun","faunie","faustina","faustine","fawn","fawne","fawnia","fay","faydra","faye","fayette","fayina","fayre","fayth","faythe","federica","fedora","felecia","felicdad","felice","felicia","felicity","felicle","felipa","felisha","felita","feliza","fenelia","feodora","ferdinanda","ferdinande","fern","fernanda","fernande","fernandina","ferne","fey","fiann","fianna","fidela","fidelia","fidelity","fifi","fifine","filia","filide","filippa","fina","fiona","fionna","fionnula","fiorenze","fleur","fleurette","flo","flor","flora","florance","flore","florella","florence","florencia","florentia","florenza","florette","flori","floria","florida","florie","florina","florinda","floris","florri","florrie","florry","flory","flossi","flossie","flossy","flss","fran","francene","frances","francesca","francine","francisca","franciska","francoise","francyne","frank","frankie","franky","franni","frannie","franny","frayda","fred","freda","freddi","freddie","freddy","fredelia","frederica","fredericka","frederique","fredi","fredia","fredra","fredrika","freida","frieda","friederike","fulvia","gabbey","gabbi","gabbie","gabey","gabi","gabie","gabriel","gabriela","gabriell","gabriella","gabrielle","gabriellia","gabrila","gaby","gae","gael","gail","gale","galina","garland","garnet","garnette","gates","gavra","gavrielle","gay","gaye","gayel","gayla","gayle","gayleen","gaylene","gaynor","gelya","gena","gene","geneva","genevieve","genevra","genia","genna","genni","gennie","gennifer","genny","genovera","genvieve","george","georgeanna","georgeanne","georgena","georgeta","georgetta","georgette","georgia","georgiana","georgianna","georgianne","georgie","georgina","georgine","geralda","geraldine","gerda","gerhardine","geri","gerianna","gerianne","gerladina","germain","germaine","germana","gerri","gerrie","gerrilee","gerry","gert","gerta","gerti","gertie","gertrud","gertruda","gertrude","gertrudis","gerty","giacinta","giana","gianina","gianna","gigi","gilberta","gilberte","gilbertina","gilbertine","gilda","gilemette","gill","gillan","gilli","gillian","gillie","gilligan","gilly","gina","ginelle","ginevra","ginger","ginni","ginnie","ginnifer","ginny","giorgia","giovanna","gipsy","giralda","gisela","gisele","gisella","giselle","giuditta","giulia","giulietta","giustina","gizela","glad","gladi","gladys","gleda","glen","glenda","glenine","glenn","glenna","glennie","glennis","glori","gloria","gloriana","gloriane","glory","glyn","glynda","glynis","glynnis","gnni","godiva","golda","goldarina","goldi","goldia","goldie","goldina","goldy","grace","gracia","gracie","grata","gratia","gratiana","gray","grayce","grazia","greer","greta","gretal","gretchen","grete","gretel","grethel","gretna","gretta","grier","griselda","grissel","guendolen","guenevere","guenna","guglielma","gui","guillema","guillemette","guinevere","guinna","gunilla","gus","gusella","gussi","gussie","gussy","gusta","gusti","gustie","gusty","gwen","gwendolen","gwendolin","gwendolyn","gweneth","gwenette","gwenneth","gwenni","gwennie","gwenny","gwenora","gwenore","gwyn","gwyneth","gwynne","gypsy","hadria","hailee","haily","haleigh","halette","haley","hali","halie","halimeda","halley","halli","hallie","hally","hana","hanna","hannah","hanni","hannie","hannis","hanny","happy","harlene","harley","harli","harlie","harmonia","harmonie","harmony","harri","harrie","harriet","harriett","harrietta","harriette","harriot","harriott","hatti","hattie","hatty","hayley","hazel","heath","heather","heda","hedda","heddi","heddie","hedi","hedvig","hedvige","hedwig","hedwiga","hedy","heida","heidi","heidie","helaina","helaine","helen","helen-elizabeth","helena","helene","helenka","helga","helge","helli","heloise","helsa","helyn","hendrika","henka","henrie","henrieta","henrietta","henriette","henryetta","hephzibah","hermia","hermina","hermine","herminia","hermione","herta","hertha","hester","hesther","hestia","hetti","hettie","hetty","hilary","hilda","hildagard","hildagarde","hilde","hildegaard","hildegarde","hildy","hillary","hilliary","hinda","holli","hollie","holly","holly-anne","hollyanne","honey","honor","honoria","hope","horatia","hortense","hortensia","hulda","hyacinth","hyacintha","hyacinthe","hyacinthia","hyacinthie","hynda","ianthe","ibbie","ibby","ida","idalia","idalina","idaline","idell","idelle","idette","ileana","ileane","ilene","ilise","ilka","illa","ilsa","ilse","ilysa","ilyse","ilyssa","imelda","imogen","imogene","imojean","ina","indira","ines","inesita","inessa","inez","inga","ingaberg","ingaborg","inge","ingeberg","ingeborg","inger","ingrid","ingunna","inna","iolande","iolanthe","iona","iormina","ira","irena","irene","irina","iris","irita","irma","isa","isabel","isabelita","isabella","isabelle","isadora","isahella","iseabal","isidora","isis","isobel","issi","issie","issy","ivett","ivette","ivie","ivonne","ivory","ivy","izabel","jacenta","jacinda","jacinta","jacintha","jacinthe","jackelyn","jacki","jackie","jacklin","jacklyn","jackquelin","jackqueline","jacky","jaclin","jaclyn","jacquelin","jacqueline","jacquelyn","jacquelynn","jacquenetta","jacquenette","jacquetta","jacquette","jacqui","jacquie","jacynth","jada","jade","jaime","jaimie","jaine","jami","jamie","jamima","jammie","jan","jana","janaya","janaye","jandy","jane","janean","janeczka","janeen","janel","janela","janella","janelle","janene","janenna","janessa","janet","janeta","janetta","janette","janeva","janey","jania","janice","janie","janifer","janina","janine","janis","janith","janka","janna","jannel","jannelle","janot","jany","jaquelin","jaquelyn","jaquenetta","jaquenette","jaquith","jasmin","jasmina","jasmine","jayme","jaymee","jayne","jaynell","jazmin","jean","jeana","jeane","jeanelle","jeanette","jeanie","jeanine","jeanna","jeanne","jeannette","jeannie","jeannine","jehanna","jelene","jemie","jemima","jemimah","jemmie","jemmy","jen","jena","jenda","jenelle","jeni","jenica","jeniece","jenifer","jeniffer","jenilee","jenine","jenn","jenna","jennee","jennette","jenni","jennica","jennie","jennifer","jennilee","jennine","jenny","jeralee","jere","jeri","jermaine","jerrie","jerrilee","jerrilyn","jerrine","jerry","jerrylee","jess","jessa","jessalin","jessalyn","jessamine","jessamyn","jesse","jesselyn","jessi","jessica","jessie","jessika","jessy","jewel","jewell","jewelle","jill","jillana","jillane","jillayne","jilleen","jillene","jilli","jillian","jillie","jilly","jinny","jo","jo ann","jo-ann","jo-anne","joan","joana","joane","joanie","joann","joanna","joanne","joannes","jobey","jobi","jobie","jobina","joby","jobye","jobyna","jocelin","joceline","jocelyn","jocelyne","jodee","jodi","jodie","jody","joeann","joela","joelie","joell","joella","joelle","joellen","joelly","joellyn","joelynn","joete","joey","johanna","johannah","johna","johnath","johnette","johnna","joice","jojo","jolee","joleen","jolene","joletta","joli","jolie","joline","joly","jolyn","jolynn","jonell","joni","jonie","jonis","jordain","jordan","jordana","jordanna","jorey","jori","jorie","jorrie","jorry","joscelin","josee","josefa","josefina","josepha","josephina","josephine","josey","josi","josie","josselyn","josy","jourdan","joy","joya","joyan","joyann","joyce","joycelin","joye","jsandye","juana","juanita","judi","judie","judith","juditha","judy","judye","juieta","julee","juli","julia","juliana","juliane","juliann","julianna","julianne","julie","julienne","juliet","julieta","julietta","juliette","julina","juline","julissa","julita","june","junette","junia","junie","junina","justina","justine","justinn","jyoti","kacey","kacie","kacy","kaela","kai","kaia","kaila","kaile","kailey","kaitlin","kaitlyn","kaitlynn","kaja","kakalina","kala","kaleena","kali","kalie","kalila","kalina","kalinda","kalindi","kalli","kally","kameko","kamila","kamilah","kamillah","kandace","kandy","kania","kanya","kara","kara-lynn","karalee","karalynn","kare","karee","karel","karen","karena","kari","karia","karie","karil","karilynn","karin","karina","karine","kariotta","karisa","karissa","karita","karla","karlee","karleen","karlen","karlene","karlie","karlotta","karlotte","karly","karlyn","karmen","karna","karol","karola","karole","karolina","karoline","karoly","karon","karrah","karrie","karry","kary","karyl","karylin","karyn","kasey","kass","kassandra","kassey","kassi","kassia","kassie","kat","kata","katalin","kate","katee","katerina","katerine","katey","kath","katha","katharina","katharine","katharyn","kathe","katherina","katherine","katheryn","kathi","kathie","kathleen","kathlin","kathrine","kathryn","kathryne","kathy","kathye","kati","katie","katina","katine","katinka","katleen","katlin","katrina","katrine","katrinka","katti","kattie","katuscha","katusha","katy","katya","kay","kaycee","kaye","kayla","kayle","kaylee","kayley","kaylil","kaylyn","keeley","keelia","keely","kelcey","kelci","kelcie","kelcy","kelila","kellen","kelley","kelli","kellia","kellie","kellina","kellsie","kelly","kellyann","kelsey","kelsi","kelsy","kendra","kendre","kenna","keri","keriann","kerianne","kerri","kerrie","kerrill","kerrin","kerry","kerstin","kesley","keslie","kessia","kessiah","ketti","kettie","ketty","kevina","kevyn","ki","kiah","kial","kiele","kiersten","kikelia","kiley","kim","kimberlee","kimberley","kimberli","kimberly","kimberlyn","kimbra","kimmi","kimmie","kimmy","kinna","kip","kipp","kippie","kippy","kira","kirbee","kirbie","kirby","kiri","kirsten","kirsteni","kirsti","kirstin","kirstyn","kissee","kissiah","kissie","kit","kitti","kittie","kitty","kizzee","kizzie","klara","klarika","klarrisa","konstance","konstanze","koo","kora","koral","koralle","kordula","kore","korella","koren","koressa","kori","korie","korney","korrie","korry","kris","krissie","krissy","krista","kristal","kristan","kriste","kristel","kristen","kristi","kristien","kristin","kristina","kristine","kristy","kristyn","krysta","krystal","krystalle","krystle","krystyna","kyla","kyle","kylen","kylie","kylila","kylynn","kym","kynthia","kyrstin","la verne","lacee","lacey","lacie","lacy","ladonna","laetitia","laina","lainey","lana","lanae","lane","lanette","laney","lani","lanie","lanita","lanna","lanni","lanny","lara","laraine","lari","larina","larine","larisa","larissa","lark","laryssa","latashia","latia","latisha","latrena","latrina","laura","lauraine","laural","lauralee","laure","lauree","laureen","laurel","laurella","lauren","laurena","laurene","lauretta","laurette","lauri","laurianne","laurice","laurie","lauryn","lavena","laverna","laverne","lavina","lavinia","lavinie","layla","layne","layney","lea","leah","leandra","leann","leanna","leanor","leanora","lebbie","leda","lee","leeann","leeanne","leela","leelah","leena","leesa","leese","legra","leia","leigh","leigha","leila","leilah","leisha","lela","lelah","leland","lelia","lena","lenee","lenette","lenka","lenna","lenora","lenore","leodora","leoine","leola","leoline","leona","leonanie","leone","leonelle","leonie","leonora","leonore","leontine","leontyne","leora","leshia","lesley","lesli","leslie","lesly","lesya","leta","lethia","leticia","letisha","letitia","letizia","letta","letti","lettie","letty","lexi","lexie","lexine","lexis","lexy","leyla","lezlie","lia","lian","liana","liane","lianna","lianne","lib","libbey","libbi","libbie","libby","licha","lida","lidia","liesa","lil","lila","lilah","lilas","lilia","lilian","liliane","lilias","lilith","lilla","lilli","lillian","lillis","lilllie","lilly","lily","lilyan","lin","lina","lind","linda","lindi","lindie","lindsay","lindsey","lindsy","lindy","linea","linell","linet","linette","linn","linnea","linnell","linnet","linnie","linzy","lira","lisa","lisabeth","lisbeth","lise","lisetta","lisette","lisha","lishe","lissa","lissi","lissie","lissy","lita","liuka","liv","liva","livia","livvie","livvy","livvyy","livy","liz","liza","lizabeth","lizbeth","lizette","lizzie","lizzy","loella","lois","loise","lola","loleta","lolita","lolly","lona","lonee","loni","lonna","lonni","lonnie","lora","lorain","loraine","loralee","loralie","loralyn","loree","loreen","lorelei","lorelle","loren","lorena","lorene","lorenza","loretta","lorette","lori","loria","lorianna","lorianne","lorie","lorilee","lorilyn","lorinda","lorine","lorita","lorna","lorne","lorraine","lorrayne","lorri","lorrie","lorrin","lorry","lory","lotta","lotte","lotti","lottie","lotty","lou","louella","louisa","louise","louisette","loutitia","lu","luce","luci","lucia","luciana","lucie","lucienne","lucila","lucilia","lucille","lucina","lucinda","lucine","lucita","lucky","lucretia","lucy","ludovika","luella","luelle","luisa","luise","lula","lulita","lulu","lura","lurette","lurleen","lurlene","lurline","lusa","luz","lyda","lydia","lydie","lyn","lynda","lynde","lyndel","lyndell","lyndsay","lyndsey","lyndsie","lyndy","lynea","lynelle","lynett","lynette","lynn","lynna","lynne","lynnea","lynnell","lynnelle","lynnet","lynnett","lynnette","lynsey","lyssa","mab","mabel","mabelle","mable","mada","madalena","madalyn","maddalena","maddi","maddie","maddy","madel","madelaine","madeleine","madelena","madelene","madelin","madelina","madeline","madella","madelle","madelon","madelyn","madge","madlen","madlin","madonna","mady","mae","maegan","mag","magda","magdaia","magdalen","magdalena","magdalene","maggee","maggi","maggie","maggy","mahala","mahalia","maia","maible","maiga","maighdiln","mair","maire","maisey","maisie","maitilde","mala","malanie","malena","malia","malina","malinda","malinde","malissa","malissia","mallissa","mallorie","mallory","malorie","malory","malva","malvina","malynda","mame","mamie","manda","mandi","mandie","mandy","manon","manya","mara","marabel","marcela","marcelia","marcella","marcelle","marcellina","marcelline","marchelle","marci","marcia","marcie","marcile","marcille","marcy","mareah","maren","marena","maressa","marga","margalit","margalo","margaret","margareta","margarete","margaretha","margarethe","margaretta","margarette","margarita","margaux","marge","margeaux","margery","marget","margette","margi","margie","margit","margo","margot","margret","marguerite","margy","mari","maria","mariam","marian","mariana","mariann","marianna","marianne","maribel","maribelle","maribeth","marice","maridel","marie","marie-ann","marie-jeanne","marieann","mariejeanne","mariel","mariele","marielle","mariellen","marietta","mariette","marigold","marijo","marika","marilee","marilin","marillin","marilyn","marin","marina","marinna","marion","mariquilla","maris","marisa","mariska","marissa","marita","maritsa","mariya","marj","marja","marje","marji","marjie","marjorie","marjory","marjy","marketa","marla","marlane","marleah","marlee","marleen","marlena","marlene","marley","marlie","marline","marlo","marlyn","marna","marne","marney","marni","marnia","marnie","marquita","marrilee","marris","marrissa","marsha","marsiella","marta","martelle","martguerita","martha","marthe","marthena","marti","martica","martie","martina","martita","marty","martynne","mary","marya","maryann","maryanna","maryanne","marybelle","marybeth","maryellen","maryjane","maryjo","maryl","marylee","marylin","marylinda","marylou","marylynne","maryrose","marys","marysa","masha","matelda","mathilda","mathilde","matilda","matilde","matti","mattie","matty","maud","maude","maudie","maura","maure","maureen","maureene","maurene","maurine","maurise","maurita","maurizia","mavis","mavra","max","maxi","maxie","maxine","maxy","may","maybelle","maye","mead","meade","meagan","meaghan","meara","mechelle","meg","megan","megen","meggi","meggie","meggy","meghan","meghann","mehetabel","mei","mel","mela","melamie","melania","melanie","melantha","melany","melba","melesa","melessa","melicent","melina","melinda","melinde","melisa","melisande","melisandra","melisenda","melisent","melissa","melisse","melita","melitta","mella","melli","mellicent","mellie","mellisa","mellisent","melloney","melly","melodee","melodie","melody","melonie","melony","melosa","melva","mercedes","merci","mercie","mercy","meredith","meredithe","meridel","meridith","meriel","merilee","merilyn","meris","merissa","merl","merla","merle","merlina","merline","merna","merola","merralee","merridie","merrie","merrielle","merrile","merrilee","merrili","merrill","merrily","merry","mersey","meryl","meta","mia","micaela","michaela","michaelina","michaeline","michaella","michal","michel","michele","michelina","micheline","michell","michelle","micki","mickie","micky","midge","mignon","mignonne","miguela","miguelita","mikaela","mil","mildred","mildrid","milena","milicent","milissent","milka","milli","millicent","millie","millisent","milly","milzie","mimi","min","mina","minda","mindy","minerva","minetta","minette","minna","minnaminnie","minne","minni","minnie","minnnie","minny","minta","miof mela","miquela","mira","mirabel","mirabella","mirabelle","miran","miranda","mireielle","mireille","mirella","mirelle","miriam","mirilla","mirna","misha","missie","missy","misti","misty","mitzi","modesta","modestia","modestine","modesty","moina","moira","moll","mollee","molli","mollie","molly","mommy","mona","monah","monica","monika","monique","mora","moreen","morena","morgan","morgana","morganica","morganne","morgen","moria","morissa","morna","moselle","moyna","moyra","mozelle","muffin","mufi","mufinella","muire","mureil","murial","muriel","murielle","myra","myrah","myranda","myriam","myrilla","myrle","myrlene","myrna","myrta","myrtia","myrtice","myrtie","myrtle","nada","nadean","nadeen","nadia","nadine","nadiya","nady","nadya","nalani","nan","nana","nananne","nance","nancee","nancey","nanci","nancie","nancy","nanete","nanette","nani","nanice","nanine","nannette","nanni","nannie","nanny","nanon","naoma","naomi","nara","nari","nariko","nat","nata","natala","natalee","natalie","natalina","nataline","natalya","natasha","natassia","nathalia","nathalie","natividad","natka","natty","neala","neda","nedda","nedi","neely","neila","neile","neilla","neille","nelia","nelie","nell","nelle","nelli","nellie","nelly","nerissa","nerita","nert","nerta","nerte","nerti","nertie","nerty","nessa","nessi","nessie","nessy","nesta","netta","netti","nettie","nettle","netty","nevsa","neysa","nichol","nichole","nicholle","nicki","nickie","nicky","nicol","nicola","nicole","nicolea","nicolette","nicoli","nicolina","nicoline","nicolle","nikaniki","nike","niki","nikki","nikkie","nikoletta","nikolia","nina","ninetta","ninette","ninnetta","ninnette","ninon","nissa","nisse","nissie","nissy","nita","nixie","noami","noel","noelani","noell","noella","noelle","noellyn","noelyn","noemi","nola","nolana","nolie","nollie","nomi","nona","nonah","noni","nonie","nonna","nonnah","nora","norah","norean","noreen","norene","norina","norine","norma","norri","norrie","norry","novelia","nydia","nyssa","octavia","odele","odelia","odelinda","odella","odelle","odessa","odetta","odette","odilia","odille","ofelia","ofella","ofilia","ola","olenka","olga","olia","olimpia","olive","olivette","olivia","olivie","oliy","ollie","olly","olva","olwen","olympe","olympia","olympie","ondrea","oneida","onida","oona","opal","opalina","opaline","ophelia","ophelie","ora","oralee","oralia","oralie","oralla","oralle","orel","orelee","orelia","orelie","orella","orelle","oriana","orly","orsa","orsola","ortensia","otha","othelia","othella","othilia","othilie","ottilie","page","paige","paloma","pam","pamela","pamelina","pamella","pammi","pammie","pammy","pandora","pansie","pansy","paola","paolina","papagena","pat","patience","patrica","patrice","patricia","patrizia","patsy","patti","pattie","patty","paula","paule","pauletta","paulette","pauli","paulie","paulina","pauline","paulita","pauly","pavia","pavla","pearl","pearla","pearle","pearline","peg","pegeen","peggi","peggie","peggy","pen","penelopa","penelope","penni","pennie","penny","pepi","pepita","peri","peria","perl","perla","perle","perri","perrine","perry","persis","pet","peta","petra","petrina","petronella","petronia","petronilla","petronille","petunia","phaedra","phaidra","phebe","phedra","phelia","phil","philipa","philippa","philippe","philippine","philis","phillida","phillie","phillis","philly","philomena","phoebe","phylis","phyllida","phyllis","phyllys","phylys","pia","pier","pierette","pierrette","pietra","piper","pippa","pippy","polly","pollyanna","pooh","poppy","portia","pris","prisca","priscella","priscilla","prissie","pru","prudence","prudi","prudy","prue","queenie","quentin","querida","quinn","quinta","quintana","quintilla","quintina","rachael","rachel","rachele","rachelle","rae","raeann","raf","rafa","rafaela","rafaelia","rafaelita","rahal","rahel","raina","raine","rakel","ralina","ramona","ramonda","rana","randa","randee","randene","randi","randie","randy","ranee","rani","rania","ranice","ranique","ranna","raphaela","raquel","raquela","rasia","rasla","raven","ray","raychel","raye","rayna","raynell","rayshell","rea","reba","rebbecca","rebe","rebeca","rebecca","rebecka","rebeka","rebekah","rebekkah","ree","reeba","reena","reeta","reeva","regan","reggi","reggie","regina","regine","reiko","reina","reine","remy","rena","renae","renata","renate","rene","renee","renell","renelle","renie","rennie","reta","retha","revkah","rey","reyna","rhea","rheba","rheta","rhetta","rhiamon","rhianna","rhianon","rhoda","rhodia","rhodie","rhody","rhona","rhonda","riane","riannon","rianon","rica","ricca","rici","ricki","rickie","ricky","riki","rikki","rina","risa","rita","riva","rivalee","rivi","rivkah","rivy","roana","roanna","roanne","robbi","robbie","robbin","robby","robbyn","robena","robenia","roberta","robin","robina","robinet","robinett","robinetta","robinette","robinia","roby","robyn","roch","rochell","rochella","rochelle","rochette","roda","rodi","rodie","rodina","rois","romola","romona","romonda","romy","rona","ronalda","ronda","ronica","ronna","ronni","ronnica","ronnie","ronny","roobbie","rora","rori","rorie","rory","ros","rosa","rosabel","rosabella","rosabelle","rosaleen","rosalia","rosalie","rosalind","rosalinda","rosalinde","rosaline","rosalyn","rosalynd","rosamond","rosamund","rosana","rosanna","rosanne","rose","roseann","roseanna","roseanne","roselia","roselin","roseline","rosella","roselle","rosemaria","rosemarie","rosemary","rosemonde","rosene","rosetta","rosette","roshelle","rosie","rosina","rosita","roslyn","rosmunda","rosy","row","rowe","rowena","roxana","roxane","roxanna","roxanne","roxi","roxie","roxine","roxy","roz","rozalie","rozalin","rozamond","rozanna","rozanne","roze","rozele","rozella","rozelle","rozina","rubetta","rubi","rubia","rubie","rubina","ruby","ruperta","ruth","ruthann","ruthanne","ruthe","ruthi","ruthie","ruthy","ryann","rycca","saba","sabina","sabine","sabra","sabrina","sacha","sada","sadella","sadie","sadye","saidee","sal","salaidh","sallee","salli","sallie","sally","sallyann","sallyanne","saloma","salome","salomi","sam","samantha","samara","samaria","sammy","sande","sandi","sandie","sandra","sandy","sandye","sapphira","sapphire","sara","sara-ann","saraann","sarah","sarajane","saree","sarena","sarene","sarette","sari","sarina","sarine","sarita","sascha","sasha","sashenka","saudra","saundra","savina","sayre","scarlet","scarlett","sean","seana","seka","sela","selena","selene","selestina","selia","selie","selina","selinda","seline","sella","selle","selma","sena","sephira","serena","serene","shae","shaina","shaine","shalna","shalne","shana","shanda","shandee","shandeigh","shandie","shandra","shandy","shane","shani","shanie","shanna","shannah","shannen","shannon","shanon","shanta","shantee","shara","sharai","shari","sharia","sharity","sharl","sharla","sharleen","sharlene","sharline","sharon","sharona","sharron","sharyl","shaun","shauna","shawn","shawna","shawnee","shay","shayla","shaylah","shaylyn","shaylynn","shayna","shayne","shea","sheba","sheela","sheelagh","sheelah","sheena","sheeree","sheila","sheila-kathryn","sheilah","shel","shela","shelagh","shelba","shelbi","shelby","shelia","shell","shelley","shelli","shellie","shelly","shena","sher","sheree","sheri","sherie","sherill","sherilyn","sherline","sherri","sherrie","sherry","sherye","sheryl","shina","shir","shirl","shirlee","shirleen","shirlene","shirley","shirline","shoshana","shoshanna","siana","sianna","sib","sibbie","sibby","sibeal","sibel","sibella","sibelle","sibilla","sibley","sibyl","sibylla","sibylle","sidoney","sidonia","sidonnie","sigrid","sile","sileas","silva","silvana","silvia","silvie","simona","simone","simonette","simonne","sindee","siobhan","sioux","siouxie","sisely","sisile","sissie","sissy","siusan","sofia","sofie","sondra","sonia","sonja","sonni","sonnie","sonnnie","sonny","sonya","sophey","sophi","sophia","sophie","sophronia","sorcha","sosanna","stace","stacee","stacey","staci","stacia","stacie","stacy","stafani","star","starla","starlene","starlin","starr","stefa","stefania","stefanie","steffane","steffi","steffie","stella","stepha","stephana","stephani","stephanie","stephannie","stephenie","stephi","stephie","stephine","stesha","stevana","stevena","stoddard","storm","stormi","stormie","stormy","sue","suellen","sukey","suki","sula","sunny","sunshine","susan","susana","susanetta","susann","susanna","susannah","susanne","susette","susi","susie","susy","suzann","suzanna","suzanne","suzette","suzi","suzie","suzy","sybil","sybila","sybilla","sybille","sybyl","sydel","sydelle","sydney","sylvia","tabatha","tabbatha","tabbi","tabbie","tabbitha","tabby","tabina","tabitha","taffy","talia","tallia","tallie","tallou","tallulah","tally","talya","talyah","tamar","tamara","tamarah","tamarra","tamera","tami","tamiko","tamma","tammara","tammi","tammie","tammy","tamqrah","tamra","tana","tandi","tandie","tandy","tanhya","tani","tania","tanitansy","tansy","tanya","tara","tarah","tarra","tarrah","taryn","tasha","tasia","tate","tatiana","tatiania","tatum","tawnya","tawsha","ted","tedda","teddi","teddie","teddy","tedi","tedra","teena","teirtza","teodora","tera","teresa","terese","teresina","teresita","teressa","teri","teriann","terra","terri","terrie","terrijo","terry","terrye","tersina","terza","tess","tessa","tessi","tessie","tessy","thalia","thea","theadora","theda","thekla","thelma","theo","theodora","theodosia","theresa","therese","theresina","theresita","theressa","therine","thia","thomasa","thomasin","thomasina","thomasine","tiena","tierney","tiertza","tiff","tiffani","tiffanie","tiffany","tiffi","tiffie","tiffy","tilda","tildi","tildie","tildy","tillie","tilly","tim","timi","timmi","timmie","timmy","timothea","tina","tine","tiphani","tiphanie","tiphany","tish","tisha","tobe","tobey","tobi","toby","tobye","toinette","toma","tomasina","tomasine","tomi","tommi","tommie","tommy","toni","tonia","tonie","tony","tonya","tonye","tootsie","torey","tori","torie","torrie","tory","tova","tove","tracee","tracey","traci","tracie","tracy","trenna","tresa","trescha","tressa","tricia","trina","trish","trisha","trista","trix","trixi","trixie","trixy","truda","trude","trudey","trudi","trudie","trudy","trula","tuesday","twila","twyla","tybi","tybie","tyne","ula","ulla","ulrica","ulrika","ulrikaumeko","ulrike","umeko","una","ursa","ursala","ursola","ursula","ursulina","ursuline","uta","val","valaree","valaria","vale","valeda","valencia","valene","valenka","valentia","valentina","valentine","valera","valeria","valerie","valery","valerye","valida","valina","valli","vallie","vally","valma","valry","van","vanda","vanessa","vania","vanna","vanni","vannie","vanny","vanya","veda","velma","velvet","venita","venus","vera","veradis","vere","verena","verene","veriee","verile","verina","verine","verla","verna","vernice","veronica","veronika","veronike","veronique","vevay","vi","vicki","vickie","vicky","victoria","vida","viki","vikki","vikky","vilhelmina","vilma","vin","vina","vinita","vinni","vinnie","vinny","viola","violante","viole","violet","violetta","violette","virgie","virgina","virginia","virginie","vita","vitia","vitoria","vittoria","viv","viva","vivi","vivia","vivian","viviana","vivianna","vivianne","vivie","vivien","viviene","vivienne","viviyan","vivyan","vivyanne","vonni","vonnie","vonny","vyky","wallie","wallis","walliw","wally","waly","wanda","wandie","wandis","waneta","wanids","wenda","wendeline","wendi","wendie","wendy","wendye","wenona","wenonah","whitney","wileen","wilhelmina","wilhelmine","wilie","willa","willabella","willamina","willetta","willette","willi","willie","willow","willy","willyt","wilma","wilmette","wilona","wilone","wilow","windy","wini","winifred","winna","winnah","winne","winni","winnie","winnifred","winny","winona","winonah","wren","wrennie","wylma","wynn","wynne","wynnie","wynny","xaviera","xena","xenia","xylia","xylina","yalonda","yasmeen","yasmin","yelena","yetta","yettie","yetty","yevette","ynes","ynez","yoko","yolanda","yolande","yolane","yolanthe","yoshi","yoshiko","yovonnda","ysabel","yvette","yvonne","zabrina","zahara","zandra","zaneta","zara","zarah","zaria","zarla","zea","zelda","zelma","zena","zenia","zia","zilvia","zita","zitella","zoe","zola","zonda","zondra","zonnya","zora","zorah","zorana","zorina","zorine","zsa zsa","zsazsa","zulema","zuzana","stay","nature","orders","availability","africa","summary","turn","mean","growth","notes","agency","king","monday","european","activity","copy","although","drug","pics","western","income","force","cash","employment","overall","bay","river","commission","ad","package","contents","seen","players","engine","port","album","regional","stop","supplies","started","administration","bar","institute","views","plans","double","dog","build","screen","exchange","types","soon","sponsored","lines","electronic","continue","across","benefits","needed","season","apply","someone","held","ny","anything","printer","condition","effective","believe","organization","effect","asked","eur","mind","sunday","selection","casino","pdf","lost","tour","menu","volume","cross","anyone","mortgage","hope","silver","corporation","wish","inside","solution","mature","role","rather","weeks","addition","came","supply","nothing","certain","usr","executive","running","lower","necessary","union","jewelry","according","dc","clothing","mon","com","particular","fine","names","robert","homepage","hour","gas","skills","six","bush","islands","advice","career","military","rental","decision","leave","british","teens","pre","huge","sat","woman","facilities","zip","bid","kind","sellers","middle","move","cable","opportunities","taking","values","division","coming","tuesday","object","lesbian","appropriate","machine","logo","length","actually","nice","score","statistics","client","ok","returns","capital","follow","sample","investment","sent","shown","saturday","christmas","england","culture","band","flash","ms","lead","george","choice","went","starting","registration","fri","thursday","courses","consumer","hi","airport","foreign","artist","outside","furniture","levels","channel","letter","mode","phones","ideas","wednesday","structure","fund","summer","allow","degree","contract","button","releases","wed","homes","super","male","matter","custom","virginia","almost","took","located","multiple","asian","distribution","editor","inn","industrial","cause","potential","song","cnet","ltd","los","hp","focus","late","fall","featured","idea","rooms","female","responsible","inc","communications","win","associated","thomas","primary","cancer","numbers","reason","tool","browser","spring","foundation","answer","voice","eg","friendly","schedule","documents","communication","purpose","feature","bed","comes","police","everyone","independent","ip","approach","cameras","brown","physical","operating","hill","maps","medicine","deal","hold","ratings","chicago","forms","glass","happy","tue","smith","wanted","developed","thank","safe","unique","survey","prior","telephone","sport","ready","feed","animal","sources","mexico","population","pa","regular","secure","navigation","operations","therefore","ass","simply","evidence","station","christian","round","paypal","favorite","understand","option","master","valley","recently","probably","thu","rentals","sea","built","publications","blood","cut","worldwide","improve","connection","publisher","hall","larger","anti","networks","earth","parents","nokia","impact","transfer","introduction","kitchen","strong","tel","carolina","wedding","properties","hospital","ground","overview","ship","accommodation","owners","disease","tx","excellent","paid","italy","perfect","hair","opportunity","kit","classic","basis","command","cities","william","express","anal","award","distance","tree","peter","assessment","ensure","thus","wall","ie","involved","el","extra","especially","interface","pussy","partners","budget","rated","guides","success","maximum","ma","operation","existing","quite","selected","boy","amazon","patients","restaurants","beautiful","warning","wine","locations","horse","vote","forward","flowers","stars","significant","lists","technologies","owner","retail","animals","useful","directly","manufacturer","ways","est","son","providing","rule","mac","housing","takes","iii","gmt","bring","catalog","searches","max","trying","mother","authority","considered","told","xml","traffic","programme","joined","input","strategy","feet","agent","valid","bin","modern","senior","ireland","sexy","teaching","door","grand","testing","trial","charge","units","instead","canadian","cool","normal","wrote","enterprise","ships","entire","educational","md","leading","metal","positive","fl","fitness","chinese","opinion","mb","asia","football","abstract","uses","output","funds","mr","greater","likely","develop","employees","artists","alternative","processing","responsibility","resolution","java","guest","seems","publication","pass","relations","trust","van","contains","session","multi","photography","republic","fees","components","vacation","century","academic","assistance","completed","skin","graphics","indian","prev","ads","mary","il","expected","ring","grade","dating","pacific","mountain","organizations","pop","filter","mailing","vehicle","longer","consider","int","northern","behind","panel","floor","german","buying","match","proposed","default","require","iraq","boys","outdoor","deep","morning","otherwise","allows","rest","protein","plant","reported","hit","transportation","mm","pool","mini","politics","partner","disclaimer","authors","boards","faculty","parties","fish","membership","mission","eye","string","sense","modified","pack","released","stage","internal","goods","recommended","born","unless","richard","detailed","japanese","race","approved","background","target","except","character","usb","maintenance","ability","maybe","functions","ed","moving","brands","places","php","pretty","trademarks","phentermine","spain","southern","yourself","etc","winter","rape","battery","youth","pressure","submitted","boston","incest","debt","keywords","medium","television","interested","core","break","purposes","throughout","sets","dance","wood","msn","itself","defined","papers","playing","awards","fee","studio","reader","virtual","device","established","answers","rent","las","remote","dark","programming","external","apple","le","regarding","instructions","min","offered","theory","enjoy","remove","aid","surface","minimum","visual","host","variety","teachers","isbn","martin","manual","block","subjects","agents","increased","repair","fair","civil","steel","understanding","songs","fixed","wrong","beginning","hands","associates","finally","az","updates","desktop","classes","paris","ohio","gets","sector","capacity","requires","jersey","un","fat","fully","father","electric","saw","instruments","quotes","officer","driver","businesses","dead","respect","unknown","specified","restaurant","mike","trip","pst","worth","mi","procedures","poor","teacher","xxx","eyes","relationship","workers","farm","fucking","georgia","peace","traditional","campus","tom","showing","creative","coast","benefit","progress","funding","devices","lord","grant","sub","agree","fiction","hear","sometimes","watches","careers","beyond","goes","families","led","museum","themselves","fan","transport","interesting","blogs","wife","evaluation","accepted","former","implementation","ten","hits","zone","complex","th","cat","galleries","references","die","presented","jack","flat","flow","agencies","literature","respective","parent","spanish","michigan","columbia","setting","dr","scale","stand","economy","highest","helpful","monthly","critical","frame","musical","definition","secretary","angeles","networking","path","australian","employee","chief","gives","kb","bottom","magazines","packages","detail","francisco","laws","changed","pet","heard","begin","individuals","colorado","royal","clean","switch","russian","largest","african","guy","titles","relevant","guidelines","justice","connect","bible","dev","cup","basket","applied","weekly","vol","installation","described","demand","pp","suite","vegas","na","square","chris","attention","advance","skip","diet","army","auction","gear","lee","os","difference","allowed","correct","charles","nation","selling","lots","piece","sheet","firm","seven","older","illinois","regulations","elements","species","jump","cells","module","resort","facility","random","pricing","dvds","certificate","minister","motion","looks","fashion","directions","visitors","documentation","monitor","trading","forest","calls","whose","coverage","couple","giving","chance","vision","ball","ending","clients","actions","listen","discuss","accept","automotive","naked","goal","successful","sold","wind","communities","clinical","situation","sciences","markets","lowest","highly","publishing","appear","emergency","developing","lives","currency","leather","determine","milf","temperature","palm","announcements","patient","actual","historical","stone","bob","commerce","ringtones","perhaps","persons","difficult","scientific","satellite","fit","tests","village","accounts","amateur","ex","met","pain","xbox","particularly","factors","coffee","www","settings","cum","buyer","cultural","steve","easily","oral","ford","poster","edge","functional","root","au","fi","closed","holidays","ice","pink","zealand","balance","monitoring","graduate","replies","shot","nc","architecture","initial","label","thinking","scott","llc","sec","recommend","canon","hardcore","league","waste","minute","bus","provider","optional","dictionary","cold","accounting","manufacturing","sections","chair","fishing","effort","phase","fields","bag","fantasy","po","letters","motor","va","professor","context","install","shirt","apparel","generally","continued","foot","mass","crime","count","breast","techniques","ibm","rd","johnson","sc","quickly","dollars","websites","religion","claim","driving","permission","surgery","patch","heat","wild","measures","generation","kansas","miss","chemical","doctor","task","reduce","brought","himself","nor","component","enable","exercise","bug","santa","mid","guarantee","leader","diamond","israel","se","processes","soft","servers","alone","meetings","seconds","jones","arizona","keyword","interests","flight","congress","fuel","username","walk","fuck","produced","italian","paperback","classifieds","wait","supported","pocket","saint","rose","freedom","argument","competition","creating","jim","drugs","joint","premium","providers","fresh","characters","attorney","upgrade","di","factor","growing","thousands","km","stream","apartments","pick","hearing","eastern","auctions","therapy","entries","dates","generated","signed","upper","administrative","serious","prime","samsung","limit","began","louis","steps","errors","shops","bondage","del","efforts","informed","ga","ac","thoughts","creek","ft","worked","quantity","urban","practices","sorted","reporting","essential","myself","tours","platform","load","affiliate","labor","immediately","admin","nursing","defense","machines","designated","tags","heavy","covered","recovery","joe","guys","integrated","configuration","cock","merchant","comprehensive","expert","universal","protect","drop","solid","cds","presentation","languages","became","orange","compliance","vehicles","prevent","theme","rich","im","campaign","marine","improvement","vs","guitar","finding","pennsylvania","examples","ipod","saying","spirit","ar","claims","porno","challenge","motorola","acceptance","strategies","mo","seem","affairs","touch","intended","towards","sa","goals","hire","election","suggest","branch","charges","serve","affiliates","reasons","magic","mount","smart","talking","gave","ones","latin","multimedia","xp","tits","avoid","certified","manage","corner","rank","computing","oregon","element","birth","virus","abuse","interactive","requests","separate","quarter","procedure","leadership","tables","define","racing","religious","facts","breakfast","kong","column","plants","faith","chain","developer","identify","avenue","missing","died","approximately","domestic","sitemap","recommendations","moved","houston","reach","comparison","mental","viewed","moment","extended","sequence","inch","attack","sorry","centers","opening","damage","lab","reserve","recipes","cvs","gamma","plastic","produce","snow","placed","truth","counter","failure","follows","eu","weekend","dollar","camp","ontario","automatically","des","minnesota","films","bridge","native","fill","williams","movement","printing","baseball","owned","approval","draft","chart","played","contacts","cc","jesus","readers","clubs","lcd","wa","jackson","equal","adventure","matching","offering","shirts","profit","leaders","posters","institutions","assistant","variable","ave","dj","advertisement","expect","parking","headlines","yesterday","compared","determined","wholesale","workshop","russia","gone","codes","kinds","extension","seattle","statements","golden","completely","teams","fort","cm","wi","lighting","senate","forces","funny","brother","gene","turned","portable","tried","electrical","applicable","disc","returned","pattern","ct","hentai","boat","named","theatre","laser","earlier","manufacturers","sponsor","classical","icon","warranty","dedicated","indiana","direction","harry","basketball","objects","ends","delete","evening","assembly","nuclear","taxes","mouse","signal","criminal","issued","brain","sexual","wisconsin","powerful","dream","obtained","false","da","cast","flower","felt","personnel","passed","supplied","identified","falls","pic","soul","aids","opinions","promote","stated","stats","hawaii","professionals","appears","carry","flag","decided","nj","covers","hr","em","advantage","hello","designs","maintain","tourism","priority","newsletters","adults","clips","savings","iv","graphic","atom","payments","rw","estimated","binding","brief","ended","winning","eight","anonymous","iron","straight","script","served","wants","miscellaneous","prepared","void","dining","alert","integration","atlanta","dakota","tag","interview","mix","framework","disk","installed","queen","vhs","credits","clearly","fix","handle","sweet","desk","criteria","pubmed","dave","massachusetts","diego","hong","vice","associate","ne","truck","behavior","enlarge","ray","frequently","revenue","measure","changing","votes","du","duty","looked","discussions","bear","gain","festival","laboratory","ocean","flights","experts","signs","lack","depth","iowa","whatever","logged","laptop","vintage","train","exactly","dry","explore","maryland","spa","concept","nearly","eligible","checkout","reality","forgot","handling","origin","knew","gaming","feeds","billion","destination","scotland","faster","intelligence","dallas","bought","con","ups","nations","route","followed","specifications","broken","tripadvisor","frank","alaska","zoom","blow","battle","residential","anime","speak","decisions","industries","protocol","query","clip","partnership","editorial","nt","expression","es","equity","provisions","speech","wire","principles","suggestions","rural","shared","sounds","replacement","tape","strategic","judge","spam","economics","acid","bytes","cent","forced","compatible","fight","apartment","height","null","zero","speaker","filed","gb","netherlands","obtain","bc","consulting","recreation","offices","designer","remain","managed","pr","failed","marriage","roll","korea","banks","fr","participants","secret","bath","aa","kelly","leads","negative","austin","favorites","toronto","theater","springs","missouri","andrew","var","perform","healthy","translation","estimates","font","assets","injury","mt","joseph","ministry","drivers","lawyer","figures","married","protected","proposal","sharing","philadelphia","portal","waiting","birthday","beta","fail","gratis","banking","officials","brian","toward","won","slightly","assist","conduct","contained","lingerie","shemale","legislation","calling","parameters","jazz","serving","bags","profiles","miami","comics","matters","houses","doc","postal","relationships","tennessee","wear","controls","breaking","combined","ultimate","wales","representative","frequency","introduced","minor","finish","departments","residents","noted","displayed","mom","reduced","physics","rare","spent","performed","extreme","samples","davis","daniel","bars","reviewed","row","oz","forecast","removed","helps","singles","administrator","cycle","amounts","contain","accuracy","dual","rise","usd","sleep","mg","bird","pharmacy","brazil","creation","static","scene","hunter","addresses","lady","crystal","famous","writer","chairman","violence","fans","oklahoma","speakers","drink","academy","dynamic","gender","eat","permanent","agriculture","dell","cleaning","constitutes","portfolio","practical","delivered","collectibles","infrastructure","exclusive","seat","concerns","colour","vendor","originally","intel","utilities","philosophy","regulation","officers","reduction","aim","bids","referred","supports","nutrition","recording","regions","junior","toll","les","cape","ann","rings","meaning","tip","secondary","wonderful","mine","ladies","henry","ticket","announced","guess","agreed","prevention","whom","ski","soccer","math","import","posting","presence","instant","mentioned","automatic","healthcare","viewing","maintained","ch","increasing","majority","connected","christ","dan","dogs","sd","directors","aspects","austria","ahead","moon","participation","scheme","utility","preview","fly","manner","matrix","containing","combination","devel","amendment","despite","strength","guaranteed","turkey","libraries","proper","distributed","degrees","singapore","enterprises","delta","fear","seeking","inches","phoenix","rs","convention","shares","principal","daughter","standing","voyeur","comfort","colors","wars","cisco","ordering","kept","alpha","appeal","cruise","bonus","certification","previously","hey","bookmark","buildings","specials","beat","disney","household","batteries","adobe","smoking","bbc","becomes","drives","arms","alabama","tea","improved","trees","avg","achieve","positions","dress","subscription","dealer","contemporary","sky","utah","nearby","rom","carried","happen","exposure","panasonic","hide","permalink","signature","gambling","refer","miller","provision","outdoors","clothes","caused","luxury","babes","frames","viagra","certainly","indeed","newspaper","toy","circuit","layer","printed","slow","removal","easier","src","liability","trademark","hip","printers","faqs","nine","adding","kentucky","mostly","eric","spot","taylor","trackback","prints","spend","factory","interior","revised","grow","americans","optical","promotion","relative","amazing","clock","dot","hiv","identity","suites","conversion","feeling","hidden","reasonable","victoria","serial","relief","revision","broadband","influence","ratio","pda","importance","rain","onto","dsl","planet","webmaster","copies","recipe","zum","permit","seeing","proof","dna","diff","tennis","bass","prescription","bedroom","empty","instance","hole","pets","ride","licensed","orlando","specifically","tim","bureau","maine","sql","represent","conservation","pair","ideal","specs","recorded","don","pieces","finished","parks","dinner","lawyers","sydney","stress","cream","ss","runs","trends","yeah","discover","sexo","ap","patterns","boxes","louisiana","hills","javascript","fourth","nm","advisor","mn","marketplace","nd","evil","aware","wilson","shape","evolution","irish","certificates","objectives","stations","suggested","gps","op","remains","acc","greatest","firms","concerned","euro","operator","structures","generic","encyclopedia","usage","cap","ink","charts","continuing","mixed","census","interracial","peak","tn","competitive","exist","wheel","transit","dick","suppliers","salt","compact","poetry","lights","tracking","angel","bell","keeping","preparation","attempt","receiving","matches","accordance","width","noise","engines","forget","array","discussed","accurate","stephen","elizabeth","climate","reservations","pin","playstation","alcohol","greek","instruction","managing","annotation","sister","raw","differences","walking","explain","smaller","newest","establish","gnu","happened","expressed","jeff","extent","sharp","lesbians","ben","lane","paragraph","kill","mathematics","aol","compensation","ce","export","managers","aircraft","modules","sweden","conflict","conducted","versions","employer","occur","percentage","knows","mississippi","describe","concern","backup","requested","citizens","connecticut","heritage","personals","immediate","holding","trouble","spread","coach","kevin","agricultural","expand","supporting","audience","assigned","jordan","collections","ages","participate","plug","specialist","cook","affect","virgin","experienced","investigation","raised","hat","institution","directed","dealers","searching","sporting","helping","perl","affected","lib","bike","totally","plate","expenses","indicate","blonde","ab","proceedings","favourite","transmission","anderson","utc","characteristics","der","lose","organic","seek","experiences","albums","cheats","extremely","verzeichnis","contracts","guests","hosted","diseases","concerning","developers","equivalent","chemistry","tony","neighborhood","nevada","kits","thailand","variables","agenda","anyway","continues","tracks","advisory","cam","curriculum","logic","template","prince","circle","soil","grants","anywhere","psychology","responses","atlantic","wet","circumstances","edward","investor","identification","ram","leaving","wildlife","appliances","matt","elementary","cooking","speaking","sponsors","fox","unlimited","respond","sizes","plain","exit","entered","iran","arm","keys","launch","wave","checking","costa","belgium","printable","holy","acts","guidance","mesh","trail","enforcement","symbol","crafts","highway","buddy","hardcover","observed","dean","setup","poll","booking","glossary","fiscal","celebrity","styles","denver","unix","filled","bond","channels","ericsson","appendix","notify","blues","chocolate","pub","portion","scope","hampshire","supplier","cables","cotton","bluetooth","controlled","requirement","authorities","biology","dental","killed","border","ancient","debate","representatives","starts","pregnancy","causes","arkansas","biography","leisure","attractions","learned","transactions","notebook","explorer","historic","attached","opened","tm","husband","disabled","authorized","crazy","upcoming","britain","concert","retirement","scores","financing","efficiency","sp","comedy","adopted","efficient","weblog","linear","commitment","specialty","bears","jean","hop","carrier","edited","constant","visa","mouth","jewish","meter","linked","portland","interviews","concepts","nh","gun","reflect","pure","deliver","wonder","hell","lessons","fruit","begins","qualified","reform","lens","alerts","treated","discovery","draw","mysql","classified","relating","assume","confidence","alliance","fm","confirm","warm","neither","lewis","howard","offline","leaves","engineer","lifestyle","consistent","replace","clearance","connections","inventory","converter","suck","organisation","babe","checks","reached","becoming","blowjob","safari","objective","indicated","sugar","crew","legs","sam","stick","securities","allen","pdt","relation","enabled","genre","slide","montana","volunteer","tested","rear","democratic","enhance","switzerland","exact","bound","parameter","adapter","processor","node","formal","dimensions","contribute","lock","hockey","storm","micro","colleges","laptops","mile","showed","challenges","editors","mens","threads","bowl","supreme","brothers","recognition","presents","ref","tank","submission","dolls","estimate","encourage","navy","kid","regulatory","inspection","consumers","cancel","limits","territory","transaction","manchester","weapons","paint","delay","pilot","outlet","contributions","continuous","db","czech","resulting","cambridge","initiative","novel","pan","execution","disability","increases","ultra","winner","idaho","contractor","ph","episode","examination","potter","dish","plays","bulletin","ia","pt","indicates","modify","oxford","adam","truly","epinions","painting","committed","extensive","affordable","universe","candidate","databases","patent","slot","psp","outstanding","ha","eating","perspective","planned","watching","lodge","messenger","mirror","tournament","consideration","ds","discounts","sterling","sessions","kernel","boobs","stocks","buyers","journals","gray","catalogue","ea","jennifer","antonio","charged","broad","taiwan","und","chosen","demo","greece","lg","swiss","sarah","clark","labour","hate","terminal","publishers","nights","behalf","caribbean","liquid","rice","nebraska","loop","salary","reservation","foods","gourmet","guard","properly","orleans","saving","nfl","remaining","empire","resume","twenty","newly","raise","prepare","avatar","gary","depending","illegal","expansion","vary","hundreds","rome","arab","lincoln","helped","premier","tomorrow","purchased","milk","decide","consent","drama","visiting","performing","downtown","keyboard","contest","collected","nw","bands","boot","suitable","ff","absolutely","millions","lunch","dildo","audit","push","chamber","guinea","findings","muscle","featuring","iso","implement","clicking","scheduled","polls","typical","tower","yours","sum","misc","calculator","significantly","chicken","temporary","attend","shower","alan","sending","jason","tonight","dear","sufficient","holdem","shell","province","catholic","oak","vat","awareness","vancouver","governor","beer","seemed","contribution","measurement","swimming","spyware","formula","constitution","packaging","solar","jose","catch","jane","pakistan","ps","reliable","consultation","northwest","sir","doubt","earn","finder","unable","periods","classroom","tasks","democracy","attacks","kim","wallpaper","merchandise","const","resistance","doors","symptoms","resorts","biggest","memorial","visitor","twin","forth","insert","baltimore","gateway","ky","dont","alumni","drawing","candidates","charlotte","ordered","biological","fighting","transition","happens","preferences","spy","romance","instrument","bruce","split","themes","powers","heaven","br","bits","pregnant","twice","classification","focused","egypt","physician","hollywood","bargain","wikipedia","cellular","norway","vermont","asking","blocks","normally","lo","spiritual","hunting","diabetes","suit","ml","shift","chip","res","sit","bodies","photographs","cutting","wow","simon","writers","marks","flexible","loved","favourites","mapping","numerous","relatively","birds","satisfaction","represents","char","indexed","pittsburgh","superior","preferred","saved","paying","cartoon","shots","intellectual","moore","granted","choices","carbon","spending","comfortable","magnetic","interaction","listening","effectively","registry","crisis","outlook","massive","denmark","employed","bright","treat","header","cs","poverty","formed","piano","echo","que","grid","sheets","patrick","experimental","puerto","revolution","consolidation","displays","plasma","allowing","earnings","voip","mystery","landscape","dependent","mechanical","journey","delaware","bidding","consultants","risks","banner","applicant","charter","fig","barbara","cooperation","counties","acquisition","ports","implemented","sf","directories","recognized","dreams","blogger","notification","kg","licensing","stands","teach","occurred","textbooks","rapid","pull","hairy","diversity","cleveland","ut","reverse","deposit","seminar","investments","latina","nasa","wheels","sexcam","specify","accessibility","dutch","sensitive","templates","formats","tab","depends","boots","holds","router","concrete","si","editing","poland","folder","womens","css","completion","upload","pulse","universities","technique","contractors","milfhunter","voting","courts","notices","subscriptions","calculate","mc","detroit","alexander","broadcast","converted","metro","toshiba","anniversary","improvements","strip","specification","pearl","accident","nick","accessible","accessory","resident","plot","qty","possibly","airline","typically","representation","regard","pump","exists","arrangements","smooth","conferences","uniprotkb","beastiality","strike","consumption","birmingham","flashing","lp","narrow","afternoon","threat","surveys","sitting","putting","consultant","controller","ownership","committees","penis","legislative","researchers","vietnam","trailer","anne","castle","gardens","missed","malaysia","unsubscribe","antique","labels","willing","bio","molecular","upskirt","acting","heads","stored","exam","logos","residence","attorneys","milfs","antiques","density","hundred","ryan","operators","strange","sustainable","philippines","statistical","beds","breasts","mention","innovation","pcs","employers","grey","parallel","honda","amended","operate","bills","bold","bathroom","stable","opera","definitions","von","doctors","lesson","cinema","asset","ag","scan","elections","drinking","blowjobs","reaction","blank","enhanced","entitled","severe","generate","stainless","newspapers","hospitals","vi","deluxe","humor","aged","monitors","exception","lived","duration","bulk","successfully","indonesia","pursuant","sci","fabric","edt","visits","primarily","tight","domains","capabilities","pmid","contrast","recommendation","flying","recruitment","sin","berlin","cute","organized","ba","para","siemens","adoption","improving","cr","expensive","meant","capture","pounds","buffalo","organisations","plane","pg","explained","seed","programmes","desire","expertise","mechanism","camping","ee","jewellery","meets","welfare","peer","caught","eventually","marked","driven","measured","medline","bottle","agreements","considering","innovative","marshall","massage","rubber","conclusion","closing","tampa","thousand","meat","legend","grace","susan","ing","ks","adams","python","monster","alex","bang","villa","bone","columns","disorders","bugs","collaboration","hamilton","detection","ftp","cookies","inner","formation","tutorial","med","engineers","entity","cruises","gate","holder","proposals","moderator","sw","tutorials","settlement","portugal","lawrence","roman","duties","valuable","erotic","tone","collectables","ethics","forever","dragon","busy","captain","fantastic","imagine","brings","heating","leg","neck","hd","wing","governments","purchasing","scripts","abc","stereo","appointed","taste","dealing","commit","tiny","operational","rail","airlines","liberal","livecam","jay","trips","gap","sides","tube","turns","corresponding","descriptions","cache","belt","jacket","determination","animation","oracle","er","matthew","lease","productions","aviation","hobbies","proud","excess","disaster","console","commands","jr","telecommunications","instructor","giant","achieved","injuries","shipped","bestiality","seats","approaches","biz","alarm","voltage","anthony","nintendo","usual","loading","stamps","appeared","franklin","angle","rob","vinyl","highlights","mining","designers","melbourne","ongoing","worst","imaging","betting","scientists","liberty","wyoming","blackjack","argentina","era","convert","possibility","analyst","commissioner","dangerous","garage","exciting","reliability","thongs","gcc","unfortunately","respectively","volunteers","attachment","ringtone","finland","morgan","derived","pleasure","honor","asp","oriented","eagle","desktops","pants","columbus","nurse","prayer","appointment","workshops","hurricane","quiet","luck","postage","producer","represented","mortgages","dial","responsibilities","cheese","comic","carefully","jet","productivity","investors","crown","par","underground","diagnosis","maker","crack","principle","picks","vacations","gang","semester","calculated","cumshot","fetish","applies","casinos","appearance","smoke","apache","filters","incorporated","nv","craft","cake","notebooks","apart","fellow","blind","lounge","mad","algorithm","semi","coins","andy","gross","strongly","cafe","valentine","hilton","ken","proteins","horror","su","exp","familiar","capable","douglas","debian","till","involving","pen","investing","christopher","admission","epson","shoe","elected","carrying","victory","sand","madison","terrorism","joy","editions","cpu","mainly","ethnic","ran","parliament","actor","finds","seal","situations","fifth","allocated","citizen","vertical","corrections","structural","municipal","describes","prize","sr","occurs","jon","absolute","disabilities","consists","anytime","substance","prohibited","addressed","lies","pipe","soldiers","nr","guardian","lecture","simulation","layout","initiatives","ill","concentration","classics","lbs","lay","interpretation","horses","lol","dirty","deck","wayne","donate","taught","bankruptcy","mp","worker","optimization","alive","temple","substances","prove","discovered","wings","breaks","genetic","restrictions","participating","waters","promise","thin","exhibition","prefer","ridge","cabinet","modem","harris","mph","bringing","sick","dose","evaluate","tiffany","tropical","collect","bet","composition","toyota","streets","nationwide","vector","definitely","shaved","turning","buffer","purple","existence","commentary","larry","limousines","developments","def","immigration","destinations","lets","mutual","pipeline","necessarily","syntax","li","attribute","prison","skill","chairs","nl","everyday","apparently","surrounding","mountains","moves","popularity","inquiry","ethernet","checked","exhibit","throw","trend","sierra","visible","cats","desert","postposted","ya","oldest","rhode","nba","busty","coordinator","obviously","mercury","steven","handbook","greg","navigate","worse","summit","victims","epa","spaces","fundamental","burning","escape","coupons","somewhat","receiver","substantial","tr","progressive","cialis","bb","boats","glance","scottish","championship","arcade","richmond","sacramento","impossible","ron","russell","tells","obvious","fiber","depression","graph","covering","platinum","judgment","bedrooms","talks","filing","foster","modeling","passing","awarded","testimonials","trials","tissue","nz","memorabilia","clinton","masters","bonds","cartridge","alberta","explanation","folk","org","commons","cincinnati","subsection","fraud","electricity","permitted","spectrum","arrival","okay","pottery","emphasis","roger","aspect","workplace","awesome","mexican","confirmed","counts","priced","wallpapers","hist","crash","lift","desired","inter","closer","assumes","heights","shadow","riding","infection","firefox","lisa","expense","grove","eligibility","venture","clinic","korean","healing","princess","mall","entering","packet","spray","studios","involvement","dad","buttons","placement","observations","vbulletin","funded","thompson","winners","extend","roads","subsequent","pat","dublin","rolling","fell","motorcycle","yard","disclosure","establishment","memories","nelson","te","arrived","creates","faces","tourist","cocks","av","mayor","murder","sean","adequate","senator","yield","presentations","grades","cartoons","pour","digest","reg","lodging","tion","dust","hence","wiki","entirely","replaced","radar","rescue","undergraduate","losses","combat","reducing","stopped","occupation","lakes","butt","donations","associations","citysearch","closely","radiation","diary","seriously","kings","shooting","kent","adds","nsw","ear","flags","pci","baker","launched","elsewhere","pollution","conservative","guestbook","shock","effectiveness","walls","abroad","ebony","tie","ward","drawn","arthur","ian","visited","roof","walker","demonstrate","atmosphere","suggests","kiss","beast","ra","operated","experiment","targets","overseas","purchases","dodge","counsel","federation","pizza","invited","yards","assignment","chemicals","gordon","mod","farmers","rc","queries","bmw","rush","ukraine","absence","nearest","cluster","vendors","mpeg","whereas","yoga","serves","woods","surprise","lamp","rico","partial","shoppers","phil","everybody","couples","nashville","ranking","jokes","cst","http","ceo","simpson","twiki","sublime","counseling","palace","acceptable","satisfied","glad","wins","measurements","verify","globe","trusted","copper","milwaukee","rack","medication","warehouse","shareware","ec","rep","dicke","kerry","receipt","supposed","ordinary","nobody","ghost","violation","configure","stability","mit","applying","southwest","boss","pride","institutional","expectations","independence","knowing","reporter","metabolism","keith","champion","cloudy","linda","ross","personally","chile","anna","plenty","solo","sentence","throat","ignore","maria","uniform","excellence","wealth","tall","rm","somewhere","vacuum","dancing","attributes","recognize","brass","writes","plaza","pdas","outcomes","survival","quest","publish","sri","screening","toe","thumbnail","trans","jonathan","whenever","nova","lifetime","api","pioneer","booty","forgotten","acrobat","plates","acres","venue","athletic","thermal","essays","behaviour","vital","telling","fairly","coastal","config","cf","charity","intelligent","edinburgh","vt","excel","modes","obligation","campbell","wake","stupid","harbor","hungary","traveler","urw","segment","realize","regardless","lan","enemy","puzzle","rising","aluminum","wells","wishlist","opens","insight","sms","shit","restricted","republican","secrets","lucky","latter","merchants","thick","trailers","repeat","syndrome","philips","attendance","penalty","drum","glasses","enables","nec","iraqi","builder","vista","jessica","chips","terry","flood","foto","ease","arguments","amsterdam","orgy","arena","adventures","pupils","stewart","announcement","tabs","outcome","xx","appreciate","expanded","casual","grown","polish","lovely","extras","gm","centres","jerry","clause","smile","lands","ri","troops","indoor","bulgaria","armed","broker","charger","regularly","believed","pine","cooling","tend","gulf","rt","rick","trucks","cp","mechanisms","divorce","laura","shopper","tokyo","partly","nikon","customize","tradition","candy","pills","tiger","donald","folks","sensor","exposed","telecom","hunt","angels","deputy","indicators","sealed","thai","emissions","physicians","loaded","fred","complaint","scenes","experiments","balls","afghanistan","dd","boost","spanking","scholarship","governance","mill","founded","supplements","chronic","icons","tranny","moral","den","catering","aud","finger","keeps","pound","locate","camcorder","pl","trained","burn","implementing","roses","labs","ourselves","bread","tobacco","wooden","motors","tough","roberts","incident","gonna","dynamics","lie","crm","rf","conversation","decrease","cumshots","chest","pension","billy","revenues","emerging","worship","bukkake","capability","ak","fe","craig","herself","producing","churches","precision","damages","reserves","contributed","solve","shorts","reproduction","minority","td","diverse","amp","ingredients","sb","ah","johnny","sole","franchise","recorder","complaints","facing","sm","nancy","promotions","tones","passion","rehabilitation","maintaining","sight","laid","clay","defence","patches","weak","refund","usc","towns","environments","trembl","divided","blvd","reception","amd","wise","emails","cyprus","wv","odds","correctly","insider","seminars","consequences","makers","hearts","geography","appearing","integrity","worry","ns","discrimination","eve","carter","legacy","marc","pleased","danger","vitamin","widely","processed","phrase","genuine","raising","implications","functionality","paradise","hybrid","reads","roles","intermediate","emotional","sons","leaf","pad","glory","platforms","ja","bigger","billing","diesel","versus","combine","overnight","geographic","exceed","bs","rod","saudi","fault","cuba","hrs","preliminary","districts","introduce","silk","promotional","kate","chevrolet","babies","bi","karen","compiled","romantic","revealed","specialists","generator","albert","examine","jimmy","graham","suspension","bristol","margaret","compaq","sad","correction","wolf","slowly","authentication","communicate","rugby","supplement","showtimes","cal","portions","infant","promoting","sectors","samuel","fluid","grounds","fits","kick","regards","meal","ta","hurt","machinery","bandwidth","unlike","equation","baskets","probability","pot","dimension","wright","img","barry","proven","schedules","admissions","cached","warren","slip","studied","reviewer","involves","quarterly","rpm","profits","devil","grass","comply","marie","florist","illustrated","cherry","continental","alternate","deutsch","achievement","limitations","kenya","webcam","cuts","funeral","nutten","earrings","enjoyed","automated","chapters","pee","charlie","quebec","nipples","passenger","convenient","dennis","mars","francis","tvs","sized","manga","noticed","socket","silent","literary","egg","mhz","signals","caps","orientation","pill","theft","childhood","swing","symbols","lat","meta","humans","analog","facial","choosing","talent","dated","flexibility","seeker","wisdom","shoot","boundary","mint","packard","offset","payday","philip","elite","gi","spin","holders","believes","swedish","poems","deadline","jurisdiction","robot","displaying","witness","collins","equipped","stages","encouraged","sur","winds","powder","broadway","acquired","assess","wash","cartridges","stones","entrance","gnome","roots","declaration","losing","attempts","gadgets","noble","glasgow","automation","impacts","rev","gospel","advantages","shore","loves","induced","ll","knight","preparing","loose","aims","recipient","linking","extensions","appeals","cl","earned","illness","islamic","athletics","southeast","ieee","ho","alternatives","pending","parker","determining","lebanon","corp","personalized","kennedy","gt","sh","conditioning","teenage","soap","ae","triple","cooper","nyc","vincent","jam","secured","unusual","answered","partnerships","destruction","slots","increasingly","migration","disorder","routine","toolbar","basically","rocks","conventional","titans","applicants","wearing","axis","sought","genes","mounted","habitat","firewall","median","guns","scanner","herein","occupational","animated","horny","judicial","rio","hs","adjustment","hero","integer","treatments","bachelor","attitude","camcorders","engaged","falling","basics","montreal","carpet","rv","struct","lenses","binary","genetics","attended","difficulty","punk","collective","coalition","pi","dropped","enrollment","duke","walter","ai","pace","besides","wage","producers","ot","collector","arc","hosts","interfaces","advertisers","moments","atlas","strings","dawn","representing","observation","feels","torture","carl","deleted","coat","mitchell","mrs","rica","restoration","convenience","returning","ralph","opposition","container","yr","defendant","warner","confirmation","app","embedded","inkjet","supervisor","wizard","corps","actors","liver","peripherals","liable","brochure","morris","bestsellers","petition","eminem","recall","antenna","picked","assumed","departure","minneapolis","belief","killing","bikini","memphis","shoulder","decor","lookup","texts","harvard","brokers","roy","ion","diameter","ottawa","doll","ic","podcast","tit","seasons","peru","interactions","refine","bidder","singer","evans","herald","literacy","fails","aging","nike","intervention","pissing","fed","plugin","attraction","diving","invite","modification","alice","latinas","suppose","customized","reed","involve","moderate","terror","younger","thirty","mice","opposite","understood","rapidly","dealtime","ban","temp","intro","mercedes","zus","assurance","fisting","clerk","happening","vast","mills","outline","amendments","tramadol","holland","receives","jeans","metropolitan","compilation","verification","fonts","ent","odd","wrap","refers","mood","favor","veterans","quiz","mx","sigma","gr","attractive","xhtml","occasion","recordings","jefferson","victim","demands","sleeping","careful","ext","beam","gardening","obligations","arrive","orchestra","sunset","tracked","moreover","minimal","polyphonic","lottery","tops","framed","aside","outsourcing","licence","adjustable","allocation","michelle","essay","discipline","amy","ts","demonstrated","dialogue","identifying","alphabetical","camps","declared","dispatched","aaron","handheld","trace","disposal","shut","florists","packs","ge","installing","switches","romania","voluntary","ncaa","thou","consult","phd","greatly","blogging","mask","cycling","midnight","ng","commonly","pe","photographer","inform","turkish","coal","cry","messaging","pentium","quantum","murray","intent","tt","zoo","largely","pleasant","announce","constructed","additions","requiring","spoke","aka","arrow","engagement","sampling","rough","weird","tee","refinance","lion","inspired","holes","weddings","blade","suddenly","oxygen","cookie","meals","canyon","goto","meters","merely","calendars","arrangement","conclusions","passes","bibliography","pointer","compatibility","stretch","durham","furthermore","permits","cooperative","muslim","xl","neil","sleeve","netscape","cleaner","cricket","beef","feeding","stroke","township","rankings","measuring","cad","hats","robin","robinson","jacksonville","strap","headquarters","sharon","crowd","tcp","transfers","surf","olympic","transformation","remained","attachments","dv","dir","entities","customs","administrators","personality","rainbow","hook","roulette","decline","gloves","israeli","medicare","cord","skiing","cloud","facilitate","subscriber","valve","val","hewlett","explains","proceed","flickr","feelings","knife","jamaica","priorities","shelf","bookstore","timing","liked","parenting","adopt","denied","fotos","incredible","britney","freeware","fucked","donation","outer","crop","deaths","rivers","commonwealth","pharmaceutical","manhattan","tales","katrina","workforce","islam","nodes","tu","fy","thumbs","seeds","cited","lite","ghz","hub","targeted","organizational","skype","realized","twelve","founder","decade","gamecube","rr","dispute","portuguese","tired","titten","adverse","everywhere","excerpt","eng","steam","discharge","ef","drinks","ace","voices","acute","halloween","climbing","stood","sing","tons","perfume","carol","honest","albany","hazardous","restore","stack","methodology","somebody","sue","ep","housewares","reputation","resistant","democrats","recycling","hang","gbp","curve","creator","amber","qualifications","museums","coding","slideshow","tracker","variation","passage","transferred","trunk","hiking","lb","damn","pierre","jelsoft","headset","photograph","oakland","colombia","waves","camel","distributor","lamps","underlying","hood","wrestling","suicide","archived","photoshop","jp","chi","bt","arabia","gathering","projection","juice","chase","mathematical","logical","sauce","fame","extract","specialized","diagnostic","panama","indianapolis","af","payable","corporations","courtesy","criticism","automobile","confidential","rfc","statutory","accommodations","athens","northeast","downloaded","judges","sl","seo","retired","isp","remarks","detected","decades","paintings","walked","arising","nissan","bracelet","ins","eggs","juvenile","injection","yorkshire","populations","protective","afraid","acoustic","railway","cassette","initially","indicator","pointed","hb","jpg","causing","mistake","norton","locked","eliminate","tc","fusion","mineral","sunglasses","ruby","steering","beads","fortune","preference","canvas","threshold","parish","claimed","screens","cemetery","planner","croatia","flows","stadium","venezuela","exploration","mins","fewer","sequences","coupon","nurses","ssl","stem","proxy","gangbang","astronomy","lanka","opt","edwards","drew","contests","flu","translate","announces","mlb","costume","tagged","berkeley","voted","killer","bikes","gates","adjusted","rap","tune","bishop","pulled","corn","gp","shaped","compression","seasonal","establishing","farmer","counters","puts","constitutional","grew","perfectly","tin","slave","instantly","cultures","norfolk","coaching","examined","trek","encoding","litigation","submissions","oem","heroes","painted","lycos","ir","zdnet","broadcasting","horizontal","artwork","cosmetic","resulted","portrait","terrorist","informational","ethical","carriers","ecommerce","mobility","floral","builders","ties","struggle","schemes","suffering","neutral","fisher","rat","spears","prospective","dildos","bedding","ultimately","joining","heading","equally","artificial","bearing","spectacular","coordination","connector","brad","combo","seniors","worlds","guilty","affiliated","activation","naturally","haven","tablet","jury","dos","tail","subscribers","charm","lawn","violent","mitsubishi","underwear","basin","soup","potentially","ranch","constraints","crossing","inclusive","dimensional","cottage","drunk","considerable","crimes","resolved","mozilla","byte","toner","nose","latex","branches","anymore","oclc","delhi","holdings","alien","locator","selecting","processors","pantyhose","plc","broke","nepal","zimbabwe","difficulties","juan","complexity","msg","constantly","browsing","resolve","barcelona","presidential","documentary","cod","territories","melissa","moscow","thesis","thru","jews","nylon","palestinian","discs","rocky","bargains","frequent","trim","nigeria","ceiling","pixels","ensuring","hispanic","cv","cb","legislature","hospitality","gen","anybody","procurement","diamonds","espn","fleet","untitled","bunch","totals","marriott","singing","theoretical","afford","exercises","starring","referral","nhl","surveillance","optimal","quit","distinct","protocols","lung","highlight","substitute","inclusion","hopefully","brilliant","turner","sucking","cents","reuters","ti","fc","gel","todd","spoken","omega","evaluated","stayed","civic","assignments","fw","manuals","doug","sees","termination","watched","saver","thereof","grill","households","gs","redeem","rogers","grain","aaa","authentic","regime","wanna","wishes","bull","montgomery","architectural","louisville","depend","differ","macintosh","movements","ranging","monica","repairs","breath","amenities","virtually","cole","mart","candle","hanging","colored","authorization","tale","verified","lynn","formerly","projector","bp","situated","comparative","std","seeks","herbal","loving","strictly","routing","docs","stanley","psychological","surprised","retailer","vitamins","elegant","gains","renewal","vid","genealogy","opposed","deemed","scoring","expenditure","panties","brooklyn","liverpool","sisters","critics","connectivity","spots","oo","algorithms","hacker","madrid","similarly","margin","coin","bbw","solely","fake","salon","collaborative","norman","fda","excluding","turbo","headed","voters","cure","madonna","commander","arch","ni","murphy","thinks","thats","suggestion","hdtv","soldier","phillips","asin","aimed","justin","bomb","harm","interval","mirrors","spotlight","tricks","reset","brush","investigate","thy","expansys","panels","repeated","assault","connecting","spare","logistics","deer","kodak","tongue","bowling","tri","danish","pal","monkey","proportion","filename","skirt","florence","invest","honey","um","analyses","drawings","significance","scenario","ye","fs","lovers","atomic","approx","symposium","arabic","gauge","essentials","junction","protecting","nn","faced","mat","rachel","solving","transmitted","weekends","screenshots","produces","oven","ted","intensive","chains","kingston","sixth","engage","deviant","noon","switching","quoted","adapters","correspondence","farms","imports","supervision","cheat","bronze","expenditures","sandy","separation","testimony","suspect","celebrities","macro","sender","mandatory","boundaries","crucial","syndication","gym","celebration","kde","adjacent","filtering","tuition","spouse","exotic","viewer","signup","threats","luxembourg","puzzles","reaching","vb","damaged","cams","receptor","piss","laugh","joel","surgical","destroy","citation","pitch","autos","yo","premises","perry","proved","offensive","imperial","dozen","benjamin","deployment","teeth","cloth","studying","colleagues","stamp","lotus","salmon","olympus","separated","proc","cargo","tan","directive","fx","salem","mate","dl","starter","upgrades","likes","butter","pepper","weapon","luggage","burden","chef","tapes","zones","races","isle","stylish","slim","maple","luke","grocery","offshore","governing","retailers","depot","kenneth","comp","alt","pie","blend","harrison","ls","julie","occasionally","cbs","attending","emission","pete","spec","finest","realty","janet","bow","penn","recruiting","apparent","instructional","phpbb","autumn","traveling","probe","midi","permissions","biotechnology","toilet","ranked","jackets","routes","packed","excited","outreach","helen","mounting","recover","tied","lopez","balanced","prescribed","catherine","timely","talked","upskirts","debug","delayed","chuck","reproduced","hon","dale","explicit","calculation","villas","ebook","consolidated","boob","exclude","peeing","occasions","brooks","equations","newton","oils","sept","exceptional","anxiety","bingo","whilst","spatial","respondents","unto","lt","ceramic","prompt","precious","minds","annually","considerations","scanners","atm","xanax","eq","pays","cox","fingers","sunny","ebooks","delivers","je","queensland","necklace","musicians","leeds","composite","unavailable","cedar","arranged","lang","theaters","advocacy","raleigh","stud","fold","essentially","designing","threaded","uv","qualify","fingering","blair","hopes","assessments","cms","mason","diagram","burns","pumps","slut","ejaculation","footwear","sg","vic","beijing","peoples","victor","mario","pos","attach","licenses","utils","removing","advised","brunswick","spider","phys","ranges","pairs","sensitivity","trails","preservation","hudson","isolated","calgary","interim","assisted","divine","streaming","approve","chose","compound","intensity","technological","syndicate","abortion","dialog","venues","blast","wellness","calcium","newport","antivirus","addressing","pole","discounted","indians","shield","harvest","membrane","prague","previews","bangladesh","constitute","locally","concluded","pickup","desperate","mothers","nascar","iceland","demonstration","governmental","manufactured","candles","graduation","mega","bend","sailing","variations","moms","sacred","addiction","morocco","chrome","tommy","springfield","refused","brake","exterior","greeting","ecology","oliver","congo","glen","botswana","nav","delays","synthesis","olive","undefined","unemployment","cyber","verizon","scored","enhancement","newcastle","clone","dicks","velocity","lambda","relay","composed","tears","performances","oasis","baseline","cab","angry","fa","societies","silicon","brazilian","identical","petroleum","compete","ist","norwegian","lover","belong","honolulu","beatles","lips","escort","retention","exchanges","pond","rolls","thomson","barnes","soundtrack","wondering","malta","daddy","lc","ferry","rabbit","profession","seating","dam","cnn","separately","physiology","lil","collecting","das","exports","omaha","tire","participant","scholarships","recreational","dominican","chad","electron","loads","friendship","heather","passport","motel","unions","treasury","warrant","sys","solaris","frozen","occupied","josh","royalty","scales","rally","observer","sunshine","strain","drag","ceremony","somehow","arrested","expanding","provincial","investigations","icq","ripe","yamaha","rely","medications","hebrew","gained","rochester","dying","laundry","stuck","solomon","placing","stops","homework","adjust","assessed","advertiser","enabling","encryption","filling","downloadable","sophisticated","imposed","silence","scsi","focuses","soviet","possession","cu","laboratories","treaty","vocal","trainer","organ","stronger","volumes","advances","vegetables","lemon","toxic","dns","thumbnails","darkness","pty","ws","nuts","nail","bizrate","vienna","implied","span","stanford","sox","stockings","joke","respondent","packing","statute","rejected","satisfy","destroyed","shelter","chapel","gamespot","manufacture","layers","wordpress","guided","vulnerability","accountability","celebrate","accredited","appliance","compressed","bahamas","powell","mixture","zoophilia","bench","univ","tub","rider","scheduling","radius","perspectives","mortality","logging","hampton","christians","borders","therapeutic","pads","butts","inns","bobby","impressive","sheep","accordingly","architect","railroad","lectures","challenging","wines","nursery","harder","cups","ash","microwave","cheapest","accidents","travesti","relocation","stuart","contributors","salvador","ali","salad","np","monroe","tender","violations","foam","temperatures","paste","clouds","competitions","discretion","tft","tanzania","preserve","jvc","poem","vibrator","unsigned","staying","cosmetics","easter","theories","repository","praise","jeremy","venice","jo","concentrations","vibrators","estonia","christianity","veteran","streams","landing","signing","executed","katie","negotiations","realistic","dt","cgi","showcase","integral","asks","relax","namibia","generating","christina","congressional","synopsis","hardly","prairie","reunion","composer","bean","sword","absent","photographic","sells","ecuador","hoping","accessed","spirits","modifications","coral","pixel","float","colin","bias","imported","paths","bubble","por","acquire","contrary","millennium","tribune","vessel","acids","focusing","viruses","cheaper","admitted","dairy","admit","mem","fancy","equality","samoa","gc","achieving","tap","stickers","fisheries","exceptions","reactions","leasing","lauren","beliefs","ci","macromedia","companion","squad","analyze","ashley","scroll","relate","divisions","swim","wages","additionally","suffer","forests","fellowship","nano","invalid","concerts","martial","males","victorian","retain","colours","execute","tunnel","genres","cambodia","patents","copyrights","yn","chaos","lithuania","mastercard","wheat","chronicles","obtaining","beaver","updating","distribute","readings","decorative","kijiji","confused","compiler","enlargement","eagles","bases","vii","accused","bee","campaigns","unity","loud","conjunction","bride","rats","defines","airports","instances","indigenous","begun","cfr","brunette","packets","anchor","socks","validation","parade","corruption","stat","trigger","incentives","cholesterol","gathered","essex","slovenia","notified","differential","beaches","folders","dramatic","surfaces","terrible","routers","cruz","pendant","dresses","baptist","scientist","starsmerchant","hiring","clocks","arthritis","bios","females","wallace","nevertheless","reflects","taxation","fever","pmc","cuisine","surely","practitioners","transcript","myspace","theorem","inflation","thee","nb","ruth","pray","stylus","compounds","pope","drums","contracting","topless","arnold","structured","reasonably","jeep","chicks","bare","hung","cattle","mba","radical","graduates","rover","recommends","controlling","treasure","reload","distributors","flame","levitra","tanks","assuming","monetary","elderly","pit","arlington","mono","particles","floating","extraordinary","tile","indicating","bolivia","spell","hottest","stevens","coordinate","kuwait","exclusively","emily","alleged","limitation","widescreen","compile","squirting","webster","struck","rx","illustration","plymouth","warnings","construct","apps","inquiries","bridal","annex","mag","gsm","inspiration","tribal","curious","affecting","freight","rebate","meetup","eclipse","sudan","ddr","downloading","rec","shuttle","aggregate","stunning","cycles","affects","forecasts","detect","sluts","actively","ciao","ampland","knee","prep","pb","complicated","chem","fastest","butler","shopzilla","injured","decorating","payroll","cookbook","expressions","ton","courier","uploaded","shakespeare","hints","collapse","americas","connectors","twinks","unlikely","oe","gif","pros","conflicts","techno","beverage","tribute","wired","elvis","immune","latvia","travelers","forestry","barriers","cant","jd","rarely","gpl","infected","offerings","martha","genesis","barrier","argue","incorrect","trains","metals","bicycle","furnishings","letting","arise","guatemala","celtic","thereby","irc","jamie","particle","perception","minerals","advise","humidity","bottles","boxing","wy","dm","bangkok","renaissance","pathology","sara","bra","ordinance","hughes","photographers","bitch","infections","jeffrey","chess","operates","brisbane","configured","survive","oscar","festivals","menus","joan","possibilities","duck","reveal","canal","amino","phi","contributing","herbs","clinics","mls","cow","manitoba","analytical","missions","watson","lying","costumes","strict","dive","saddam","circulation","drill","offense","threesome","bryan","cet","protest","handjob","assumption","jerusalem","hobby","tries","transexuales","invention","nickname","fiji","technician","inline","executives","enquiries","washing","audi","staffing","cognitive","exploring","trick","enquiry","closure","raid","ppc","timber","volt","intense","div","playlist","registrar","showers","supporters","ruling","steady","dirt","statutes","withdrawal","myers","drops","predicted","wider","saskatchewan","jc","cancellation","plugins","enrolled","sensors","screw","ministers","publicly","hourly","blame","geneva","freebsd","veterinary","acer","prostores","reseller","dist","handed","suffered","intake","informal","relevance","incentive","butterfly","tucson","mechanics","heavily","swingers","fifty","headers","mistakes","numerical","ons","geek","uncle","defining","xnxx","counting","reflection","sink","accompanied","assure","invitation","devoted","princeton","jacob","sodium","randy","spirituality","hormone","meanwhile","proprietary","timothy","childrens","brick","grip","naval","thumbzilla","medieval","porcelain","avi","bridges","pichunter","captured","watt","thehun","decent","casting","dayton","translated","shortly","cameron","columnists","pins","carlos","reno","donna","andreas","warrior","diploma","cabin","innocent","bdsm","scanning","ide","consensus","polo","valium","copying","rpg","delivering","cordless","patricia","horn","eddie","uganda","fired","journalism","pd","prot","trivia","adidas","perth","frog","grammar","intention","syria","disagree","klein","harvey","tires","logs","undertaken","tgp","hazard","retro","leo","livesex","statewide","semiconductor","gregory","episodes","boolean","circular","anger","diy","mainland","illustrations","suits","chances","interact","snap","happiness","arg","substantially","bizarre","glenn","ur","auckland","olympics","fruits","identifier","geo","worldsex","ribbon","calculations","doe","jpeg","conducting","startup","suzuki","trinidad","ati","kissing","wal","handy","swap","exempt","crops","reduces","accomplished","calculators","geometry","impression","abs","slovakia","flip","guild","correlation","gorgeous","capitol","sim","dishes","rna","barbados","chrysler","nervous","refuse","extends","fragrance","mcdonald","replica","plumbing","brussels","tribe","neighbors","trades","superb","buzz","transparent","nuke","rid","trinity","charleston","handled","legends","boom","calm","champions","floors","selections","projectors","inappropriate","exhaust","comparing","shanghai","speaks","burton","vocational","davidson","copied","scotia","farming","gibson","pharmacies","fork","troy","ln","roller","introducing","batch","organize","appreciated","alter","nicole","latino","ghana","edges","uc","mixing","handles","skilled","fitted","albuquerque","harmony","distinguished","asthma","projected","assumptions","shareholders","twins","developmental","rip","zope","regulated","triangle","amend","anticipated","oriental","reward","windsor","zambia","completing","gmbh","buf","ld","hydrogen","webshots","sprint","comparable","chick","advocate","sims","confusion","copyrighted","tray","inputs","warranties","genome","escorts","documented","thong","medal","paperbacks","coaches","vessels","harbour","walks","sucks","sol","keyboards","sage","knives","eco","vulnerable","arrange","artistic","bat","honors","booth","indie","reflected","unified","bones","breed","detector","ignored","polar","fallen","precise","sussex","respiratory","notifications","msgid","transexual","mainstream","invoice","evaluating","lip","subcommittee","sap","gather","suse","maternity","backed","alfred","colonial","mf","carey","motels","forming","embassy","cave","journalists","danny","rebecca","slight","proceeds","indirect","amongst","wool","foundations","msgstr","arrest","volleyball","mw","adipex","horizon","nu","deeply","toolbox","ict","marina","liabilities","prizes","bosnia","browsers","decreased","patio","dp","tolerance","surfing","creativity","lloyd","describing","optics","pursue","lightning","overcome","eyed","ou","quotations","grab","inspector","attract","brighton","beans","bookmarks","ellis","disable","snake","succeed","leonard","lending","oops","reminder","nipple","xi","searched","behavioral","riverside","bathrooms","plains","sku","ht","raymond","insights","abilities","initiated","sullivan","za","midwest","karaoke","trap","lonely","fool","ve","nonprofit","lancaster","suspended","hereby","observe","julia","containers","attitudes","karl","berry","collar","simultaneously","racial","integrate","bermuda","amanda","sociology","mobiles","screenshot","exhibitions","kelkoo","confident","retrieved","exhibits","officially","consortium","dies","terrace","bacteria","pts","replied","seafood","novels","rh","rrp","recipients","playboy","ought","delicious","traditions","fg","jail","safely","finite","kidney","periodically","fixes","sends","durable","mazda","allied","throws","moisture","hungarian","roster","referring","symantec","spencer","wichita","nasdaq","uruguay","ooo","hz","transform","timer","tablets","tuning","gotten","educators","tyler","futures","vegetable","verse","highs","humanities","independently","wanting","custody","scratch","launches","ipaq","alignment","masturbating","henderson","bk","britannica","comm","ellen","competitors","nhs","rocket","aye","bullet","towers","racks","lace","nasty","visibility","latitude","consciousness","ste","tumor","ugly","deposits","beverly","mistress","encounter","trustees","watts","duncan","reprints","hart","bernard","resolutions","ment","accessing","forty","tubes","attempted","col","midlands","priest","floyd","ronald","analysts","queue","dx","sk","trance","locale","nicholas","biol","yu","bundle","hammer","invasion","witnesses","runner","rows","administered","notion","sq","skins","mailed","oc","fujitsu","spelling","arctic","exams","rewards","beneath","strengthen","defend","aj","frederick","medicaid","treo","infrared","seventh","gods","une","welsh","belly","aggressive","tex","advertisements","quarters","stolen","cia","sublimedirectory","soonest","haiti","disturbed","determines","sculpture","poly","ears","dod","wp","fist","naturals","neo","motivation","lenders","pharmacology","fitting","fixtures","bloggers","mere","agrees","passengers","quantities","petersburg","consistently","powerpoint","cons","surplus","elder","sonic","obituaries","cheers","dig","taxi","punishment","appreciation","subsequently","om","belarus","nat","zoning","gravity","providence","thumb","restriction","incorporate","backgrounds","treasurer","guitars","essence","flooring","lightweight","ethiopia","tp","mighty","athletes","humanity","transcription","jm","holmes","complications","scholars","dpi","scripting","gis","remembered","galaxy","chester","snapshot","caring","loc","worn","synthetic","shaw","vp","segments","testament","expo","dominant","twist","specifics","itunes","stomach","partially","buried","cn","newbie","minimize","darwin","ranks","wilderness","debut","generations","tournaments","bradley","deny","anatomy","bali","judy","sponsorship","headphones","fraction","trio","proceeding","cube","defects","volkswagen","uncertainty","breakdown","milton","marker","reconstruction","subsidiary","strengths","clarity","rugs","sandra","adelaide","encouraging","furnished","monaco","settled","folding","emirates","terrorists","airfare","comparisons","beneficial","distributions","vaccine","belize","crap","fate","viewpicture","promised","volvo","penny","robust","bookings","threatened","minolta","republicans","discusses","gui","porter","gras","jungle","ver","rn","responded","rim","abstracts","zen","ivory","alpine","dis","prediction","pharmaceuticals","andale","fabulous","remix","alias","thesaurus","individually","battlefield","literally","newer","kay","ecological","spice","oval","implies","cg","soma","ser","cooler","appraisal","consisting","maritime","periodic","submitting","overhead","ascii","prospect","shipment","breeding","citations","geographical","donor","mozambique","tension","href","benz","trash","shapes","wifi","tier","fwd","earl","manor","envelope","diane","homeland","disclaimers","championships","excluded","andrea","breeds","rapids","disco","sheffield","bailey","aus","endif","finishing","emotions","wellington","incoming","prospects","lexmark","cleaners","bulgarian","hwy","eternal","cashiers","guam","cite","aboriginal","remarkable","rotation","nam","preventing","productive","boulevard","eugene","ix","gdp","pig","metric","compliant","minus","penalties","bennett","imagination","hotmail","refurbished","joshua","armenia","varied","grande","closest","activated","actress","mess","conferencing","assign","armstrong","politicians","trackbacks","lit","accommodate","tigers","aurora","una","slides","milan","premiere","lender","villages","shade","chorus","christine","rhythm","digit","argued","dietary","symphony","clarke","sudden","accepting","precipitation","marilyn","lions","findlaw","ada","pools","tb","lyric","claire","isolation","speeds","sustained","matched","approximate","rope","carroll","rational","programmer","fighters","chambers","dump","greetings","inherited","warming","incomplete","vocals","chronicle","fountain","chubby","grave","legitimate","biographies","burner","yrs","foo","investigator","gba","plaintiff","finnish","gentle","bm","prisoners","deeper","muslims","hose","mediterranean","nightlife","footage","howto","worthy","reveals","architects","saints","entrepreneur","carries","sig","freelance","duo","excessive","devon","screensaver","helena","saves","regarded","valuation","unexpected","cigarette","fog","characteristic","marion","lobby","egyptian","tunisia","metallica","outlined","consequently","headline","treating","punch","appointments","str","gotta","cowboy","narrative","bahrain","enormous","karma","consist","betty","queens","academics","pubs","quantitative","shemales","lucas","screensavers","subdivision","tribes","vip","defeat","clicks","distinction","honduras","naughty","hazards","insured","harper","livestock","mardi","exemption","tenant","sustainability","cabinets","tattoo","shake","algebra","shadows","holly","formatting","silly","nutritional","yea","mercy","hartford","freely","marcus","sunrise","wrapping","mild","fur","nicaragua","weblogs","timeline","tar","belongs","rj","readily","affiliation","soc","fence","nudist","infinite","diana","ensures","relatives","lindsay","clan","legally","shame","satisfactory","revolutionary","bracelets","sync","civilian","telephony","mesa","fatal","remedy","realtors","breathing","briefly","thickness","adjustments","graphical","genius","discussing","aerospace","fighter","meaningful","flesh","retreat","adapted","barely","wherever","estates","rug","democrat","borough","maintains","failing","shortcuts","ka","retained","voyeurweb","pamela","andrews","marble","extending","jesse","specifies","hull","logitech","surrey","briefing","belkin","dem","accreditation","wav","blackberry","highland","meditation","modular","microphone","macedonia","combining","brandon","instrumental","giants","organizing","shed","balloon","moderators","winston","memo","ham","solved","tide","kazakhstan","hawaiian","standings","partition","invisible","gratuit","consoles","funk","fbi","qatar","magnet","translations","porsche","cayman","jaguar","reel","sheer","commodity","posing","wang","kilometers","rp","bind","thanksgiving","rand","hopkins","urgent","guarantees","infants","gothic","cylinder","witch","buck","indication","eh","congratulations","tba","cohen","sie","usgs","puppy","kathy","acre","graphs","surround","cigarettes","revenge","expires","enemies","lows","controllers","aqua","chen","emma","consultancy","finances","accepts","enjoying","conventions","eva","patrol","smell","pest","hc","italiano","coordinates","rca","fp","carnival","roughly","sticker","promises","responding","reef","physically","divide","stakeholders","hydrocodone","gst","consecutive","cornell","satin","bon","deserve","attempting","mailto","promo","jj","representations","chan","worried","tunes","garbage","competing","combines","mas","beth","bradford","len","phrases","kai","peninsula","chelsea","boring","reynolds","dom","jill","accurately","speeches","reaches","schema","considers","sofa","catalogs","ministries","vacancies","quizzes","parliamentary","obj","prefix","lucia","savannah","barrel","typing","nerve","dans","planets","deficit","boulder","pointing","renew","coupled","viii","myanmar","metadata","harold","circuits","floppy","texture","handbags","jar","ev","somerset","incurred","acknowledge","thoroughly","antigua","nottingham","thunder","tent","caution","identifies","questionnaire","qualification","locks","modelling","namely","miniature","dept","hack","dare","euros","interstate","pirates","aerial","hawk","consequence","rebel","systematic","perceived","origins","hired","makeup","textile","lamb","madagascar","nathan","tobago","presenting","cos","troubleshooting","uzbekistan","indexes","pac","rl","erp","centuries","gl","magnitude","ui","richardson","hindu","dh","fragrances","vocabulary","licking","earthquake","vpn","fundraising","fcc","markers","weights","albania","geological","assessing","lasting","wicked","eds","introduces","kills","roommate","webcams","pushed","webmasters","ro","df","computational","acdbentity","participated","junk","handhelds","wax","lucy","answering","hans","impressed","slope","reggae","failures","poet","conspiracy","surname","theology","nails","evident","whats","rides","rehab","epic","saturn","organizer","nut","allergy","sake","twisted","combinations","preceding","merit","enzyme","cumulative","zshops","planes","edmonton","tackle","disks","condo","pokemon","amplifier","ambien","arbitrary","prominent","retrieve","lexington","vernon","sans","worldcat","titanium","irs","fairy","builds","contacted","shaft","lean","bye","cdt","recorders","occasional","leslie","casio","deutsche","ana","postings","innovations","kitty","postcards","dude","drain","monte","fires","algeria","blessed","luis","reviewing","cardiff","cornwall","favors","potato","panic","explicitly","sticks","leone","transsexual","ez","citizenship","excuse","reforms","basement","onion","strand","pf","sandwich","uw","lawsuit","alto","informative","girlfriend","bloomberg","cheque","hierarchy","influenced","banners","reject","eau","abandoned","bd","circles","italic","beats","merry","mil","scuba","gore","complement","cult","dash","passive","mauritius","valued","cage","checklist","bangbus","requesting","courage","verde","lauderdale","scenarios","gazette","hitachi","divx","extraction","batman","elevation","hearings","coleman","hugh","lap","utilization","beverages","calibration","jake","eval","efficiently","anaheim","ping","textbook","dried","entertaining","prerequisite","luther","frontier","settle","stopping","refugees","knights","hypothesis","palmer","medicines","flux","derby","sao","peaceful","altered","pontiac","regression","doctrine","scenic","trainers","muze","enhancements","renewable","intersection","passwords","sewing","consistency","collectors","conclude","recognised","munich","oman","celebs","gmc","propose","hh","azerbaijan","lighter","rage","adsl","uh","prix","astrology","advisors","pavilion","tactics","trusts","occurring","supplemental","travelling","talented","annie","pillow","induction","derek","precisely","shorter","harley","spreading","provinces","relying","finals","paraguay","steal","parcel","refined","fd","bo","fifteen","widespread","incidence","fears","predict","boutique","acrylic","rolled","tuner","avon","incidents","peterson","rays","asn","shannon","toddler","enhancing","flavor","alike","walt","homeless","horrible","hungry","metallic","acne","blocked","interference","warriors","palestine","listprice","libs","undo","cadillac","atmospheric","malawi","wm","pk","sagem","knowledgestorm","dana","halo","ppm","curtis","parental","referenced","strikes","lesser","publicity","marathon","ant","proposition","gays","pressing","gasoline","apt","dressed","scout","belfast","exec","dealt","niagara","inf","eos","warcraft","charms","catalyst","trader","bucks","allowance","vcr","denial","uri","designation","thrown","prepaid","raises","gem","duplicate","electro","criterion","badge","wrist","civilization","analyzed","vietnamese","heath","tremendous","ballot","lexus","varying","remedies","validity","trustee","maui","handjobs","weighted","angola","squirt","performs","plastics","realm","corrected","jenny","helmet","salaries","postcard","elephant","yemen","encountered","tsunami","scholar","nickel","internationally","surrounded","psi","buses","expedia","geology","pct","wb","creatures","coating","commented","wallet","cleared","smilies","vids","accomplish","boating","drainage","shakira","corners","broader","vegetarian","rouge","yeast","yale","newfoundland","sn","qld","pas","clearing","investigated","dk","ambassador","coated","intend","stephanie","contacting","vegetation","doom","findarticles","louise","kenny","specially","owen","routines","hitting","yukon","beings","bite","issn","aquatic","reliance","habits","striking","myth","infectious","podcasts","singh","gig","gilbert","sas","ferrari","continuity","brook","fu","outputs","phenomenon","ensemble","insulin","assured","biblical","weed","conscious","accent","mysimon","eleven","wives","ambient","utilize","mileage","oecd","prostate","adaptor","auburn","unlock","hyundai","pledge","vampire","angela","relates","nitrogen","xerox","dice","merger","softball","referrals","quad","dock","differently","firewire","mods","nextel","framing","organised","musician","blocking","rwanda","sorts","integrating","vsnet","limiting","dispatch","revisions","papua","restored","hint","armor","riders","chargers","remark","dozens","varies","msie","reasoning","wn","liz","rendered","picking","charitable","guards","annotated","ccd","sv","convinced","openings","buys","burlington","replacing","researcher","watershed","councils","occupations","acknowledged","nudity","kruger","pockets","granny","pork","zu","equilibrium","viral","inquire","pipes","characterized","laden","aruba","cottages","realtor","merge","privilege","edgar","develops","qualifying","chassis","dubai","estimation","barn","pushing","llp","fleece","pediatric","boc","fare","dg","asus","pierce","allan","dressing","techrepublic","sperm","vg","bald","filme","craps","fuji","frost","leon","institutes","mold","dame","fo","sally","yacht","tracy","prefers","drilling","brochures","herb","tmp","alot","ate","breach","whale","traveller","appropriations","suspected","tomatoes","benchmark","beginners","instructors","highlighted","bedford","stationery","idle","mustang","unauthorized","clusters","antibody","competent","momentum","fin","wiring","io","pastor","mud","calvin","uni","shark","contributor","demonstrates","phases","grateful","emerald","gradually","laughing","grows","cliff","desirable","tract","ul","ballet","ol","journalist","abraham","js","bumper","afterwards","webpage","religions","garlic","hostels","shine","senegal","explosion","pn","banned","wendy","briefs","signatures","diffs","cove","mumbai","ozone","disciplines","casa","mu","daughters","conversations","radios","tariff","nvidia","opponent","pasta","simplified","muscles","serum","wrapped","swift","motherboard","runtime","inbox","focal","bibliographic","vagina","eden","distant","incl","champagne","ala","decimal","hq","deviation","superintendent","propecia","dip","nbc","samba","hostel","housewives","employ","mongolia","penguin","magical","influences","inspections","irrigation","miracle","manually","reprint","reid","wt","hydraulic","centered","robertson","flex","yearly","penetration","wound","belle","rosa","conviction","hash","omissions","writings","hamburg","lazy","mv","mpg","retrieval","qualities","cindy","lolita","fathers","carb","charging","cas","marvel","lined","cio","dow","prototype","importantly","rb","petite","apparatus","upc","terrain","dui","pens","explaining","yen","strips","gossip","rangers","nomination","empirical","mh","rotary","worm","dependence","discrete","beginner","boxed","lid","sexuality","polyester","cubic","deaf","commitments","suggesting","sapphire","kinase","skirts","mats","remainder","crawford","labeled","privileges","televisions","specializing","marking","commodities","pvc","serbia","sheriff","griffin","declined","guyana","spies","blah","mime","neighbor","motorcycles","elect","highways","thinkpad","concentrate","intimate","reproductive","preston","deadly","cunt","feof","bunny","chevy","molecules","rounds","longest","refrigerator","tions","intervals","sentences","dentists","usda","exclusion","workstation","holocaust","keen","flyer","peas","dosage","receivers","urls","customise","disposition","variance","navigator","investigators","cameroon","baking","marijuana","adaptive","computed","needle","baths","enb","gg","cathedral","brakes","og","nirvana","ko","fairfield","owns","til","invision","sticky","destiny","generous","madness","emacs","climb","blowing","fascinating","landscapes","heated","lafayette","jackie","wto","computation","hay","cardiovascular","ww","sparc","cardiac","salvation","dover","adrian","predictions","accompanying","vatican","brutal","learners","gd","selective","arbitration","configuring","token","editorials","zinc","sacrifice","seekers","guru","isa","removable","convergence","yields","gibraltar","levy","suited","numeric","anthropology","skating","kinda","aberdeen","emperor","grad","malpractice","dylan","bras","belts","blacks","educated","rebates","reporters","burke","proudly","pix","necessity","rendering","mic","inserted","pulling","basename","kyle","obesity","curves","suburban","touring","clara","vertex","bw","hepatitis","nationally","tomato","andorra","waterproof","expired","mj","travels","flush","waiver","pale","specialties","hayes","humanitarian","invitations","functioning","delight","survivor","garcia","cingular","economies","alexandria","bacterial","moses","counted","undertake","declare","continuously","johns","valves","gaps","impaired","achievements","donors","tear","jewel","teddy","lf","convertible","ata","teaches","ventures","nil","bufing","stranger","tragedy","julian","nest","pam","dryer","painful","velvet","tribunal","ruled","nato","pensions","prayers","funky","secretariat","nowhere","cop","paragraphs","gale","joins","adolescent","nominations","wesley","dim","lately","cancelled","scary","mattress","mpegs","brunei","likewise","banana","introductory","slovak","cakes","stan","reservoir","occurrence","idol","bloody","mixer","remind","wc","worcester","sbjct","demographic","charming","mai","tooth","disciplinary","annoying","respected","stays","disclose","affair","drove","washer","upset","restrict","springer","beside","mines","portraits","rebound","logan","mentor","interpreted","evaluations","fought","baghdad","elimination","metres","hypothetical","immigrants","complimentary","helicopter","pencil","freeze","hk","performer","abu","titled","commissions","sphere","powerseller","moss","ratios","concord","graduated","endorsed","ty","surprising","walnut","lance","ladder","italia","unnecessary","dramatically","liberia","sherman","cork","maximize","cj","hansen","senators","workout","mali","yugoslavia","bleeding","characterization","colon","likelihood","lanes","purse","fundamentals","contamination","mtv","endangered","compromise","masturbation","optimize","stating","dome","caroline","leu","expiration","namespace","align","peripheral","bless","engaging","negotiation","crest","opponents","triumph","nominated","confidentiality","electoral","changelog","welding","orgasm","deferred","alternatively","heel","alloy","condos","plots","polished","yang","gently","greensboro","tulsa","locking","casey","controversial","draws","fridge","blanket","bloom","qc","simpsons","lou","elliott","recovered","fraser","justify","upgrading","blades","pgp","loops","surge","frontpage","trauma","aw","tahoe","advert","possess","demanding","defensive","sip","flashers","subaru","forbidden","tf","vanilla","programmers","pj","monitored","installations","deutschland","picnic","souls","arrivals","spank","cw","practitioner","motivated","wr","dumb","smithsonian","hollow","vault","securely","examining","fioricet","groove","revelation","rg","pursuit","delegation","wires","bl","dictionaries","mails","backing","greenhouse","sleeps","vc","blake","transparency","dee","travis","wx","endless","figured","orbit","currencies","niger","bacon","survivors","positioning","heater","colony","cannon","circus","promoted","forbes","mae","moldova","mel","descending","paxil","spine","trout","enclosed","feat","temporarily","ntsc","cooked","thriller","transmit","apnic","fatty","gerald","pressed","frequencies","scanned","reflections","hunger","mariah","sic","municipality","usps","joyce","detective","surgeon","cement","experiencing","fireplace","endorsement","bg","planners","disputes","textiles","missile","intranet","closes","seq","psychiatry","persistent","deborah","conf","marco","assists","summaries","glow","gabriel","auditor","wma","aquarium","violin","prophet","cir","bracket","looksmart","isaac","oxide","oaks","magnificent","erik","colleague","naples","promptly","modems","adaptation","hu","harmful","paintball","prozac","sexually","enclosure","acm","dividend","newark","kw","paso","glucose","phantom","norm","playback","supervisors","westminster","turtle","ips","distances","absorption","treasures","dsc","warned","neural","ware","fossil","mia","hometown","badly","transcripts","apollo","wan","disappointed","persian","continually","communist","collectible","handmade","greene","entrepreneurs","robots","grenada","creations","jade","scoop","acquisitions","foul","keno","gtk","earning","mailman","sanyo","nested","biodiversity","excitement","somalia","movers","verbal","blink","presently","seas","carlo","workflow","mysterious","novelty","bryant","tiles","voyuer","librarian","subsidiaries","switched","stockholm","tamil","garmin","ru","pose","fuzzy","indonesian","grams","therapist","richards","mrna","budgets","toolkit","promising","relaxation","goat","render","carmen","ira","sen","thereafter","hardwood","erotica","temporal","sail","forge","commissioners","dense","dts","brave","forwarding","qt","awful","nightmare","airplane","reductions","southampton","istanbul","impose","organisms","sega","telescope","viewers","asbestos","portsmouth","cdna","meyer","enters","pod","savage","advancement","wu","harassment","willow","resumes","bolt","gage","throwing","existed","whore","generators","lu","wagon","barbie","dat","favour","soa","knock","urge","smtp","generates","potatoes","thorough","replication","inexpensive","kurt","receptors","peers","roland","optimum","neon","interventions","quilt","huntington","creature","ours","mounts","syracuse","internship","lone","refresh","aluminium","snowboard","beastality","webcast","michel","evanescence","subtle","coordinated","notre","shipments","maldives","stripes","firmware","antarctica","cope","shepherd","lm","canberra","cradle","chancellor","mambo","lime","kirk","flour","controversy","legendary","bool","sympathy","choir","avoiding","beautifully","blond","expects","cho","jumping","fabrics","antibodies","polymer","hygiene","wit","poultry","virtue","burst","examinations","surgeons","bouquet","immunology","promotes","mandate","wiley","departmental","bbs","spas","ind","corpus","johnston","terminology","gentleman","fibre","reproduce","convicted","shades","jets","indices","roommates","adware","qui","intl","threatening","spokesman","zoloft","activists","frankfurt","prisoner","daisy","halifax","encourages","ultram","cursor","assembled","earliest","donated","stuffed","restructuring","insects","terminals","crude","morrison","maiden","simulations","cz","sufficiently","examines","viking","myrtle","bored","cleanup","yarn","knit","conditional","mug","crossword","bother","budapest","conceptual","knitting","attacked","hl","bhutan","liechtenstein","mating","compute","redhead","arrives","translator","automobiles","tractor","allah","continent","ob","unwrap","fares","longitude","resist","challenged","telecharger","hoped","pike","safer","insertion","instrumentation","ids","hugo","wagner","constraint","groundwater","touched","strengthening","cologne","gzip","wishing","ranger","smallest","insulation","newman","marsh","ricky","ctrl","scared","theta","infringement","bent","laos","subjective","monsters","asylum","lightbox","robbie","stake","cocktail","outlets","swaziland","varieties","arbor","mediawiki","configurations","poison"]} diff --git a/conf/help.json b/conf/help.json index 13ee310..b4a3205 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,7 +20,7 @@ "mon": "mon -h", "chans": "chans [ ...]", "users": "users [ ...]", - "alias": "alias [ ]", + "alias": "alias [ OR auto]", "relay": "relay [ ]", "network": "network [
]", "provision": "provision []", diff --git a/main.py b/main.py index 34add10..67664df 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ filemap = { "relay": ["relay.json", "relay list"], "network": ["network.json", "network list"], "tokens": ["tokens.json", "authentication tokens"], + "aliasdata": ["aliasdata.json", "data for alias generation"] } connections = {} diff --git a/modules/alias.py b/modules/alias.py new file mode 100644 index 0000000..0248088 --- /dev/null +++ b/modules/alias.py @@ -0,0 +1,65 @@ +import main +import random +import re + +def generate_alias(): + nick = random.choice(main.aliasdata["stubs"]) + rand = random.randint(1, 2) + if rand == 1: + nick = nick.capitalize() + rand = random.randint(1, 4) + while rand == 1: + split = random.randint(0, len(nick)-1) + nick = nick[:split] + nick[split+1:] + rand = random.randint(1, 4) + rand = random.randint(1, 3) + if rand == 1 or rand == 4: + nick = random.choice(main.aliasdata["stubs"]) + nick + if rand == 2 or rand == 5: + nick = random.choice(main.aliasdata["stubs"]).capitalize() + nick + if rand > 2: + nick = nick + str(random.randint(0, 100)) + nick = nick[:11] + + altnick = nick + rand = random.randint(1, 3) + if rand == 1: + altnick += "_" + elif rand == 2: + altnick += "1" + else: + altnick = "_" + altnick + + namebase = random.choice(main.aliasdata["realnames"]) + + ident = nick[:10] + rand = random.randint(1, 7) + if rand == 1: + ident = "quassel" + elif rand == 2: + ident = ident.lower() + elif rand == 3: + ident = re.sub("[0-9]", "", nick) + ident = ident[:10] + elif rand == 4: + ident = namebase.split(" ")[0].lower() + ident = ident[:10] + elif rand == 5: + ident = namebase.split(" ")[0] + ident = ident[:10] + elif rand == 6: + ident = re.sub("\s", "", namebase).lower() + ident = ident[:10] + + realname = nick + rand = random.randint(1, 5) + if rand == 1: + realname = namebase + elif rand == 2 or rand == 3: + realname = re.sub("[0-9]", "", realname) + if rand == 3 or rand == 4: + realname = realname.capitalize() + + password = "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) + + return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password} From 2757256d4f4c79fd1c77e0877206f42e289e96bf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 11 Aug 2019 20:52:10 +0100 Subject: [PATCH 129/394] Implement a single function for all callbacks from IRC hooks and send a seperate notification if an action takes place that concerns a bot --- .gitignore | 1 - commands/cmd.py | 1 - conf/example/config.json | 15 ++-- conf/help.json | 1 - core/bot.py | 181 ++++++++++++++++++++------------------- core/relay.py | 6 +- main.py | 1 - modules/monitor.py | 5 -- modules/provision.py | 3 + modules/userinfo.py | 5 +- 10 files changed, 106 insertions(+), 113 deletions(-) diff --git a/.gitignore b/.gitignore index e23d8a6..e81873f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ conf/config.json conf/pool.json conf/wholist.json conf/counters.json -conf/masterbuf.json conf/monitor.json conf/alias.json conf/relay.json diff --git a/commands/cmd.py b/commands/cmd.py index 5908bda..68a63ce 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -13,7 +13,6 @@ class Cmd: return commands = {spl[3]: [" ".join(spl[4:])]} - print(" ".join(spl[4:])) success("Sending commands to relay %s as user %s" % (spl[1], spl[2])) deliverRelayCommands(spl[1], commands, user=spl[2]) return diff --git a/conf/example/config.json b/conf/example/config.json index 27e79fb..bc67bdf 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -15,19 +15,15 @@ "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, - "Notifications": { - "Highlight": true, - "Connection": true, - "Query": true - }, - "Compat": { - "ZNC": true - }, "Dist": { "Enabled": true, "SendOutput": false, "File": "conf/dist.sh" }, + "Toggles": { + "Who": false, + "CycleChans": true + }, "Password": "s", "Tweaks": { "ZNC": { @@ -42,6 +38,5 @@ "Factor": 2.718281828459045, "Jitter": 0.11962656472 } - }, - "Master": [null, null] + } } diff --git a/conf/help.json b/conf/help.json index 13ee310..0211171 100644 --- a/conf/help.json +++ b/conf/help.json @@ -4,7 +4,6 @@ "del": "del ", "mod": "mod [] []", "get": "get ", - "key": "key [] [] [] []", "who": "who ", "join": "join []", "part": "part ", diff --git a/core/bot.py b/core/bot.py index bb91bc7..bc7a6f5 100644 --- a/core/bot.py +++ b/core/bot.py @@ -6,6 +6,7 @@ from twisted.internet import reactor from string import digits from random import randint +from copy import deepcopy from modules import userinfo from modules import counters as count @@ -73,8 +74,7 @@ class IRCRelay(IRCClient): def signedOn(self): self.connected = True log("signed on as a relay: %s" % self.relay) - if main.config["Notifications"]["Connection"]: - sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) + #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares for i in self.relayCommands.keys(): for x in self.relayCommands[i]: self.msg(main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) @@ -133,29 +133,69 @@ class IRCBot(IRCClient): return [nick, ident, host] + def event(self, **cast): + for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over + if cast[i] == "": # a dictionary that changes length with each iteration + del cast[i] + if "muser" in cast.keys(): + cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) + del cast["muser"] + if set(["nick", "ident", "host", "message"]).issubset(set(cast)): + if "message" in cast.keys(): + if cast["ident"] == "znc" and cast["host"] == "znc.in": + cast["type"] = "znc" + cast["name"] = self.name + del cast["nick"] + del cast["ident"] + del cast["host"] + del cast["target"] + if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: + if "target" in cast.keys(): + if cast["target"] == self.nickname: + #castDup = deepcopy(cast) + cast["mtype"] = cast["type"] + cast["type"] = "query" + cast["name"] = self.name + #self.event(**castDup) + # Don't call self.event for this one because queries are not events on a + # channel, but we still want to see them + if "user" in cast.keys(): + if cast["user"] == self.nickname: + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "self" + castDup["name"] = self.name + self.event(**castDup) + if "nick" in cast.keys(): + if cast["nick"] == self.nickname: + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "self" + castDup["name"] = self.name + self.event(**castDup) + if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries + if self.nickname in cast["message"]: + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "highlight" + castDup["name"] = self.name + self.event(**castDup) + + if "name" not in cast.keys(): + cast["name"] = self.net + if cast["type"] in ["msg", "notice", "action"]: + userinfo.editUser(self.net, cast["nick"]+"!"+cast["ident"]+"@"+cast["host"]) + count.event(self.net, cast["type"]) + monitor.event(self.net, self.name, cast) + def privmsg(self, user, channel, msg): - nick, ident, host = self.parsen(user) - # ZNC adds messages when no users are in the channel, messing up tracking - #userinfo.editUser(self.net, channel, nick, user) - count.event(self.net, "msg") - #event = None - #if self.nickname.lower() in msg.lower(): - # event = "highlight" - monitor.event(self.net, self.name, {"type": "msg", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) + self.event(type="msg", muser=user, target=channel, message=msg) def noticed(self, user, channel, msg): - nick, ident, host = self.parsen(user) - #userinfo.editUser(self.net, channel, nick, user) - count.event(self.net, "notice") - - monitor.event(self.net, self.name, {"type": "notice", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) + self.event(type="notice", muser=user, target=channel, message=msg) def action(self, user, channel, msg): - nick, ident, host = self.parsen(user) - #userinfo.editUser(self.net, channel, nick, user) - count.event(self.net, "action") - - monitor.event(self.net, self.name, {"type": "action", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg}) + self.event(type="action", muser=user, target=channel, message=msg) def get(self, var): try: @@ -177,7 +217,7 @@ class IRCBot(IRCClient): oldnick, ident, host = self.parsen(olduser) userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host) self.nickname = newnick - count.event(self.net, "selfnick") + self.event(type="self", mtype="nick", nick=oldnick, ident=ident, host=host, user=newnick) def irc_ERR_NICKNAMEINUSE(self, prefix, params): self._attemptedNick = self.alterCollidedNick(self._attemptedNick) @@ -197,7 +237,7 @@ class IRCBot(IRCClient): def irc_RPL_WHOREPLY(self, prefix, params): channel = params[1] - user = params[2] + ident = params[2] host = params[3] server = params[4] nick = params[5] @@ -206,9 +246,8 @@ class IRCBot(IRCClient): if channel not in self._who: return n = self._who[channel][1] - n.append([nick, user, host, server, status, realname]) - count.event(self.net, "whoreply") - monitor.event(self.net, self.name, {"type": "who", "nick": nick, "ident": user, "host": host, "realname": realname, "target": channel, "server": server, "status": status}) + n.append([nick, nick, host, server, status, realname]) + self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, target=channel, server=server, status=status) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -260,7 +299,7 @@ class IRCBot(IRCClient): newNicklist = [] for i in nicklist[1]: for x in i: - f = self.sanit(x) + f = self.sanit(x) # need to store this as well, or potentially just do not remove it... if f: newNicklist.append(f) userinfo.initialNames(self.net, nicklist[0], newNicklist) @@ -320,11 +359,8 @@ class IRCBot(IRCClient): channel = params[0] kicked = params[1] message = params[-1] - if kicked.lower() == self.nickname.lower(): - # Yikes! - self.kickedFrom(channel, prefix, message) - else: - self.userKicked(kicked, channel, prefix, message) + # Checks on whether it was us that was kicked are done in userKicked + self.userKicked(kicked, channel, prefix, message) def irc_TOPIC(self, prefix, params): """ @@ -339,26 +375,20 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True log("signed on: %s" % self.name) - if main.config["Notifications"]["Connection"]: - sendRelayNotification(self.name, {"type": "conn", "status": "connected"}) - count.event(self.net, "signedon") + self.event(type="conn", status="connected") def joined(self, channel): if not channel in self.channels: self.channels.append(channel) self.names(channel).addCallback(self.got_names) self.who(channel).addCallback(self.got_who) - count.event(self.net, "selfjoin") - if self.name == main.config["Master"][0] and channel == main.config["Master"][1]: - for i in range(len(main.masterbuf)): - self.msg(channel, main.masterbuf.pop(0)) - main.saveConf("masterbuf") - lc = LoopingCall(self.who, channel) - self._getWho[channel] = lc - intrange = main.config["Tweaks"]["Delays"]["WhoRange"] - minint = main.config["Tweaks"]["Delays"]["WhoLoop"] - interval = randint(minint, minint+intrange) - lc.start(interval) + if main.config["Toggles"]["Who"]: + lc = LoopingCall(self.who, channel) + self._getWho[channel] = lc + intrange = main.config["Tweaks"]["Delays"]["WhoRange"] + minint = main.config["Tweaks"]["Delays"]["WhoLoop"] + interval = randint(minint, minint+intrange) + lc.start(interval) def botLeft(self, channel): if channel in self.channels: @@ -368,79 +398,58 @@ class IRCBot(IRCClient): lc.stop() del self._getWho[channel] userinfo.delChannel(self.net, channel) + log("Can no longer cover %s, removing records" % channel) def left(self, user, channel, message): - nick, ident, host = self.parsen(user) - count.event(self.net, "selfpart") - monitor.event(self.net, self.name, {"type": "part", "target": channel, "message": message}) - monitor.event(self.net, self.name, {"type": "part", "self": True, "target": channel, "message": message}) - self.botLeft(channel) - - def kickedFrom(self, channel, kicker, message): - nick, ident, host = self.parsen(kicker) - if channel in self.channels: - self.channels.remove(channel) - count.event(self.net, "selfkick") - monitor.event(self.net, self.name, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) - monitor.event(self.net, self.name, {"type": "kick", "self": True, "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) + self.event(type="part", muser=user, target=channel, message=message) self.botLeft(channel) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) userinfo.addUser(self.net, channel, nick, user) - count.event(self.net, "join") - monitor.event(self.net, self.name, {"type": "join", "nick": nick, "ident": ident, "host": host, "target": channel}) + self.event(type="join", nick=nick, ident=ident, host=host, target=channel) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) userinfo.delUser(self.net, channel, nick, user) - count.event(self.net, "part") - monitor.event(self.net, self.name, {"type": "part", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message}) + self.event(type="part", nick=nick, ident=ident, host=host, target=channel, message=message) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.delUserByNetwork(self.net, nick, user) - count.event(self.net, "quit") - - monitor.event(self.net, self.name, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + self.event(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) def botQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) userinfo.delUserByNetwork(self.net, nick, user) - count.event(self.net, "selfquit") - - monitor.event(self.net, self.name, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage}) - monitor.event(self.net, self.name, {"type": "quit", "self": True, "nick": nick, "ident": ident, "host": host, "message": quitMessage}) + self.event(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.editUser(self.net, channel, nick, kicker) + userinfo.editUser(self.net, kicker) userinfo.delUserByNick(self.net, channel, kickee) - count.event(self.net, "kick") - - monitor.event(self.net, self.name, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message, "user": kickee}) + if kickee.lower == self.nickname.lower: + self.botLeft(channel) + self.event(type="kick", nick=nick, ident=ident, host=host, target=channel, message=message, user=kickee) def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) - count.event(self.net, "nick") - monitor.event(self.net, self.name, {"type": "nick", "nick": nick, "ident": ident, "host": host, "user": newname}) + self.event(type="nick", nick=nick, ident=ident, host=host, user=newname) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, channel, nick, user) - count.event(self.net, "topic") + userinfo.editUser(self.net, user) - monitor.event(self.net, self.name, {"type": "topic", "nick": nick, "ident": ident, "host": host, "target": channel, "message": newTopic}) + self.event(type="topic", nick=nick, ident=ident, host=host, target=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, channel, nick, user) - count.event(self.net, "mode") + userinfo.editUser(self.net, user) argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - monitor.event(self.net, self.name, {"type": "mode", "nick": nick, "ident": ident, "host": host, "target": channel, "modes": m, "modeargs": a}) + self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): @@ -469,7 +478,7 @@ class IRCBotFactory(ReconnectingClientFactory): def clientConnectionLost(self, connector, reason): if not self.relay: - userinfo.delNetwork(self.net, self.client.channels) + userinfo.delNetwork(self.name, self.client.channels) if not self.client == None: self.client.connected = False self.client.channels = [] @@ -477,9 +486,7 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s: connection lost: %s" % (self.name, error)) if not self.relay: sendAll("%s: connection lost: %s" % (self.name, error)) - if main.config["Notifications"]["Connection"]: - sendRelayNotification(self.name, {"type": "conn", "status": "lost", "reason": error}) - if not self.relay: + sendRelayNotification(self.name, {"type": "conn", "status": "lost", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -491,9 +498,7 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s: connection failed: %s" % (self.name, error)) if not self.relay: sendAll("%s: connection failed: %s" % (self.name, error)) - if main.config["Notifications"]["Connection"]: - sendRelayNotification(self.name, {"type": "conn", "status": "failed", "reason": error}) - if not self.relay: + sendRelayNotification(self.name, {"type": "conn", "status": "failed", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/relay.py b/core/relay.py index 76dbc87..765b783 100644 --- a/core/relay.py +++ b/core/relay.py @@ -6,9 +6,7 @@ from datetime import datetime import main from utils.logging.log import * -validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err"] - -selfTypes = ["query", "self", "highlight"] +validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"] class Relay(Protocol): def __init__(self, addr): @@ -137,7 +135,7 @@ class RelayFactory(Factory): def sendRelayNotification(name, cast): for i in main.relayConnections.keys(): if main.relayConnections[i].authed: - if cast["type"] in main.relayConnections[i].subscriptions or set(main.relayConnections[i].subscriptions).issubset(cast): + if cast["type"] in main.relayConnections[i].subscriptions: newCast = deepcopy(cast) newCast["name"] = name newCast["time"] = str(datetime.now()) diff --git a/main.py b/main.py index 34add10..1aa0cdf 100644 --- a/main.py +++ b/main.py @@ -11,7 +11,6 @@ filemap = { "pool": ["pool.json", "network, alias and relay mappings"], "help": ["help.json", "command help"], "counters": ["counters.json", "counters file"], - "masterbuf": ["masterbuf.json", "master buffer"], "monitor": ["monitor.json", "monitoring database"], "alias": ["alias.json", "alias details"], "relay": ["relay.json", "relay list"], diff --git a/modules/monitor.py b/modules/monitor.py index 45f5118..08fae94 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -55,11 +55,6 @@ def event(name, numberedName, cast, event=None): target = cast["target"] else: target = None - if set(["nick", "ident", "host", "message"]).issubset(set(cast)): - if main.config["Compat"]["ZNC"] and "message" in cast.keys(): - if cast["nick"][0] == main.config["Tweaks"]["ZNC"]["Prefix"] and cast["ident"] == "znc" and cast["host"] == "znc.in": - sendRelayNotification(numberedName, {"type": "znc", "message": cast["message"]}) - return sendRelayNotification(name, cast) monitorGroups = testNetTarget(name, target) diff --git a/modules/provision.py b/modules/provision.py index e1e5bc0..0eb7464 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -39,6 +39,9 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass if not main.config["ConnectOnCreate"]: stage3commands["status"] = [] stage3commands["status"].append("Disconnect") + if main.config["Toggles"]["CycleChans"]: + stage2commands["status"] = [] + stage2commands["status"].append("LoadMod disconkick") deliverRelayCommands(relay, commands, stage2=[[alias+"/"+network, stage2commands], [alias+"/"+network, stage3commands]]) diff --git a/modules/userinfo.py b/modules/userinfo.py index 42f2d48..d9bdddf 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -76,7 +76,7 @@ def initialNames(name, channel, names): p.sadd("live.chan."+name+"."+i, channel) p.execute() -def editUser(name, channel, nick, user): +def editUser(name, user): gnamespace = "live.who.%s" % name main.r.sadd(gnamespace, user) @@ -151,7 +151,7 @@ def delUserByNetwork(name, nick, user): p.delete(chanspace) p.execute() -def delChannel(name, channel): +def delChannel(name, channel): # This function is extremely expensive, look to replace gnamespace = "live.who.%s" % name namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() @@ -171,4 +171,5 @@ def delNetwork(name, channels): log("Purging channels for %s" % name) for i in channels: delChannel(name, i) + log("Finished purging channels for %s" % name) return From 3a92ebab6bc421df7e569023363543d36368b8a2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 11 Aug 2019 20:53:26 +0100 Subject: [PATCH 130/394] Convert nickname and messages to lowercase before comparison --- core/bot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/bot.py b/core/bot.py index bc7a6f5..d06c32b 100644 --- a/core/bot.py +++ b/core/bot.py @@ -151,7 +151,7 @@ class IRCBot(IRCClient): del cast["target"] if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: if "target" in cast.keys(): - if cast["target"] == self.nickname: + if cast["target"].lower() == self.nickname.lower(): #castDup = deepcopy(cast) cast["mtype"] = cast["type"] cast["type"] = "query" @@ -160,21 +160,21 @@ class IRCBot(IRCClient): # Don't call self.event for this one because queries are not events on a # channel, but we still want to see them if "user" in cast.keys(): - if cast["user"] == self.nickname: + if cast["user"].lower() == self.nickname.lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" castDup["name"] = self.name self.event(**castDup) if "nick" in cast.keys(): - if cast["nick"] == self.nickname: + if cast["nick"].lower() == self.nickname.lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" castDup["name"] = self.name self.event(**castDup) if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries - if self.nickname in cast["message"]: + if self.nickname.lower() in cast["message"].lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "highlight" From 1c8cbf808bb3c06c1797aa5ed8c7d4a4ec43b163 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 11 Aug 2019 21:52:46 +0100 Subject: [PATCH 131/394] Send fake quit and nick messages to every channel --- core/bot.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/core/bot.py b/core/bot.py index d06c32b..ee5d8b7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -336,10 +336,10 @@ class IRCBot(IRCClient): Called when a user has quit. """ nick = prefix.split('!')[0] - if nick == self.nickname: - self.botQuit(prefix, params[0]) - else: - self.userQuit(prefix, params[0]) + #if nick == self.nickname: + #self.botQuit(prefix, params[0]) + #else: + self.userQuit(prefix, params[0]) def irc_NICK(self, prefix, params): """ @@ -416,13 +416,8 @@ class IRCBot(IRCClient): def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) + self.chanlessEvent(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) userinfo.delUserByNetwork(self.net, nick, user) - self.event(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) - - def botQuit(self, user, quitMessage): - nick, ident, host = self.parsen(user) - userinfo.delUserByNetwork(self.net, nick, user) - self.event(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) @@ -432,10 +427,19 @@ class IRCBot(IRCClient): self.botLeft(channel) self.event(type="kick", nick=nick, ident=ident, host=host, target=channel, message=message, user=kickee) + def chanlessEvent(self, **cast): + chans = userinfo.getChansSingle(self.net, cast["nick"]) + if chans == None: + self.event(**cast) + return + for i in chans: + cast["target"] = i + self.event(**cast) + def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) + self.chanlessEvent(type="nick", nick=nick, ident=ident, host=host, user=newname) userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) - self.event(type="nick", nick=nick, ident=ident, host=host, user=newname) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) @@ -449,7 +453,7 @@ class IRCBot(IRCClient): argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, modeargs=a) + self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): @@ -478,7 +482,7 @@ class IRCBotFactory(ReconnectingClientFactory): def clientConnectionLost(self, connector, reason): if not self.relay: - userinfo.delNetwork(self.name, self.client.channels) + userinfo.delNetwork(self.net, self.client.channels) if not self.client == None: self.client.connected = False self.client.channels = [] From e5adcfef4c59f8b5100fc6bdad85bda42427bb2c Mon Sep 17 00:00:00 2001 From: Al Beano Date: Sun, 11 Aug 2019 21:58:14 +0100 Subject: [PATCH 132/394] Rework data structures, storing all front-end network data in Network objects --- .gitignore | 6 ++-- commands/alias.py | 61 --------------------------------------- commands/cmd.py | 14 +++++---- commands/del.py | 30 ++++++++++--------- commands/disable.py | 36 +++++++++++++---------- commands/enable.py | 27 +++++++++-------- commands/join.py | 30 +++++++++++-------- commands/list.py | 13 --------- commands/mod.py | 55 +++++++++++++++-------------------- commands/msg.py | 19 +++++++----- commands/network.py | 6 ++-- commands/part.py | 17 ++++++----- commands/relay.py | 36 ++++++++++++----------- commands/stats.py | 3 +- conf/example/alias.json | 1 - conf/example/network.json | 1 - conf/example/pool.json | 1 - conf/example/relay.json | 1 - conf/help.json | 22 +++++++------- core/bot.py | 22 +++++++------- core/helper.py | 27 ----------------- main.py | 49 ++++++++++++++++++++----------- modules/network.py | 57 ++++++++++++++++++++++++++++++++++++ threshold | 8 ++--- 24 files changed, 258 insertions(+), 284 deletions(-) delete mode 100644 commands/alias.py delete mode 100644 commands/list.py delete mode 100644 conf/example/alias.json delete mode 100644 conf/example/network.json delete mode 100644 conf/example/pool.json delete mode 100644 conf/example/relay.json delete mode 100644 core/helper.py create mode 100644 modules/network.py diff --git a/.gitignore b/.gitignore index e23d8a6..08f3657 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,13 @@ *.pyc *.pem +*.swp __pycache__/ conf/config.json -conf/pool.json conf/wholist.json conf/counters.json conf/masterbuf.json conf/monitor.json -conf/alias.json -conf/relay.json -conf/network.json conf/tokens.json +conf/network.dat conf/dist.sh env/ diff --git a/commands/alias.py b/commands/alias.py deleted file mode 100644 index ef5ac6f..0000000 --- a/commands/alias.py +++ /dev/null @@ -1,61 +0,0 @@ -import main -from yaml import dump -import modules.alias as alias - -class AliasCommand: - def __init__(self, *args): - self.alias(*args) - - def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - if length == 8: - if spl[1] == "add": - if spl[2] in main.alias.keys(): - failure("Alias already exists: %s" % spl[2]) - return - else: - main.alias[spl[2]] = {"nick": spl[3], - "altnick": spl[4], - "ident": spl[5], - "realname": spl[6], - "password": spl[7]} - success("Successfully created alias: %s" % spl[2]) - main.saveConf("alias") - return - else: - incUsage("alias") - return - - elif length == 3: - if spl[1] == "del": - if spl[2] in main.alias.keys(): - del main.alias[spl[2]] - success("Successfully removed alias: %s" % spl[2]) - main.saveConf("alias") - return - else: - failure("No such alias: %s" % spl[2]) - return - elif spl[1] == "add" and spl[2] == "auto": - newalias = alias.generate_alias() - while newalias["nick"] in main.alias.keys(): - newalias = alias.generate_alias() - main.alias[newalias["nick"]] = newalias - success("Successfully created alias: %s" % newalias["nick"]) - main.saveConf("alias") - return - else: - incUsage("alias") - return - elif length == 2: - if spl[1] == "list": - info(dump(main.alias)) - return - else: - incUsage("alias") - return - else: - incUsage("alias") - return - else: - incUsage(None) diff --git a/commands/cmd.py b/commands/cmd.py index 6c75ec5..819789b 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -8,14 +8,16 @@ class CmdCommand: def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 4: - if not spl[1] in main.relay.keys(): - failure("No such relay: %s" % spl[1]) + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("No such relay: %s on network: %s" % (spl[2], spl[1])) return - commands = {spl[3]: [" ".join(spl[4:])]} - print(" ".join(spl[4:])) - success("Sending commands to relay %s as user %s" % (spl[1], spl[2])) - deliverRelayCommands(spl[1], commands, user=spl[2]) + commands = {spl[4]: [" ".join(spl[5:])]} + success("Sending commands to relay %s as user %s" % (spl[2], spl[3])) + deliverRelayCommands(main.network[spl[1]].relays[spl[2]], commands, user=spl[3]+"/"+spl[1]) return else: incUsage("cmd") diff --git a/commands/del.py b/commands/del.py index ef6f9ad..eb59eaa 100644 --- a/commands/del.py +++ b/commands/del.py @@ -6,21 +6,25 @@ class DelCommand: def delete(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 2: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + if length == 3: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) return - del main.pool[spl[1]] - if spl[1] in main.ReactorPool.keys(): - if spl[1] in main.FactoryPool.keys(): - main.FactoryPool[spl[1]].stopTrying() - main.ReactorPool[spl[1]].disconnect() - if spl[1] in main.IRCPool.keys(): - del main.IRCPool[spl[1]] - del main.ReactorPool[spl[1]] - del main.FactoryPool[spl[1]] + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("No such relay: %s in network %s" % (spl[2], spl[1])) + return + + main.network[spl[1]].delete_relay(int(spl[2])) + if spl[1]+spl[2] in main.ReactorPool.keys(): + if spl[1]+spl[2] in main.FactoryPool.keys(): + main.FactoryPool[spl[1]+spl[2]].stopTrying() + main.ReactorPool[spl[1]+spl[2]].disconnect() + if spl[1]+spl[2] in main.IRCPool.keys(): + del main.IRCPool[spl[1]+spl[2]] + del main.ReactorPool[spl[1]+spl[2]] + del main.FactoryPool[spl[1]+spl[2]] success("Successfully removed bot: %s" % spl[1]) - main.saveConf("pool") + main.saveConf("network") return else: incUsage("del") diff --git a/commands/disable.py b/commands/disable.py index d9182fd..20ace48 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -7,26 +7,30 @@ class DisableCommand: def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 2: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + if length == 3: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) return - main.pool[spl[1]]["enabled"] = False - user = main.pool[spl[1]]["alias"] - network = main.pool[spl[1]]["network"] - relay = main.pool[spl[1]]["relay"] + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("No such relay: %s in network %s" % (spl[2], spl[1])) + return + + main.network[spl[1]].relays[int(spl[2])]["enabled"] = False + user = main.network[spl[1]].aliases[int(spl[2])] + network = spl[1] + relay = main.network[spl[1]].relays[int(spl[2])] commands = {"status": ["Disconnect"]} deliverRelayCommands(relay, commands, user=user+"/"+network) - main.saveConf("pool") - if spl[1] in main.ReactorPool.keys(): - if spl[1] in main.FactoryPool.keys(): - main.FactoryPool[spl[1]].stopTrying() - main.ReactorPool[spl[1]].disconnect() + main.saveConf("network") + if spl[1]+spl[2] in main.ReactorPool.keys(): + if spl[1]+spl[2] in main.FactoryPool.keys(): + main.FactoryPool[spl[1]+spl[2]].stopTrying() + main.ReactorPool[spl[1]+spl[2]].disconnect() if spl[1] in main.IRCPool.keys(): - del main.IRCPool[spl[1]] - del main.ReactorPool[spl[1]] - del main.FactoryPool[spl[1]] - success("Successfully disabled bot %s" % spl[1]) + del main.IRCPool[spl[1]+spl[2]] + del main.ReactorPool[spl[1]+spl[2]] + del main.FactoryPool[spl[1]+spl[2]] + success("Successfully disabled bot %s on network %s" % (spl[2], spl[1])) return else: incUsage("disable") diff --git a/commands/enable.py b/commands/enable.py index 81f1139..e25857b 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -1,5 +1,4 @@ import main -from core.helper import startBot from core.bot import deliverRelayCommands class EnableCommand: @@ -8,22 +7,26 @@ class EnableCommand: def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 2: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + if length == 3: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) return - main.pool[spl[1]]["enabled"] = True - user = main.pool[spl[1]]["alias"] - network = main.pool[spl[1]]["network"] - relay = main.pool[spl[1]]["relay"] + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("No such relay: %s in network %s" % (spl[2], spl[1])) + return + + main.network[spl[1]].relays[int(spl[2])]["enabled"] = True + user = main.network[spl[1]].aliases[int(spl[2])] + network = spl[1] + relay = main.network[spl[1]].relays[int(spl[2])] commands = {"status": ["Connect"]} deliverRelayCommands(relay, commands, user=user+"/"+network) - main.saveConf("pool") - if not spl[1] in main.IRCPool.keys(): - startBot(spl[1]) + main.saveConf("network") + if not spl[1]+spl[2] in main.IRCPool.keys(): + main.network[spl[1]].start_bot(int(spl[2])) else: pass - success("Successfully enabled bot %s" % spl[1]) + success("Successfully enabled bot %s on network %s" % (spl[2], spl[1])) return else: incUsage("enable") diff --git a/commands/join.py b/commands/join.py index edf6577..2f36ab4 100644 --- a/commands/join.py +++ b/commands/join.py @@ -6,25 +6,31 @@ class JoinCommand: def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 3: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + if length == 4: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) return - if not spl[1] in main.IRCPool.keys(): + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("Relay % does not exist on network %", (spl[2], spl[1])) + return + if not spl[1]+spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - main.IRCPool[spl[1]].join(spl[2]) - success("Joined %s" % spl[2]) + main.IRCPool[spl[1]+spl[2]].join(spl[3]) + success("Joined %s" % spl[3]) return - elif length == 4: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + elif length == 5: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) return - if not spl[1] in main.IRCPool.keys(): + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("Relay % does not exist on network %", (spl[2], spl[1])) + return + if not spl[1]+spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - main.IRCPool[spl[1]].join(spl[2], spl[3]) - success("Joined %s with key %s" % (spl[2], spl[3])) + main.IRCPool[spl[1]+spl[2]].join(spl[3], spl[4]) + success("Joined %s with key %s" % (spl[3], spl[4])) return else: incUsage("join") diff --git a/commands/list.py b/commands/list.py deleted file mode 100644 index 55ba053..0000000 --- a/commands/list.py +++ /dev/null @@ -1,13 +0,0 @@ -import main -from yaml import dump - -class ListCommand: - def __init__(self, *args): - self.list(*args) - - def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - info(dump(main.pool)) - return - else: - incUsage(None) diff --git a/commands/mod.py b/commands/mod.py index 7f5a822..611ab9b 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -2,49 +2,42 @@ import main from yaml import dump class ModCommand: + # This could be greatly improved, but not really important right now def __init__(self, *args): self.mod(*args) def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 2: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - info(dump({spl[1]: main.pool[spl[1]]})) - return - - elif length == 3: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[2] in main.pool[spl[1]].keys(): - failure("No such key: %s" % spl[2]) - return - info("%s: %s" % (spl[2], main.pool[spl[1]][spl[2]])) - return - - elif length == 4: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[2] in main.pool[spl[1]].keys(): - failure("No such key: %s" % spl[2]) + if length == 4: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) return - if spl[3] == main.pool[spl[1]][spl[2]]: - failure("Value already exists: %s" % spl[3]) + try: + setattr(main.network[spl[1]], spl[2], spl[3]) + except e: + failure("Something went wrong.") return - if spl[2] == "enabled": - failure("Use the enable and disable commands to manage this") - return - - main.pool[spl[1]][spl[2]] = spl[3] - main.saveConf("pool") + main.saveConf("network") success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return + elif length == 6: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) + return + if not int(spl[3]) in main.network[spl[1]].relays.keys(): + failure("Relay/alias does not exist: %s" % spl[3]) + return + + try: + x = getattr(main.network[spl[1]], spl[2]) + x[spl[3]] = spl[4] + except e: + failure("Something went wrong.") + return + else: incUsage("mod") return diff --git a/commands/msg.py b/commands/msg.py index 001cbe8..4ea2b2f 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -6,17 +6,20 @@ class MsgCommand: def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length >= 4: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + if length >= 5: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) return - if not spl[1] in main.IRCPool.keys(): + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("Relay % does not exist on network %" % (spl[2], spl[1])) + return + if not spl[1]+spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - if not spl[2] in main.IRCPool[spl[1]].channels: - info("Bot not on channel: %s" % spl[2]) - main.IRCPool[spl[1]].msg(spl[2], " ".join(spl[3:])) - success("Sent %s to %s on %s" % (" ".join(spl[3:]), spl[2], spl[1])) + if not spl[3] in main.IRCPool[spl[1]+spl[2]].channels: + info("Bot not on channel: %s" % spl[3]) + main.IRCPool[spl[1]+spl[2]].msg(spl[3], " ".join(spl[4:])) + success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1])) return else: incUsage("msg") diff --git a/commands/network.py b/commands/network.py index 9f0d67a..a4d04f2 100644 --- a/commands/network.py +++ b/commands/network.py @@ -1,5 +1,6 @@ import main from yaml import dump +from modules.network import Network class NetworkCommand: def __init__(self, *args): @@ -22,10 +23,7 @@ class NetworkCommand: failure("Auth must be sasl, ns or none, not %s" % spl[5]) return else: - main.network[spl[2]] = {"host": spl[3], - "port": spl[4], - "security": spl[5].lower(), - "auth": spl[6].lower()} + main.network[spl[2]] = Network(spl[2], spl[3], spl[4], spl[5].lower(), spl[6].lower()) success("Successfully created network: %s" % spl[2]) main.saveConf("network") return diff --git a/commands/part.py b/commands/part.py index a63a20d..e34d7ff 100644 --- a/commands/part.py +++ b/commands/part.py @@ -6,15 +6,18 @@ class PartCommand: def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 3: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) + if length == 4: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) return - if not spl[1] in main.IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]) + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("Relay % does not exist on network %", (spl[2], spl[1])) return - main.IRCPool[spl[1]].part(spl[2]) - success("Left %s" % spl[2]) + if not spl[1]+spl[2] in main.IRCPool.keys(): + failure("Name has no instance: %s" % spl[1]+spl[2]) + return + main.IRCPool[spl[1]+spl[2]].part(spl[3]) + success("Left %s" % spl[3]) return else: incUsage("part") diff --git a/commands/relay.py b/commands/relay.py index 92ba014..4736950 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -9,40 +9,42 @@ class RelayCommand: if authed: if length == 7: if spl[1] == "add": - if spl[2] in main.relay.keys(): - failure("Relay already exists: %s" % spl[2]) + if spl[2] not in main.network.keys(): + failure("No such network: %s" % spl[2]) return if not spl[4].isdigit(): failure("Port must be an integer, not %s" % spl[4]) return else: - main.relay[spl[2]] = {"host": spl[3], - "port": spl[4], - "user": spl[5], - "password": spl[6]} - success("Successfully created relay: %s" % spl[2]) - main.saveConf("relay") + id, alias = main.network[spl[2]].add_relay(spl[3], spl[4], spl[5], spl[6]) + success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) + main.saveConf("network") return else: incUsage("relay") return - elif length == 3: + elif length == 4: if spl[1] == "del": - if spl[2] in main.relay.keys(): - del main.relay[spl[2]] - success("Successfully removed relay: %s" % spl[2]) - main.saveConf("relay") + if spl[2] not in main.network.keys(): + failure("No such network: %s" % spl[2]) return - else: - failure("No such relay: %s" % spl[2]) + if int(spl[3]) not in main.network[spl[2]].relays.keys(): + failure("No such relay: %s on network %s" % (spl[3], spl[2])) return + main.network[spl[2]].delete_relay(int(spl[3])) + success("Successfully deleted relay %s on network %s" % (spl[3], spl[2])) + main.saveConf("network") + return else: incUsage("relay") return - elif length == 2: + elif length == 3: if spl[1] == "list": - info(dump(main.relay)) + if spl[2] not in main.network.keys(): + failure("No such network: %s" % spl[2]) + return + info(dump(main.network[spl[2]].relays)) return else: incUsage("relay") diff --git a/commands/stats.py b/commands/stats.py index 3281a19..e205d02 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -17,8 +17,7 @@ class StatsCommand: numChannels += len(main.IRCPool[i].channels) numWhoEntries += userinfo.getNumTotalWhoEntries() stats.append("Registered servers:") - stats.append(" Total: %s" % len(main.pool.keys())) - stats.append(" Unique: %s" % len(main.nets())) + stats.append(" Total: %s" % len(main.network.keys())) stats.append("Online servers:") stats.append(" Total: %s" % len(main.IRCPool.keys())) stats.append(" Unique: %s" % len(main.liveNets())) diff --git a/conf/example/alias.json b/conf/example/alias.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/alias.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/example/network.json b/conf/example/network.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/network.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/example/pool.json b/conf/example/pool.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/pool.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/example/relay.json b/conf/example/relay.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/relay.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/help.json b/conf/help.json index b4a3205..5fdaab5 100644 --- a/conf/help.json +++ b/conf/help.json @@ -1,29 +1,27 @@ { "pass": "pass ", "logout": "logout", - "del": "del ", - "mod": "mod [] []", - "get": "get ", + "del": "del ", + "mod": "mod ", + "get": "get ", "key": "key [] [] [] []", "who": "who ", - "join": "join []", - "part": "part ", - "enable": "enable ", - "disable": "disable ", - "list": "list", + "join": "join []", + "part": "part ", + "enable": "enable ", + "disable": "disable ", "stats": "stats []", "save": "save <(file)|list|all>", "load": "load <(file)|list|all>", "dist": "dist", "loadmod": "loadmod ", - "msg": "msg ", + "msg": "msg ", "mon": "mon -h", "chans": "chans [ ...]", "users": "users [ ...]", - "alias": "alias [ OR auto]", - "relay": "relay [ ]", + "relay": "relay [ | | ]", "network": "network [
]", "provision": "provision []", - "cmd": "cmd ", + "cmd": "cmd ", "token": "token [] []" } diff --git a/core/bot.py b/core/bot.py index bb91bc7..def9e2a 100644 --- a/core/bot.py +++ b/core/bot.py @@ -20,13 +20,14 @@ from utils.logging.send import * from twisted.internet.ssl import DefaultOpenSSLContextFactory def deliverRelayCommands(relay, relayCommands, user=None, stage2=None): + # where relay is a dictionary extracted from the Network object keyFN = main.certPath+main.config["Key"] certFN = main.certPath+main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) bot = IRCBotFactory(None, relay, relayCommands, user, stage2) - rct = reactor.connectSSL(main.relay[relay]["host"], - int(main.relay[relay]["port"]), + rct = reactor.connectSSL(relay["host"], + int(relay["port"]), bot, contextFactory) class IRCRelay(IRCClient): @@ -34,10 +35,10 @@ class IRCRelay(IRCClient): self.connected = False self.buffer = "" if user == None: - self.user = main.relay[relay]["user"] + self.user = relay["user"] else: self.user = user - password = main.relay[relay]["password"] + password = relay["password"] self.nickname = self.user self.realname = self.user self.username = self.user @@ -88,7 +89,7 @@ class IRCRelay(IRCClient): return class IRCBot(IRCClient): - def __init__(self, name): + def __init__(self, name, relay): self.connected = False self.channels = [] self.net = "".join([x for x in name if not x in digits]) @@ -96,14 +97,11 @@ class IRCBot(IRCClient): error("Network with all numbers: %s" % name) self.buffer = "" self.name = name - inst = main.pool[name] - alias = main.alias[inst["alias"]] - relay = main.relay[inst["relay"]] - network = main.network[inst["network"]] + alias = relay["alias"] self.nickname = alias["nick"] self.realname = alias["realname"] - self.username = inst["alias"]+"/"+inst["network"] + self.username = alias["nick"]+"/"+relay["net"] self.password = relay["password"] self.userinfo = None self.fingerReply = None @@ -448,7 +446,7 @@ class IRCBotFactory(ReconnectingClientFactory): self.name = name self.net = "".join([x for x in self.name if not x in digits]) else: - self.name = "Relay to "+relay + self.name = "Relay to "+relay["net"]+relay["id"] self.client = None self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"] self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"] @@ -459,7 +457,7 @@ class IRCBotFactory(ReconnectingClientFactory): def buildProtocol(self, addr): if self.relay == None: - entry = IRCBot(self.name) + entry = IRCBot(self.name, self.relay) main.IRCPool[self.name] = entry else: entry = IRCRelay(self.relay, self.relayCommands, self.user, self.stage2) diff --git a/core/helper.py b/core/helper.py deleted file mode 100644 index 3d4dd55..0000000 --- a/core/helper.py +++ /dev/null @@ -1,27 +0,0 @@ -from twisted.internet import reactor -from core.bot import IRCBot, IRCBotFactory -from twisted.internet.ssl import DefaultOpenSSLContextFactory -import main -from utils.logging.log import * - -def startBot(name): - inst = main.pool[name] - relay, alias, network = inst["relay"], inst["alias"], inst["network"] - host = main.relay[relay]["host"] - port = int(main.relay[relay]["port"]) - - log("Started bot %s to %s network %s" % (name, relay, network)) - - keyFN = main.certPath+main.config["Key"] - certFN = main.certPath+main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), - certFN.encode("utf-8", "replace")) - bot = IRCBotFactory(name) - rct = reactor.connectSSL(host, - port, - bot, contextFactory) - - main.ReactorPool[name] = rct - main.FactoryPool[name] = bot - return - diff --git a/main.py b/main.py index 67664df..901c191 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ -from json import load, dump, loads +import json +import pickle from redis import StrictRedis from string import digits from utils.logging.log import * @@ -7,17 +8,17 @@ configPath = "conf/" certPath = "cert/" filemap = { - "config": ["config.json", "configuration"], - "pool": ["pool.json", "network, alias and relay mappings"], - "help": ["help.json", "command help"], - "counters": ["counters.json", "counters file"], - "masterbuf": ["masterbuf.json", "master buffer"], - "monitor": ["monitor.json", "monitoring database"], - "alias": ["alias.json", "alias details"], - "relay": ["relay.json", "relay list"], - "network": ["network.json", "network list"], - "tokens": ["tokens.json", "authentication tokens"], - "aliasdata": ["aliasdata.json", "data for alias generation"] + # JSON configs + "config": ["config.json", "configuration", "json"], + "help": ["help.json", "command help", "json"], + "counters": ["counters.json", "counters file", "json"], + "masterbuf": ["masterbuf.json", "master buffer", "json"], + "monitor": ["monitor.json", "monitoring database", "json"], + "tokens": ["tokens.json", "authentication tokens", "json"], + "aliasdata": ["aliasdata.json", "data for alias generation", "json"], + + # Binary (pickle) configs + "network": ["network.dat", "network list", "pickle"] } connections = {} @@ -48,13 +49,27 @@ def liveNets(): return networks def saveConf(var): - with open(configPath+filemap[var][0], "w") as f: - dump(globals()[var], f, indent=4) - return + if filemap[var][2] == "json": + with open(configPath+filemap[var][0], "w") as f: + json.dump(globals()[var], f, indent=4) + elif filemap[var][2] == "pickle": + with open(configPath+filemap[var][0], "wb") as f: + pickle.dump(globals()[var], f) + else: + raise Exception("invalid format") def loadConf(var): - with open(configPath+filemap[var][0], "r") as f: - globals()[var] = load(f) + if filemap[var][2] == "json": + with open(configPath+filemap[var][0], "r") as f: + globals()[var] = json.load(f) + elif filemap[var][2] == "pickle": + try: + with open(configPath+filemap[var][0], "rb") as f: + globals()[var] = pickle.load(f) + except FileNotFoundError: + globals()[var] = {} + else: + raise Exception("invalid format") def initConf(): for i in filemap.keys(): diff --git a/modules/network.py b/modules/network.py new file mode 100644 index 0000000..5a89126 --- /dev/null +++ b/modules/network.py @@ -0,0 +1,57 @@ +import json +import modules.alias as alias +from twisted.internet import reactor +from core.bot import IRCBot, IRCBotFactory +from twisted.internet.ssl import DefaultOpenSSLContextFactory +import main +from utils.logging.log import * + +class Network: + def __init__(self, name, host, port, security, auth): + self.name = name + self.host = host + self.port = port + self.security = security + self.auth = auth + + self.last = 0 + self.relays = {} + self.aliases = {} + + def add_relay(self, host, port, user, password): + self.last += 1 + self.relays[self.last] = { + "host": host, + "port": port, + "user": user, + "password": password, + "enabled": False, + "net": self.name, + "id": self.last + } + self.aliases[self.last] = alias.generate_alias() + return self.last, self.aliases[self.last]["nick"] + + def delete_relay(self, id): + del self.relays[id] + del self.aliases[id] + + def start_bot(self, relay): + # a single name is given to relays in the backend + # e.g. freenode1 for the first relay on freenode network + name = self.name + relay + keyFN = main.certPath+main.config["Key"] + certFN = main.certPath+main.config["Certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) + bot = IRCBotFactory(name) + rct = reactor.connectSSL(k, port, bot, contextFactory) + + main.ReactorPool[name] = rct + main.FactoryPool[name] = bot + + log("Started bot on relay %s on %s", (relay, self.host)) + + def start_bots(self): + for relay in self.relays: + if relay["enabled"]: + start_bot(relay) diff --git a/threshold b/threshold index 5ff3a3f..b073067 100755 --- a/threshold +++ b/threshold @@ -11,7 +11,6 @@ main.initMain() from utils.logging.log import * from utils.loaders.command_loader import loadCommands -from core.helper import startBot from core.server import Server, ServerFactory from core.relay import Relay, RelayFactory import modules.counters @@ -33,10 +32,7 @@ if __name__ == "__main__": else: reactor.listenTCP(main.config["Relay"]["Port"], relay, interface=main.config["Relay"]["Address"]) log("Threshold relay running on %s:%s" % (main.config["Relay"]["Address"], main.config["Relay"]["Port"])) - for i in main.pool.keys(): - if not "enabled" in main.pool[i]: - continue - if main.pool[i]["enabled"] == True: - startBot(i) + for net in main.network.keys(): + main.network[net].start_bots() modules.counters.setupCounterLoop() reactor.run() From 51b89b9d0590ccc6a6f6bdbf548473872cd14d56 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 11 Aug 2019 22:05:34 +0100 Subject: [PATCH 133/394] Use the ISO format for time --- core/relay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/relay.py b/core/relay.py index 765b783..58ba3d5 100644 --- a/core/relay.py +++ b/core/relay.py @@ -138,5 +138,5 @@ def sendRelayNotification(name, cast): if cast["type"] in main.relayConnections[i].subscriptions: newCast = deepcopy(cast) newCast["name"] = name - newCast["time"] = str(datetime.now()) + newCast["time"] = str(datetime.now().isoformat()) main.relayConnections[i].send(dumps(newCast)) From 1ec0e1f7e608bf71e82c08660f807e978184eed1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 12 Aug 2019 21:03:47 +0100 Subject: [PATCH 134/394] Remove provisioning restrictions, move all user tracking code to monitoring module, fix proper network name not being passed to the relay --- commands/provision.py | 36 +++++++++++++-------------- core/bot.py | 57 ++++++++++++++++--------------------------- core/relay.py | 3 +-- modules/monitor.py | 48 ++++++++++++++++++++++++++---------- modules/provision.py | 30 +++++++++++------------ 5 files changed, 90 insertions(+), 84 deletions(-) diff --git a/commands/provision.py b/commands/provision.py index 3453171..7803b6f 100644 --- a/commands/provision.py +++ b/commands/provision.py @@ -19,22 +19,22 @@ class Provision: failure("No such network: %s" % spl[3]) return - if "users" in main.relay[spl[1]]: - if not spl[2] in main.relay[spl[1]]["users"]: - failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) - return - else: - failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) - return + #if "users" in main.relay[spl[1]]: + # if not spl[2] in main.relay[spl[1]]["users"]: + # failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) + # return + #else: + # failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) + # return rtrn = provision.provisionRelayForNetwork(spl[1], spl[2], spl[3]) - if rtrn == "PROVISIONED": - failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3])) - return - elif rtrn == "DUPLICATE": - failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) - return - elif rtrn: + #if rtrn == "PROVISIONED": + # failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3])) + # return + #elif rtrn == "DUPLICATE": + # failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) + # return + if rtrn: success("Started provisioning network %s on relay %s for alias %s" % (spl[3], spl[1], spl[2])) info("Instance name is %s" % rtrn) return @@ -43,10 +43,10 @@ class Provision: return if length == 3: # provision for relay and alias only rtrn = provision.provisionRelayForAlias(spl[1], spl[2]) - if rtrn == "PROVISIONED": - failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2])) - return - elif rtrn: + #if rtrn == "PROVISIONED": + # failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2])) + # return + if rtrn: success("Started provisioning relay %s for alias %s" % (spl[1], spl[2])) return else: diff --git a/core/bot.py b/core/bot.py index ee5d8b7..ae4a0fe 100644 --- a/core/bot.py +++ b/core/bot.py @@ -139,7 +139,8 @@ class IRCBot(IRCClient): del cast[i] if "muser" in cast.keys(): cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) - del cast["muser"] + #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]: + # del cast["muser"] if set(["nick", "ident", "host", "message"]).issubset(set(cast)): if "message" in cast.keys(): if cast["ident"] == "znc" and cast["host"] == "znc.in": @@ -181,12 +182,10 @@ class IRCBot(IRCClient): castDup["name"] = self.name self.event(**castDup) - if "name" not in cast.keys(): + if not "name" in cast.keys(): cast["name"] = self.net - if cast["type"] in ["msg", "notice", "action"]: - userinfo.editUser(self.net, cast["nick"]+"!"+cast["ident"]+"@"+cast["host"]) count.event(self.net, cast["type"]) - monitor.event(self.net, self.name, cast) + monitor.event(cast) def privmsg(self, user, channel, msg): self.event(type="msg", muser=user, target=channel, message=msg) @@ -214,10 +213,8 @@ class IRCBot(IRCClient): return newnick def nickChanged(self, olduser, newnick): - oldnick, ident, host = self.parsen(olduser) - userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host) self.nickname = newnick - self.event(type="self", mtype="nick", nick=oldnick, ident=ident, host=host, user=newnick) + self.event(type="self", mtype="nick", muser=olduser, user=newnick) def irc_ERR_NICKNAMEINUSE(self, prefix, params): self._attemptedNick = self.alterCollidedNick(self._attemptedNick) @@ -397,59 +394,47 @@ class IRCBot(IRCClient): lc = self._getWho[channel] lc.stop() del self._getWho[channel] - userinfo.delChannel(self.net, channel) - log("Can no longer cover %s, removing records" % channel) - - def left(self, user, channel, message): + userinfo.delChannel(self.net, channel) # < we do not need to deduplicate this + log("Can no longer cover %s, removing records" % channel) # as it will only be matched once -- + # other bots have different nicknames so + def left(self, user, channel, message): # even if they saw it, they wouldn't react self.event(type="part", muser=user, target=channel, message=message) self.botLeft(channel) def userJoined(self, user, channel): - nick, ident, host = self.parsen(user) - userinfo.addUser(self.net, channel, nick, user) - self.event(type="join", nick=nick, ident=ident, host=host, target=channel) + self.event(type="join", muser=user, target=channel) def userLeft(self, user, channel, message): - nick, ident, host = self.parsen(user) - userinfo.delUser(self.net, channel, nick, user) - self.event(type="part", nick=nick, ident=ident, host=host, target=channel, message=message) + self.event(type="part", muser=user, target=channel, message=message) def userQuit(self, user, quitMessage): - nick, ident, host = self.parsen(user) - self.chanlessEvent(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) - userinfo.delUserByNetwork(self.net, nick, user) + self.chanlessEvent(type="quit", muser=user, message=quitMessage) def userKicked(self, kickee, channel, kicker, message): - nick, ident, host = self.parsen(kicker) - userinfo.editUser(self.net, kicker) - userinfo.delUserByNick(self.net, channel, kickee) - if kickee.lower == self.nickname.lower: + if kickee.lower() == self.nickname.lower(): self.botLeft(channel) - self.event(type="kick", nick=nick, ident=ident, host=host, target=channel, message=message, user=kickee) + self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee) def chanlessEvent(self, **cast): chans = userinfo.getChansSingle(self.net, cast["nick"]) if chans == None: self.event(**cast) return + # getChansSingle returns all channels of the user, we only want to use + # ones we have common with them + realChans = set(chans).intersection(set(self.channels)) for i in chans: cast["target"] = i self.event(**cast) def userRenamed(self, oldname, newname): - nick, ident, host = self.parsen(oldname) - self.chanlessEvent(type="nick", nick=nick, ident=ident, host=host, user=newname) - userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) + self.chanlessEvent(type="nick", muser=oldname, user=newname) def topicUpdated(self, user, channel, newTopic): - nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, user) - - self.event(type="topic", nick=nick, ident=ident, host=host, target=channel, message= newTopic) + self.event(type="topic", muser=user, target=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, user) argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): @@ -490,7 +475,7 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s: connection lost: %s" % (self.name, error)) if not self.relay: sendAll("%s: connection lost: %s" % (self.name, error)) - sendRelayNotification(self.name, {"type": "conn", "status": "lost", "message": error}) + sendRelayNotification({"type": "conn", "name": self.name, "status": "lost", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -502,7 +487,7 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s: connection failed: %s" % (self.name, error)) if not self.relay: sendAll("%s: connection failed: %s" % (self.name, error)) - sendRelayNotification(self.name, {"type": "conn", "status": "failed", "message": error}) + sendRelayNotification({"type": "conn", "name": self.name, "status": "failed", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/relay.py b/core/relay.py index 58ba3d5..4067ee0 100644 --- a/core/relay.py +++ b/core/relay.py @@ -132,11 +132,10 @@ class RelayFactory(Factory): else: return -def sendRelayNotification(name, cast): +def sendRelayNotification(cast): for i in main.relayConnections.keys(): if main.relayConnections[i].authed: if cast["type"] in main.relayConnections[i].subscriptions: newCast = deepcopy(cast) - newCast["name"] = name newCast["time"] = str(datetime.now().isoformat()) main.relayConnections[i].send(dumps(newCast)) diff --git a/modules/monitor.py b/modules/monitor.py index 08fae94..16b0bb4 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -3,6 +3,7 @@ from json import dumps import main from core.relay import sendRelayNotification +from modules import userinfo def testNetTarget(name, target): called = False @@ -50,28 +51,49 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(name, numberedName, cast, event=None): - if "target" in cast.keys(): - target = cast["target"] - else: - target = None +def event(c): # yes I'm using a short variable because otherwise it goes off the screen + if not "target" in c.keys(): + c["target"] = None - sendRelayNotification(name, cast) - monitorGroups = testNetTarget(name, target) + # metadata scraping + # need to check if this was received from a relay + # in which case, do not do this + if c["type"] in ["msg", "notice", "action", "topic", "mode"]: + userinfo.editUser(c["name"], c["muser"]) + elif c["type"] == "nick": + userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + elif c["type"] == "kick": + userinfo.editUser(c["name"], c["muser"]) + userinfo.delUserByNick(c["name"], c["target"], c["user"]) + elif c["type"] == "quit": + userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) + elif c["type"] == "join": + userinfo.addUser(c["name"], c["target"], c["nick"], c["user"]) + elif c["type"] == "part": + userinfo.delUser(c["name"], c["target"], c["nick"], c["user"]) + + if "mtype" in c.keys(): + if c["mtype"] == "nick": + userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + + if "muser" in c.keys(): + del c["muser"] + sendRelayNotification(c) + monitorGroups = testNetTarget(c["name"], c["target"]) if monitorGroups == False: return for monitorGroup in monitorGroups: - matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup])) + matcher = magicFunction(deepcopy(c), deepcopy(main.monitor[monitorGroup])) if matcher == True: - cast["monitor"] = True + c["monitor"] = True if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): if isinstance(main.monitor[monitorGroup]["send"][i], bool): - sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "errdeliv"}) + sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "errdeliv"}) continue if not i in main.pool.keys(): - sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noname"}) + sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noname"}) if not i in main.IRCPool.keys(): - sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noinstance"}) + sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noinstance"}) for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, name, cast)) + main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["name"], c)) diff --git a/modules/provision.py b/modules/provision.py index 0eb7464..f964cf3 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -48,40 +48,40 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass return def provisionRelayForAlias(relay, alias): - if "users" in main.relay[relay].keys(): - if alias in main.relay[relay]["users"]: - return "PROVISIONED" - else: - main.relay[relay]["users"] = [] - main.relay[relay]["users"].append(alias) + #if "users" in main.relay[relay].keys(): + # if alias in main.relay[relay]["users"]: + # return "PROVISIONED" + #else: + # main.relay[relay]["users"] = [] + #main.relay[relay]["users"].append(alias) provisionUserData(relay, alias, main.alias[alias]["nick"], main.alias[alias]["altnick"], main.alias[alias]["ident"], main.alias[alias]["realname"], main.relay[relay]["password"]) - main.saveConf("relay") + #main.saveConf("relay") return True def provisionRelayForNetwork(relay, alias, network): - if set(["users", "networks"]).issubset(main.relay[relay].keys()): - if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: - return "PROVISIONED" - else: - main.relay[relay]["networks"] = [] - main.relay[relay]["networks"].append(network) + #if set(["users", "networks"]).issubset(main.relay[relay].keys()): + # if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: + # return "PROVISIONED" + #else: + # main.relay[relay]["networks"] = [] + #main.relay[relay]["networks"].append(network) provisionNetworkData(relay, alias, network, main.network[network]["host"], main.network[network]["port"], main.network[network]["security"], main.network[network]["auth"], main.alias[alias]["password"]) - main.saveConf("relay") + #main.saveConf("relay") storedNetwork = False num = 1 while not storedNetwork: i = str(num) if num == 1000: - error("Too many iterations in while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network)) + error("Iteration limit exceeded while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network)) return False if network+i in main.pool.keys(): From f34ddab6fc5b8eeecc47b4dd477926a9bc05447c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 15 Aug 2019 21:20:49 +0100 Subject: [PATCH 135/394] Improvements to query and self event detection, implement all command and debug flags --- commands/all.py | 23 ++++++++++++++++++++++ conf/example/config.json | 2 ++ conf/help.json | 3 ++- core/bot.py | 35 +++++++++++++++++++-------------- main.py | 6 ++++++ modules/monitor.py | 12 ++++++++--- modules/provision.py | 3 ++- modules/userinfo.py | 7 +++++++ threshold | 4 +++- utils/dedup.py | 21 ++++++++++++++++++++ utils/loaders/command_loader.py | 2 +- utils/logging/debug.py | 7 +++++++ utils/logging/log.py | 3 --- 13 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 commands/all.py create mode 100644 utils/dedup.py create mode 100644 utils/logging/debug.py diff --git a/commands/all.py b/commands/all.py new file mode 100644 index 0000000..10ec381 --- /dev/null +++ b/commands/all.py @@ -0,0 +1,23 @@ +import main +from core.bot import deliverRelayCommands + +class All: + def __init__(self, *args): + self.all(*args) + + def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 2: + for i in main.pool.keys(): + relay = main.pool[i]["relay"] + network = main.pool[i]["network"] + alias = main.pool[i]["alias"] + commands = {spl[1]: [" ".join(spl[2:])]} + success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) + deliverRelayCommands(relay, commands, user=alias+"/"+network) + return + else: + incUsage("all") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index bc67bdf..5ecb53f 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -15,6 +15,7 @@ "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, + "Debug": false, "Dist": { "Enabled": true, "SendOutput": false, @@ -26,6 +27,7 @@ }, "Password": "s", "Tweaks": { + "MaxHash": 10, "ZNC": { "Prefix": "*" }, diff --git a/conf/help.json b/conf/help.json index 0211171..522f00e 100644 --- a/conf/help.json +++ b/conf/help.json @@ -24,5 +24,6 @@ "network": "network [
]", "provision": "provision []", "cmd": "cmd ", - "token": "token [] []" + "token": "token [] []", + "all": "all " } diff --git a/core/bot.py b/core/bot.py index ae4a0fe..33a20e9 100644 --- a/core/bot.py +++ b/core/bot.py @@ -9,10 +9,11 @@ from random import randint from copy import deepcopy from modules import userinfo -from modules import counters as count +from modules import counters from modules import monitor from core.relay import sendRelayNotification +from utils.dedup import dedup import main from utils.logging.log import * @@ -151,9 +152,9 @@ class IRCBot(IRCClient): del cast["host"] del cast["target"] if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: - if "target" in cast.keys(): - if cast["target"].lower() == self.nickname.lower(): - #castDup = deepcopy(cast) + if "target" in cast.keys() and not cast["type"] == "mode": # don't handle modes here + if cast["target"].lower() == self.nickname.lower(): # as they are target == nickname + #castDup = deepcopy(cast) # however modes are not queries! cast["mtype"] = cast["type"] cast["type"] = "query" cast["name"] = self.name @@ -173,7 +174,8 @@ class IRCBot(IRCClient): castDup["mtype"] = cast["type"] castDup["type"] = "self" castDup["name"] = self.name - self.event(**castDup) + if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly + self.event(**castDup) # don't tell anyone else if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries if self.nickname.lower() in cast["message"].lower(): castDup = deepcopy(cast) @@ -184,8 +186,8 @@ class IRCBot(IRCClient): if not "name" in cast.keys(): cast["name"] = self.net - count.event(self.net, cast["type"]) - monitor.event(cast) + counters.event(self.net, cast["type"]) + monitor.event(self.name, cast) def privmsg(self, user, channel, msg): self.event(type="msg", muser=user, target=channel, message=msg) @@ -408,37 +410,40 @@ class IRCBot(IRCClient): self.event(type="part", muser=user, target=channel, message=message) def userQuit(self, user, quitMessage): - self.chanlessEvent(type="quit", muser=user, message=quitMessage) + self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): if kickee.lower() == self.nickname.lower(): self.botLeft(channel) self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee) - def chanlessEvent(self, **cast): - chans = userinfo.getChansSingle(self.net, cast["nick"]) + def chanlessEvent(self, cast): + cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) + if dedup(self.name, cast): + return # stop right there sir! + chans = userinfo.getChanList(self.net, cast["nick"]) if chans == None: - self.event(**cast) + error("No channels returned for chanless event: %s" % cast) + # self.event(**cast) -- no, should NEVER happen return # getChansSingle returns all channels of the user, we only want to use # ones we have common with them realChans = set(chans).intersection(set(self.channels)) - for i in chans: + for i in realChans: cast["target"] = i self.event(**cast) def userRenamed(self, oldname, newname): - self.chanlessEvent(type="nick", muser=oldname, user=newname) + self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname}) def topicUpdated(self, user, channel, newTopic): self.event(type="topic", muser=user, target=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): - nick, ident, host = self.parsen(user) argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a) + self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): diff --git a/main.py b/main.py index 1aa0cdf..ccb6942 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,8 @@ from json import load, dump, loads from redis import StrictRedis from string import digits +from os import urandom + from utils.logging.log import * configPath = "conf/" @@ -31,6 +33,10 @@ CommandMap = {} runningSample = 0 lastMinuteSample = 0 +# Generate 16-byte hex key for message checksums +hashKey = urandom(16) +lastEvents = {} + def nets(): if not "pool" in globals(): return diff --git a/modules/monitor.py b/modules/monitor.py index 16b0bb4..3913c6f 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,9 +1,11 @@ from copy import deepcopy from json import dumps +from datetime import datetime import main from core.relay import sendRelayNotification from modules import userinfo +from utils.dedup import dedup def testNetTarget(name, target): called = False @@ -51,10 +53,12 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(c): # yes I'm using a short variable because otherwise it goes off the screen +def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen if not "target" in c.keys(): c["target"] = None + if dedup(numName, c): + return # metadata scraping # need to check if this was received from a relay # in which case, do not do this @@ -68,9 +72,9 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the elif c["type"] == "quit": userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) elif c["type"] == "join": - userinfo.addUser(c["name"], c["target"], c["nick"], c["user"]) + userinfo.addUser(c["name"], c["target"], c["nick"], c["muser"]) elif c["type"] == "part": - userinfo.delUser(c["name"], c["target"], c["nick"], c["user"]) + userinfo.delUser(c["name"], c["target"], c["nick"], c["muser"]) if "mtype" in c.keys(): if c["mtype"] == "nick": @@ -79,6 +83,8 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the if "muser" in c.keys(): del c["muser"] sendRelayNotification(c) + + # only monitors below monitorGroups = testNetTarget(c["name"], c["target"]) if monitorGroups == False: return diff --git a/modules/provision.py b/modules/provision.py index f964cf3..fb3ea9c 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -34,7 +34,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass elif auth == "ns": stage2commands["status"] = [] stage2commands["nickserv"] = [] - stage2commands["status"].append("LoadMod NickServ") + stage2commands["status"].append("LoadMod nickserv") stage2commands["nickserv"].append("Set %s" % password) if not main.config["ConnectOnCreate"]: stage3commands["status"] = [] @@ -42,6 +42,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass if main.config["Toggles"]["CycleChans"]: stage2commands["status"] = [] stage2commands["status"].append("LoadMod disconkick") + stage2commands["status"].append("LoadMod chansaver") deliverRelayCommands(relay, commands, stage2=[[alias+"/"+network, stage2commands], [alias+"/"+network, stage3commands]]) diff --git a/modules/userinfo.py b/modules/userinfo.py index d9bdddf..00020cd 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -23,6 +23,13 @@ def getChansSingle(name, nick): return None return [i.decode() for i in result] +def getChanList(name, nick): + chanspace = "live.chan."+name+"."+nick + result = main.r.smembers(chanspace) + if len(result) == 0: + return None + return [i.decode() for i in result] + def getChans(nick): result = {} for i in main.nets(): diff --git a/threshold b/threshold index 5ff3a3f..648ed4f 100755 --- a/threshold +++ b/threshold @@ -1,6 +1,7 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory +from sys import argv #from twisted.python import log #from sys import stdout #log.startLogging(stdout) @@ -8,7 +9,8 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory import main main.initMain() - +if "--debug" in argv: # yes really + main.config["Debug"] = True from utils.logging.log import * from utils.loaders.command_loader import loadCommands from core.helper import startBot diff --git a/utils/dedup.py b/utils/dedup.py new file mode 100644 index 0000000..ee07bb5 --- /dev/null +++ b/utils/dedup.py @@ -0,0 +1,21 @@ +from datetime import datetime +from csiphash import siphash24 +from json import dumps +import main +from utils.logging.debug import debug + +def dedup(numName, c): + # deduplication + c["approxtime"] = int(datetime.utcnow().timestamp()) + castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) + del c["approxtime"] + isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) + if isDuplicate: + debug("Duplicate: %s" % (c)) + return True + if numName in main.lastEvents.keys(): + main.lastEvents[numName].insert(0, castHash) + main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]] + else: + main.lastEvents[numName] = [castHash] + return False diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index e3ea253..ad623cc 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,6 +1,6 @@ from os import listdir -from utils.logging.log import * +from utils.logging.debug import debug import commands from main import CommandMap diff --git a/utils/logging/debug.py b/utils/logging/debug.py new file mode 100644 index 0000000..5a7b067 --- /dev/null +++ b/utils/logging/debug.py @@ -0,0 +1,7 @@ +import main +# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main +# to read the setting +def debug(data): + if main.config["Debug"]: + print("[DEBUG]", data) + diff --git a/utils/logging/log.py b/utils/logging/log.py index 5e1141f..6bb6a99 100644 --- a/utils/logging/log.py +++ b/utils/logging/log.py @@ -1,9 +1,6 @@ def log(data): print("[LOG]", data) -def debug(data): - print("[DEBUG]", data) - def warn(data): print("[WARNING]", data) From 22bd0d3ac61107eb2a2f8674f2785b9c74b8eba1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 15 Aug 2019 22:14:45 +0100 Subject: [PATCH 136/394] Don't crash if the part message is null --- core/bot.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/core/bot.py b/core/bot.py index 33a20e9..09f3ce7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -175,14 +175,15 @@ class IRCBot(IRCClient): castDup["type"] = "self" castDup["name"] = self.name if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly - self.event(**castDup) # don't tell anyone else + self.event(**castDup) # don't tell anyone else if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries - if self.nickname.lower() in cast["message"].lower(): - castDup = deepcopy(cast) - castDup["mtype"] = cast["type"] - castDup["type"] = "highlight" - castDup["name"] = self.name - self.event(**castDup) + if not cast["message"] == None: + if self.nickname.lower() in cast["message"].lower(): + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "highlight" + castDup["name"] = self.name + self.event(**castDup) if not "name" in cast.keys(): cast["name"] = self.net @@ -380,8 +381,8 @@ class IRCBot(IRCClient): if not channel in self.channels: self.channels.append(channel) self.names(channel).addCallback(self.got_names) - self.who(channel).addCallback(self.got_who) if main.config["Toggles"]["Who"]: + self.who(channel).addCallback(self.got_who) lc = LoopingCall(self.who, channel) self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] From 545282e201906b0e33f94a2e28c9a65a61df21fc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 16 Aug 2019 21:27:23 +0100 Subject: [PATCH 137/394] Add deduplication precision toggle, fix printing odd characters and implement sending messages to all instances of a certain network, or all networks associated with a certain alias --- commands/allc.py | 46 ++++++++++++++++++++++++++++++++++++++++ conf/example/config.json | 1 + conf/help.json | 3 ++- threshold | 7 ++++-- utils/dedup.py | 2 +- 5 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 commands/allc.py diff --git a/commands/allc.py b/commands/allc.py new file mode 100644 index 0000000..9e662fe --- /dev/null +++ b/commands/allc.py @@ -0,0 +1,46 @@ +import main +from core.bot import deliverRelayCommands + +class Allc: + def __init__(self, *args): + self.allc(*args) + + def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 4: + targets = [] + if spl[1] == "network": + if spl[2] in main.network.keys(): + for i in main.pool.keys(): + if main.pool[i]["network"] == spl[2]: + targets.append(i) + else: + failure("No such network: %s" % spl[2]) + return + elif spl[1] == "alias": + if spl[2] in main.alias.keys(): + for i in main.pool.keys(): + if main.pool[i]["alias"] == spl[2]: + targets.append(i) + else: + failure("No such alias: %s" % spl[2]) + return + else: + incUsage("allc") + return + if len(targets) == 0: + failure("No matches found: %s" % spl[2]) + return + for i in targets: + relay = main.pool[i]["relay"] + network = main.pool[i]["network"] + alias = main.pool[i]["alias"] + commands = {spl[3]: [" ".join(spl[4:])]} + success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) + deliverRelayCommands(relay, commands, user=alias+"/"+network) + return + else: + incUsage("allc") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index 5ecb53f..a595963 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -28,6 +28,7 @@ "Password": "s", "Tweaks": { "MaxHash": 10, + "DedupPrecision": 9, "ZNC": { "Prefix": "*" }, diff --git a/conf/help.json b/conf/help.json index 522f00e..1e55f3a 100644 --- a/conf/help.json +++ b/conf/help.json @@ -25,5 +25,6 @@ "provision": "provision []", "cmd": "cmd ", "token": "token [] []", - "all": "all " + "all": "all ", + "allc": "allc <(network)|(alias)> " } diff --git a/threshold b/threshold index 648ed4f..d63a8b2 100755 --- a/threshold +++ b/threshold @@ -1,11 +1,14 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory -from sys import argv +from sys import argv, stdout, stderr #from twisted.python import log #from sys import stdout #log.startLogging(stdout) - +from codecs import getwriter # fix printing odd shit to the terminal +stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know +stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using + # unicode quit messages for no reason import main main.initMain() diff --git a/utils/dedup.py b/utils/dedup.py index ee07bb5..03255e3 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -6,7 +6,7 @@ from utils.logging.debug import debug def dedup(numName, c): # deduplication - c["approxtime"] = int(datetime.utcnow().timestamp()) + c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) From c63f301b7f785971b121c12aa040535169c60659 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 19 Aug 2019 20:12:42 +0100 Subject: [PATCH 138/394] Defer initialUsers, initialNames and delChannel to threads to improve performance --- core/bot.py | 2 +- modules/userinfo.py | 31 +++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/core/bot.py b/core/bot.py index 09f3ce7..0f41dfe 100644 --- a/core/bot.py +++ b/core/bot.py @@ -398,7 +398,7 @@ class IRCBot(IRCClient): lc.stop() del self._getWho[channel] userinfo.delChannel(self.net, channel) # < we do not need to deduplicate this - log("Can no longer cover %s, removing records" % channel) # as it will only be matched once -- + #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so def left(self, user, channel, message): # even if they saw it, they wouldn't react self.event(type="part", muser=user, target=channel, message=message) diff --git a/modules/userinfo.py b/modules/userinfo.py index 00020cd..29da9b0 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,6 +1,9 @@ -import main +from twisted.internet.threads import deferToThread from string import digits + +import main from utils.logging.log import * +from utils.logging.debug import debug def getWhoSingle(name, query): result = main.r.sscan("live.who."+name, 0, query, count=9999999) @@ -68,14 +71,19 @@ def getNamespace(name, channel, nick): chanspace = "live.chan.%s.%s" % (name, nick) return [gnamespace, namespace, chanspace] -def initialUsers(name, channel, users): +def _initialUsers(name, channel, users): gnamespace = "live.who.%s" % name p = main.r.pipeline() for i in users: p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2]) p.execute() -def initialNames(name, channel, names): +def initialUsers(name, channel, users): + debug("Initialising WHO records for %s on %s" % (channel, name)) + d = deferToThread(_initialUsers, name, channel, users) + #d.addCallback(testCallback) + +def _initialNames(name, channel, names): namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() for i in names: @@ -83,6 +91,11 @@ def initialNames(name, channel, names): p.sadd("live.chan."+name+"."+i, channel) p.execute() +def initialNames(name, channel, names): + debug("Initialising NAMES records for %s on %s" % (channel, name)) + d = deferToThread(_initialNames, name, channel, names) + #d.addCallback(testCallback) + def editUser(name, user): gnamespace = "live.who.%s" % name main.r.sadd(gnamespace, user) @@ -158,7 +171,7 @@ def delUserByNetwork(name, nick, user): p.delete(chanspace) p.execute() -def delChannel(name, channel): # This function is extremely expensive, look to replace +def _delChannel(name, channel): # This function is extremely expensive, look to replace gnamespace = "live.who.%s" % name namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() @@ -173,10 +186,16 @@ def delChannel(name, channel): # This function is extremely expensive, look to r p.srem("live.chan."+name+"."+i.decode(), channel) p.delete(namespace) p.execute() + return [name, channel] + +def delChannel(name, channel): + debug("Purging channel %s for %s" % (channel, name)) + d = deferToThread(_delChannel, name, channel) + #d.addCallback(testCallback) def delNetwork(name, channels): - log("Purging channels for %s" % name) + debug("Purging channels for %s" % name) for i in channels: delChannel(name, i) - log("Finished purging channels for %s" % name) + #log("Finished purging channels for %s" % name) return From 2d70d5af115a89b88e712bc36a489e8b03ac1a24 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 25 Aug 2019 21:29:11 +0100 Subject: [PATCH 139/394] Add error checking in places, set up automatic relay provisioning and fix starting bots --- commands/auto.py | 29 ++++++++++ commands/cmd.py | 12 ++-- commands/enable.py | 10 ++-- commands/get.py | 22 -------- commands/join.py | 2 +- commands/network.py | 2 +- commands/provision.py | 60 -------------------- commands/relay.py | 52 +++++++++-------- commands/swho.py | 40 ++++++++++++++ conf/example/config.json | 8 ++- conf/help.json | 22 ++++---- conf/masterbuf.json | 1 - core/bot.py | 72 +++++++++++++----------- core/relay.py | 2 +- modules/network.py | 47 ++++++++-------- modules/provision.py | 98 ++++++++++----------------------- threshold | 12 ++-- utils/getrelay.py | 12 ++++ utils/loaders/command_loader.py | 1 + 19 files changed, 242 insertions(+), 262 deletions(-) create mode 100644 commands/auto.py delete mode 100644 commands/get.py delete mode 100644 commands/provision.py create mode 100644 commands/swho.py delete mode 100644 conf/masterbuf.json create mode 100644 utils/getrelay.py diff --git a/commands/auto.py b/commands/auto.py new file mode 100644 index 0000000..3d29a85 --- /dev/null +++ b/commands/auto.py @@ -0,0 +1,29 @@ +import main +from modules import provision + +class AutoCommand: + def __init__(self, *args): + self.auto(*args) + + def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 3: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + if not spl[2].isdigit(): + failure("Must be integer, not %s" % spl[2]) + return + + id, alias = main.network[spl[1]].add_relay(int(spl[2])) + success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[1], alias)) + main.saveConf("network") + rtrn = provision.provisionRelay(int(spl[2]), spl[1]) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], rtrn)) + return + + else: + incUsage("auto") + return + else: + incUsage(None) diff --git a/commands/cmd.py b/commands/cmd.py index 819789b..858dc15 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -8,16 +8,12 @@ class CmdCommand: def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 4: - if not spl[1] in main.network.keys(): - failure("No such network: %s" % spl[1]) + if not spl[1].isdigit(): + failure("Must be integer, not %s" % spl[1]) return - if not int(spl[2]) in main.network[spl[1]].relays.keys(): - failure("No such relay: %s on network: %s" % (spl[2], spl[1])) - return - - commands = {spl[4]: [" ".join(spl[5:])]} + commands = {spl[3]: [" ".join(spl[4:])]} success("Sending commands to relay %s as user %s" % (spl[2], spl[3])) - deliverRelayCommands(main.network[spl[1]].relays[spl[2]], commands, user=spl[3]+"/"+spl[1]) + deliverRelayCommands(int(spl[1]), commands, user=spl[2]) return else: incUsage("cmd") diff --git a/commands/enable.py b/commands/enable.py index e25857b..0e88dce 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -11,16 +11,18 @@ class EnableCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return if not int(spl[2]) in main.network[spl[1]].relays.keys(): - failure("No such relay: %s in network %s" % (spl[2], spl[1])) + failure("No such relay on %s: %s" % (spl[2], spl[1])) return main.network[spl[1]].relays[int(spl[2])]["enabled"] = True - user = main.network[spl[1]].aliases[int(spl[2])] + user = main.network[spl[1]].aliases[int(spl[2])]["nick"] network = spl[1] - relay = main.network[spl[1]].relays[int(spl[2])] commands = {"status": ["Connect"]} - deliverRelayCommands(relay, commands, user=user+"/"+network) + deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network) main.saveConf("network") if not spl[1]+spl[2] in main.IRCPool.keys(): main.network[spl[1]].start_bot(int(spl[2])) diff --git a/commands/get.py b/commands/get.py deleted file mode 100644 index b56f5dc..0000000 --- a/commands/get.py +++ /dev/null @@ -1,22 +0,0 @@ -import main - -class GetCommand: - def __init__(self, *args): - self.get(*args) - - def get(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - if length == 3: - if not spl[1] in main.pool.keys(): - failure("Name does not exist: %s" % spl[1]) - return - if not spl[1] in main.IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]) - return - info(str(main.IRCPool[spl[1]].get(spl[2]))) - return - else: - incUsage("get") - return - else: - incUsage(None) diff --git a/commands/join.py b/commands/join.py index 2f36ab4..ff912bc 100644 --- a/commands/join.py +++ b/commands/join.py @@ -11,7 +11,7 @@ class JoinCommand: failure("Network does not exist: %s" % spl[1]) return if not int(spl[2]) in main.network[spl[1]].relays.keys(): - failure("Relay % does not exist on network %", (spl[2], spl[1])) + failure("Relay %s does not exist on network %s" % (spl[2], spl[1])) return if not spl[1]+spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) diff --git a/commands/network.py b/commands/network.py index a4d04f2..b9cfbfd 100644 --- a/commands/network.py +++ b/commands/network.py @@ -23,7 +23,7 @@ class NetworkCommand: failure("Auth must be sasl, ns or none, not %s" % spl[5]) return else: - main.network[spl[2]] = Network(spl[2], spl[3], spl[4], spl[5].lower(), spl[6].lower()) + main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()) success("Successfully created network: %s" % spl[2]) main.saveConf("network") return diff --git a/commands/provision.py b/commands/provision.py deleted file mode 100644 index 459efd8..0000000 --- a/commands/provision.py +++ /dev/null @@ -1,60 +0,0 @@ -import main -from modules import provision - -class ProvisionCommand: - def __init__(self, *args): - self.provision(*args) - - def provision(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - if length == 4 or length == 3: - if not spl[1] in main.relay.keys(): - failure("No such relay: %s" % spl[1]) - return - if not spl[2] in main.alias.keys(): - failure("No such alias: %s" % spl[2]) - return - if length == 4: # provision for relay, alias and network - if not spl[3] in main.network.keys(): - failure("No such network: %s" % spl[3]) - return - - #if "users" in main.relay[spl[1]]: - # if not spl[2] in main.relay[spl[1]]["users"]: - # failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) - # return - #else: - # failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) - # return - - rtrn = provision.provisionRelayForNetwork(spl[1], spl[2], spl[3]) - #if rtrn == "PROVISIONED": - # failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3])) - # return - #elif rtrn == "DUPLICATE": - # failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) - # return - if rtrn: - success("Started provisioning network %s on relay %s for alias %s" % (spl[3], spl[1], spl[2])) - info("Instance name is %s" % rtrn) - return - else: - failure("Failure while provisioning relay %s" % spl[1]) - return - if length == 3: # provision for relay and alias only - rtrn = provision.provisionRelayForAlias(spl[1], spl[2]) - #if rtrn == "PROVISIONED": - # failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2])) - # return - if rtrn: - success("Started provisioning relay %s for alias %s" % (spl[1], spl[2])) - return - else: - failure("Failure while provisioning relay %s" % spl[1]) - return - - else: - incUsage("provision") - return - else: - incUsage(None) diff --git a/commands/relay.py b/commands/relay.py index 4736950..49dca97 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -7,29 +7,47 @@ class RelayCommand: def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 7: + if length == 3: if spl[1] == "add": - if spl[2] not in main.network.keys(): - failure("No such network: %s" % spl[2]) - return - if not spl[4].isdigit(): - failure("Port must be an integer, not %s" % spl[4]) - return - else: - id, alias = main.network[spl[2]].add_relay(spl[3], spl[4], spl[5], spl[6]) + if spl[2] in main.network.keys(): + id, alias = main.network[spl[2]].add_relay() success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) main.saveConf("network") return + else: + failure("No such network: %s" % spl[2]) + return + elif spl[1] == "list": + if spl[2] not in main.network.keys(): + failure("No such network: %s" % spl[2]) + return + info(dump(main.network[spl[2]].relays)) + return else: incUsage("relay") return elif length == 4: - if spl[1] == "del": - if spl[2] not in main.network.keys(): + if spl[1] == "add": + if spl[2] in main.network.keys(): + if not spl[3].isdigit(): + failure("Must be a number, not %s" % spl[3]) + return + id, alias = main.network[spl[2]].add_relay(int(spl[3])) + success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) + main.saveConf("network") + return + else: failure("No such network: %s" % spl[2]) return - if int(spl[3]) not in main.network[spl[2]].relays.keys(): + elif spl[1] == "del": + if not spl[2] in main.network.keys(): + failure("No such network: %s" % spl[2]) + return + if not spl[3].isdigit(): + failure("Must be a number, not %s" % spl[3]) + return + if not int(spl[3]) in main.network[spl[2]].relays.keys(): failure("No such relay: %s on network %s" % (spl[3], spl[2])) return main.network[spl[2]].delete_relay(int(spl[3])) @@ -39,16 +57,6 @@ class RelayCommand: else: incUsage("relay") return - elif length == 3: - if spl[1] == "list": - if spl[2] not in main.network.keys(): - failure("No such network: %s" % spl[2]) - return - info(dump(main.network[spl[2]].relays)) - return - else: - incUsage("relay") - return else: incUsage("relay") return diff --git a/commands/swho.py b/commands/swho.py new file mode 100644 index 0000000..b03ca61 --- /dev/null +++ b/commands/swho.py @@ -0,0 +1,40 @@ +import main + +class SwhoCommand: + def __init__(self, *args): + self.swho(*args) + + def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) + return + for i in main.IRCPool.keys(): + if spl[1] in i: + for x in main.IRCPool[i].channels: + main.IRCPool[i].who(x) + success("Sent WHO to all channels on all networks on %s" % spl[1]) + return + elif length == 3: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) + return + matches = [] + for i in main.IRCPool.keys(): + if spl[1] in i: + for x in main.IRCPool[i].channels: + if x == spl[2]: + main.IRCPool[i].who(x) + matches.append(i) + if matches == []: + failure("No matches found for channel %s" % spl[2]) + return + success("Sent WHO to %s on: %s" % (spl[2], ", ".join(matches))) + return + + else: + incUsage("swho") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index a595963..0e6117d 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -4,7 +4,7 @@ "Address": "127.0.0.1", "UseSSL": true }, - "Relay": { + "RelayAPI": { "Enabled": true, "Port": 13868, "Address": "127.0.0.1", @@ -16,6 +16,12 @@ "UsePassword": true, "ConnectOnCreate": false, "Debug": false, + "Relay": { + "Host": "127.0.0.1", + "Port": "201x", + "User": "sir", + "Password": "sir" + }, "Dist": { "Enabled": true, "SendOutput": false, diff --git a/conf/help.json b/conf/help.json index 2f81c57..e58bfe1 100644 --- a/conf/help.json +++ b/conf/help.json @@ -1,27 +1,27 @@ { "pass": "pass ", "logout": "logout", - "del": "del ", - "mod": "mod ", - "get": "get ", + "del": "del ", + "mod": "mod [] []", "who": "who ", - "join": "join []", - "part": "part ", - "enable": "enable ", - "disable": "disable ", + "join": "join []", + "part": "part ", + "enable": "enable ", + "disable": "disable ", + "list": "list", "stats": "stats []", "save": "save <(file)|list|all>", "load": "load <(file)|list|all>", "dist": "dist", "loadmod": "loadmod ", - "msg": "msg ", + "msg": "msg ", "mon": "mon -h", "chans": "chans [ ...]", "users": "users [ ...]", - "relay": "relay [ | | ]", + "relay": "relay [] []", "network": "network [
]", - "provision": "provision []", - "cmd": "cmd ", + "auto": "auto ", + "cmd": "cmd ", "token": "token [] []", "all": "all ", "allc": "allc <(network)|(alias)> " diff --git a/conf/masterbuf.json b/conf/masterbuf.json deleted file mode 100644 index fe51488..0000000 --- a/conf/masterbuf.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/core/bot.py b/core/bot.py index 701714f..52c8044 100644 --- a/core/bot.py +++ b/core/bot.py @@ -14,6 +14,7 @@ from modules import monitor from core.relay import sendRelayNotification from utils.dedup import dedup +from utils.getrelay import getRelay import main from utils.logging.log import * @@ -21,33 +22,34 @@ from utils.logging.send import * from twisted.internet.ssl import DefaultOpenSSLContextFactory -def deliverRelayCommands(relay, relayCommands, user=None, stage2=None): +def deliverRelayCommands(num, relayCommands, user=None, stage2=None): # where relay is a dictionary extracted from the Network object keyFN = main.certPath+main.config["Key"] certFN = main.certPath+main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - bot = IRCBotFactory(None, relay, relayCommands, user, stage2) - rct = reactor.connectSSL(relay["host"], - int(relay["port"]), + bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) + host, port = getRelay(num) + rct = reactor.connectSSL(host, + port, bot, contextFactory) class IRCRelay(IRCClient): - def __init__(self, relay, relayCommands, user, stage2): + def __init__(self, num, relayCommands, user, stage2): self.connected = False self.buffer = "" if user == None: - self.user = relay["user"] + self.user = main.config["Relay"]["User"] else: self.user = user - password = relay["password"] - self.nickname = self.user - self.realname = self.user + password = main.config["Relay"]["Password"] + self.nickname = "relay" + self.realname = "relay" self.username = self.user self.password = self.user+":"+password self.relayCommands = relayCommands - self.relay = relay + self.num = num self.stage2 = stage2 def parsen(self, user): @@ -67,15 +69,16 @@ class IRCRelay(IRCClient): if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]: nick = nick[1:] if nick in self.relayCommands.keys(): - sendAll("[%s] %s -> %s" % (self.relay, nick, msg)) + sendAll("[%s] %s -> %s" % (self.num, nick, msg)) def irc_ERR_PASSWDMISMATCH(self, prefix, params): - log("%s: relay password mismatch" % self.relay) - sendAll("%s: relay password mismatch" % self.relay) + print(', '.join("%s: %s" % item for item in vars(self).items())) + log("%s: relay password mismatch" % self.num) + sendAll("%s: relay password mismatch" % self.num) def signedOn(self): self.connected = True - log("signed on as a relay: %s" % self.relay) + log("signed on as a relay: %s" % self.num) #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares for i in self.relayCommands.keys(): for x in self.relayCommands[i]: @@ -85,25 +88,24 @@ class IRCRelay(IRCClient): user = self.stage2[0].pop(0) commands = self.stage2[0].pop(0) del self.stage2[0] - deliverRelayCommands(self.relay, commands, user, self.stage2) + deliverRelayCommands(self.num, commands, user, self.stage2) deferLater(reactor, 1, self.transport.loseConnection) return class IRCBot(IRCClient): - def __init__(self, name, relay): + def __init__(self, net, num): self.connected = False self.channels = [] - self.net = "".join([x for x in name if not x in digits]) - if self.net == "": - error("Network with all numbers: %s" % name) + self.net = net + self.num = num self.buffer = "" - self.name = name - alias = relay["alias"] - + self.name = net + str(num) + alias = main.network[self.net].aliases[num] + relay = main.network[self.net].relays[num] self.nickname = alias["nick"] self.realname = alias["realname"] self.username = alias["nick"]+"/"+relay["net"] - self.password = relay["password"] + self.password = main.config["Relay"]["Password"] self.userinfo = None self.fingerReply = None self.versionName = None @@ -445,26 +447,30 @@ class IRCBot(IRCClient): self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): - def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): - if not name == None: - self.name = name - self.net = "".join([x for x in self.name if not x in digits]) + def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): + if net == None: + self.num = num + self.name = "Relay to %i" % num + self.relay = True else: - self.name = "Relay to "+relay["net"]+relay["id"] + self.name = net + str(num) + self.num = num + self.net = net + self.relay = False self.client = None self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"] self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"] self.factor = main.config["Tweaks"]["Delays"]["Factor"] self.jitter = main.config["Tweaks"]["Delays"]["Jitter"] - self.relay, self.relayCommands, self.user, self.stage2 = relay, relayCommands, user, stage2 + self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 def buildProtocol(self, addr): - if self.relay == None: - entry = IRCBot(self.name, self.relay) - main.IRCPool[self.name] = entry + if self.net == None: + entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) else: - entry = IRCRelay(self.relay, self.relayCommands, self.user, self.stage2) + entry = IRCBot(self.net, self.num) + main.IRCPool[self.name] = entry self.client = entry return entry diff --git a/core/relay.py b/core/relay.py index 4067ee0..ef20155 100644 --- a/core/relay.py +++ b/core/relay.py @@ -108,7 +108,7 @@ class Relay(Protocol): return def connectionMade(self): - log("Connection from %s:%s" % (self.addr.host, self.addr.port)) + log("Relay connection from %s:%s" % (self.addr.host, self.addr.port)) #self.send("Greetings.") def connectionLost(self, reason): diff --git a/modules/network.py b/modules/network.py index 5a89126..b6f0cfc 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,14 +1,16 @@ +from twisted.internet.ssl import DefaultOpenSSLContextFactory import json + import modules.alias as alias from twisted.internet import reactor from core.bot import IRCBot, IRCBotFactory -from twisted.internet.ssl import DefaultOpenSSLContextFactory import main from utils.logging.log import * +from utils.getrelay import getRelay class Network: - def __init__(self, name, host, port, security, auth): - self.name = name + def __init__(self, net, host, port, security, auth): + self.net = net self.host = host self.port = port self.security = security @@ -18,40 +20,39 @@ class Network: self.relays = {} self.aliases = {} - def add_relay(self, host, port, user, password): - self.last += 1 - self.relays[self.last] = { - "host": host, - "port": port, - "user": user, - "password": password, + def add_relay(self, num=None): + if not num: + self.last += 1 + num = self.last + self.relays[num] = { "enabled": False, - "net": self.name, - "id": self.last + "net": self.net, + "id": num } - self.aliases[self.last] = alias.generate_alias() - return self.last, self.aliases[self.last]["nick"] + self.aliases[num] = alias.generate_alias() + return num, self.aliases[num]["nick"] def delete_relay(self, id): del self.relays[id] del self.aliases[id] - def start_bot(self, relay): + def start_bot(self, num): # a single name is given to relays in the backend # e.g. freenode1 for the first relay on freenode network - name = self.name + relay keyFN = main.certPath+main.config["Key"] certFN = main.certPath+main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - bot = IRCBotFactory(name) - rct = reactor.connectSSL(k, port, bot, contextFactory) - + bot = IRCBotFactory(self.net, num) + #host, port = self.relays[num]["host"], self.relays[num]["port"] + host, port = getRelay(num) + rct = reactor.connectSSL(host, port, bot, contextFactory) + name = self.net + str(num) main.ReactorPool[name] = rct main.FactoryPool[name] = bot - log("Started bot on relay %s on %s", (relay, self.host)) + log("Started bot on relay %s on %s" % (num, self.host)) def start_bots(self): - for relay in self.relays: - if relay["enabled"]: - start_bot(relay) + for num in self.relays.keys(): + if self.relays[num]["enabled"]: + self.start_bot(num) diff --git a/modules/provision.py b/modules/provision.py index fb3ea9c..d2aad53 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,36 +1,35 @@ import main from core.bot import deliverRelayCommands from utils.logging.log import * -from core.helper import startBot -def provisionUserData(relay, alias, nick, altnick, ident, realname, password): +def provisionUserData(num, nick, altnick, ident, realname, unused): # last field is password, which we don't want to inherit here, but still want to use * expansion, so this is a bit of a hack commands = {} commands["controlpanel"] = [] - commands["controlpanel"].append("AddUser %s %s" % (alias, password)) - commands["controlpanel"].append("Set Nick %s %s" % (alias, nick)) - commands["controlpanel"].append("Set Altnick %s %s" % (alias, altnick)) - commands["controlpanel"].append("Set Ident %s %s" % (alias, ident)) - commands["controlpanel"].append("Set RealName %s %s" % (alias, realname)) - deliverRelayCommands(relay, commands) + commands["controlpanel"].append("AddUser %s %s" % (nick, main.config["Relay"]["Password"])) + commands["controlpanel"].append("Set Nick %s %s" % (nick, nick)) + commands["controlpanel"].append("Set Altnick %s %s" % (nick, altnick)) + commands["controlpanel"].append("Set Ident %s %s" % (nick, ident)) + commands["controlpanel"].append("Set RealName %s %s" % (nick, realname)) + deliverRelayCommands(num, commands) return -def provisionNetworkData(relay, alias, network, host, port, security, auth, password): +def provisionNetworkData(num, nick, network, host, port, security, auth, password): commands = {} stage2commands = {} stage3commands = {} commands["controlpanel"] = [] - commands["controlpanel"].append("AddNetwork %s %s" % (alias, network)) + commands["controlpanel"].append("AddNetwork %s %s" % (nick, network)) if security == "ssl": - commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (alias, network)) # Don't judge me - commands["controlpanel"].append("AddServer %s %s %s +%s" % (alias, network, host, port)) + commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (nick, network)) # Don't judge me + commands["controlpanel"].append("AddServer %s %s %s +%s" % (nick, network, host, port)) elif security == "plain": - commands["controlpanel"].append("AddServer %s %s %s %s" % (alias, network, host, port)) + commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port)) if auth == "sasl": stage2commands["status"] = [] stage2commands["sasl"] = [] stage2commands["status"].append("LoadMod sasl") stage2commands["sasl"].append("Mechanism plain") - stage2commands["sasl"].append("Set %s %s" % (alias, password)) + stage2commands["sasl"].append("Set %s %s" % (nick, password)) elif auth == "ns": stage2commands["status"] = [] stage2commands["nickserv"] = [] @@ -43,60 +42,23 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass stage2commands["status"] = [] stage2commands["status"].append("LoadMod disconkick") stage2commands["status"].append("LoadMod chansaver") - deliverRelayCommands(relay, commands, - stage2=[[alias+"/"+network, stage2commands], - [alias+"/"+network, stage3commands]]) + deliverRelayCommands(num, commands, + stage2=[[nick+"/"+network, stage2commands], + [nick+"/"+network, stage3commands]]) return -def provisionRelayForAlias(relay, alias): - #if "users" in main.relay[relay].keys(): - # if alias in main.relay[relay]["users"]: - # return "PROVISIONED" - #else: - # main.relay[relay]["users"] = [] - #main.relay[relay]["users"].append(alias) - provisionUserData(relay, alias, main.alias[alias]["nick"], - main.alias[alias]["altnick"], - main.alias[alias]["ident"], - main.alias[alias]["realname"], - main.relay[relay]["password"]) - #main.saveConf("relay") - return True +def provisionRelayForNetwork(num, alias, network): + provisionNetworkData(num, alias, network, + main.network[network].host, + main.network[network].port, + main.network[network].security, + main.network[network].auth, + main.network[network].aliases[num]["password"]) + return -def provisionRelayForNetwork(relay, alias, network): - #if set(["users", "networks"]).issubset(main.relay[relay].keys()): - # if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: - # return "PROVISIONED" - #else: - # main.relay[relay]["networks"] = [] - #main.relay[relay]["networks"].append(network) - provisionNetworkData(relay, alias, network, - main.network[network]["host"], - main.network[network]["port"], - main.network[network]["security"], - main.network[network]["auth"], - main.alias[alias]["password"]) - #main.saveConf("relay") - storedNetwork = False - num = 1 - while not storedNetwork: - i = str(num) - if num == 1000: - error("Iteration limit exceeded while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network)) - return False - - if network+i in main.pool.keys(): - if main.pool[network+i]["alias"] == alias and main.pool[network+i]["relay"] == relay: - return "DUPLICATE" - num += 1 - else: - main.pool[network+i] = {"relay": relay, - "alias": alias, - "network": network, - "enabled": main.config["ConnectOnCreate"]} - main.saveConf("pool") - if main.config["ConnectOnCreate"]: - startBot(network+i) - - storedNetwork = True - return network+i +def provisionRelay(num, network): + aliasObj = main.network[network].aliases[num] + alias = aliasObj["nick"] + provisionUserData(num, *aliasObj.values()) + provisionRelayForNetwork(num, alias, network) + return alias diff --git a/threshold b/threshold index 29a19c4..c5a3cc3 100755 --- a/threshold +++ b/threshold @@ -29,14 +29,14 @@ if __name__ == "__main__": else: reactor.listenTCP(main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"]) log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) - if main.config["Relay"]["Enabled"]: + if main.config["RelayAPI"]["Enabled"]: relay = RelayFactory() - if main.config["Relay"]["UseSSL"] == True: - reactor.listenSSL(main.config["Relay"]["Port"], relay, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["Relay"]["Address"]) - log("Threshold relay running with SSL on %s:%s" % (main.config["Relay"]["Address"], main.config["Relay"]["Port"])) + if main.config["RelayAPI"]["UseSSL"] == True: + reactor.listenSSL(main.config["RelayAPI"]["Port"], relay, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["RelayAPI"]["Address"]) + log("Threshold relay running with SSL on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) else: - reactor.listenTCP(main.config["Relay"]["Port"], relay, interface=main.config["Relay"]["Address"]) - log("Threshold relay running on %s:%s" % (main.config["Relay"]["Address"], main.config["Relay"]["Port"])) + reactor.listenTCP(main.config["RelayAPI"]["Port"], relay, interface=main.config["RelayAPI"]["Address"]) + log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() diff --git a/utils/getrelay.py b/utils/getrelay.py new file mode 100644 index 0000000..043ce44 --- /dev/null +++ b/utils/getrelay.py @@ -0,0 +1,12 @@ +import main + +def getRelay(num): + host = main.config["Relay"]["Host"].replace("x", str(num)) + port = int(str(main.config["Relay"]["Port"]).replace("x", str(num))) + user = main.config["Relay"]["User"] + password = main.config["Relay"]["Password"] + try: + port = int(port) + except ValueError: + return False + return [host, port] diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 360671d..031f2aa 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,6 +1,7 @@ from os import listdir from utils.logging.debug import debug +from utils.logging.log import * import commands from main import CommandMap From 006f8db6f66dee28212ed59c1641cebf41d1db62 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 25 Aug 2019 23:12:51 +0100 Subject: [PATCH 140/394] Update the class name in the single command loader and import the debug function which it uses --- utils/loaders/single_loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index 3d891e2..dc28252 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -2,6 +2,7 @@ from os import listdir from importlib import reload from sys import modules +from utils.logging.debug import debug from utils.logging.log import * import commands @@ -9,7 +10,7 @@ from main import CommandMap def loadSingle(commandName): if commandName+".py" in listdir("commands"): - className = commandName.capitalize() + className = commandName.capitalize()+"Command" try: if commandName in CommandMap.keys(): reload(modules["commands."+commandName]) From 15ca45e5dfdcdfc2235c55b015166c2d11caade2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Sep 2019 19:46:10 +0100 Subject: [PATCH 141/394] Implement Ctrl-C handling and fix a large number of small bugs --- commands/auto.py | 6 +++--- commands/del.py | 20 ++++++++++++-------- commands/disable.py | 32 +++++++++++++++++++------------- commands/stats.py | 2 +- commands/swho.py | 9 ++++++--- conf/help.json | 5 +++-- core/bot.py | 29 ++++++++++++++++------------- main.py | 8 -------- modules/userinfo.py | 8 ++++---- threshold | 9 ++++++--- utils/cleanup.py | 15 +++++++++++++++ utils/loaders/single_loader.py | 6 +++--- 12 files changed, 88 insertions(+), 61 deletions(-) create mode 100644 utils/cleanup.py diff --git a/commands/auto.py b/commands/auto.py index 3d29a85..05e8d6c 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -14,11 +14,11 @@ class AutoCommand: if not spl[2].isdigit(): failure("Must be integer, not %s" % spl[2]) return - - id, alias = main.network[spl[1]].add_relay(int(spl[2])) + relayNum = int(spl[2]) + id, alias = main.network[spl[1]].add_relay(relayNum) success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[1], alias)) main.saveConf("network") - rtrn = provision.provisionRelay(int(spl[2]), spl[1]) + rtrn = provision.provisionRelay(relayNum, spl[1]) success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], rtrn)) return diff --git a/commands/del.py b/commands/del.py index eb59eaa..e1108ad 100644 --- a/commands/del.py +++ b/commands/del.py @@ -10,19 +10,23 @@ class DelCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return + if not spl[2].isdigit(): + failure("Must be integer, not %s" % spl[2]) + return if not int(spl[2]) in main.network[spl[1]].relays.keys(): failure("No such relay: %s in network %s" % (spl[2], spl[1])) return main.network[spl[1]].delete_relay(int(spl[2])) - if spl[1]+spl[2] in main.ReactorPool.keys(): - if spl[1]+spl[2] in main.FactoryPool.keys(): - main.FactoryPool[spl[1]+spl[2]].stopTrying() - main.ReactorPool[spl[1]+spl[2]].disconnect() - if spl[1]+spl[2] in main.IRCPool.keys(): - del main.IRCPool[spl[1]+spl[2]] - del main.ReactorPool[spl[1]+spl[2]] - del main.FactoryPool[spl[1]+spl[2]] + name = spl[1]+spl[2] + if name in main.ReactorPool.keys(): + if name in main.FactoryPool.keys(): + main.FactoryPool[name].stopTrying() + main.ReactorPool[name].disconnect() + if name in main.IRCPool.keys(): + del main.IRCPool[name] + del main.ReactorPool[name] + del main.FactoryPool[name] success("Successfully removed bot: %s" % spl[1]) main.saveConf("network") return diff --git a/commands/disable.py b/commands/disable.py index 20ace48..7abfec0 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -11,25 +11,31 @@ class DisableCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - if not int(spl[2]) in main.network[spl[1]].relays.keys(): + if not spl[2].isdigit(): + failure("Must be integer, not %s" % spl[2]) + return + relayNum = int(spl[2]) + name = spl[1]+spl[2] + if not spl[1] in main.IRCPool.keys(): + info("Note - instance not running, proceeding anyway") + if not relayNum in main.network[spl[1]].relays.keys(): failure("No such relay: %s in network %s" % (spl[2], spl[1])) return - - main.network[spl[1]].relays[int(spl[2])]["enabled"] = False - user = main.network[spl[1]].aliases[int(spl[2])] + main.network[spl[1]].relays[relayNum]["enabled"] = False + user = main.network[spl[1]].aliases[relayNum]["nick"] network = spl[1] - relay = main.network[spl[1]].relays[int(spl[2])] + relay = main.network[spl[1]].relays[relayNum] commands = {"status": ["Disconnect"]} - deliverRelayCommands(relay, commands, user=user+"/"+network) + deliverRelayCommands(relayNum, commands, user=user+"/"+network) main.saveConf("network") - if spl[1]+spl[2] in main.ReactorPool.keys(): - if spl[1]+spl[2] in main.FactoryPool.keys(): - main.FactoryPool[spl[1]+spl[2]].stopTrying() - main.ReactorPool[spl[1]+spl[2]].disconnect() + if name in main.ReactorPool.keys(): + if name in main.FactoryPool.keys(): + main.FactoryPool[name].stopTrying() + main.ReactorPool[name].disconnect() if spl[1] in main.IRCPool.keys(): - del main.IRCPool[spl[1]+spl[2]] - del main.ReactorPool[spl[1]+spl[2]] - del main.FactoryPool[spl[1]+spl[2]] + del main.IRCPool[name] + del main.ReactorPool[name] + del main.FactoryPool[name] success("Successfully disabled bot %s on network %s" % (spl[2], spl[1])) return else: diff --git a/commands/stats.py b/commands/stats.py index e205d02..0db414a 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -17,7 +17,7 @@ class StatsCommand: numChannels += len(main.IRCPool[i].channels) numWhoEntries += userinfo.getNumTotalWhoEntries() stats.append("Registered servers:") - stats.append(" Total: %s" % len(main.network.keys())) + stats.append(" Unique: %s" % len(main.network.keys())) stats.append("Online servers:") stats.append(" Total: %s" % len(main.IRCPool.keys())) stats.append(" Unique: %s" % len(main.liveNets())) diff --git a/commands/swho.py b/commands/swho.py index b03ca61..49c71d7 100644 --- a/commands/swho.py +++ b/commands/swho.py @@ -11,18 +11,21 @@ class SwhoCommand: failure("Network does not exist: %s" % spl[1]) return for i in main.IRCPool.keys(): - if spl[1] in i: + if spl[1] == main.IRCPool[i].net: for x in main.IRCPool[i].channels: main.IRCPool[i].who(x) - success("Sent WHO to all channels on all networks on %s" % spl[1]) + success("Sent WHO to all channels on all networks for %s" % spl[1]) return elif length == 3: if not spl[1] in main.network.keys(): failure("Network does not exist: %s" % spl[1]) return matches = [] + + # This loop gets all networks where the core network matches spl[1] + # where there is also a currently joined channel matching spl[2] for i in main.IRCPool.keys(): - if spl[1] in i: + if spl[1] == main.IRCPool[i].net: for x in main.IRCPool[i].channels: if x == spl[2]: main.IRCPool[i].who(x) diff --git a/conf/help.json b/conf/help.json index e58bfe1..34eca54 100644 --- a/conf/help.json +++ b/conf/help.json @@ -1,7 +1,7 @@ { "pass": "pass ", "logout": "logout", - "del": "del ", + "del": "del ", "mod": "mod [] []", "who": "who ", "join": "join []", @@ -24,5 +24,6 @@ "cmd": "cmd ", "token": "token [] []", "all": "all ", - "allc": "allc <(network)|(alias)> " + "allc": "allc <(network)|(alias)> ", + "swho": "swho []" } diff --git a/core/bot.py b/core/bot.py index 52c8044..466a2a4 100644 --- a/core/bot.py +++ b/core/bot.py @@ -72,7 +72,6 @@ class IRCRelay(IRCClient): sendAll("[%s] %s -> %s" % (self.num, nick, msg)) def irc_ERR_PASSWDMISMATCH(self, prefix, params): - print(', '.join("%s: %s" % item for item in vars(self).items())) log("%s: relay password mismatch" % self.num) sendAll("%s: relay password mismatch" % self.num) @@ -113,7 +112,7 @@ class IRCBot(IRCClient): self.versionEnv = None self.sourceURL = None - self._who = {} + self._tempWho = {} self._getWho = {} self._names = {} @@ -227,14 +226,17 @@ class IRCBot(IRCClient): log("%s: password mismatch" % self.name) sendAll("%s: password mismatch" % self.name) - def who(self, channel): + def _who(self, channel): d = Deferred() - if channel not in self._who: - self._who[channel] = ([], []) - self._who[channel][0].append(d) + if channel not in self._tempWho: + self._tempWho[channel] = ([], []) + self._tempWho[channel][0].append(d) self.sendLine("WHO %s" % channel) return d + def who(self, channel): + self._who(channel).addCallback(self.got_who) + def irc_RPL_WHOREPLY(self, prefix, params): channel = params[1] ident = params[2] @@ -243,20 +245,20 @@ class IRCBot(IRCClient): nick = params[5] status = params[6] realname = params[7] - if channel not in self._who: + if channel not in self._tempWho: return - n = self._who[channel][1] + n = self._tempWho[channel][1] n.append([nick, nick, host, server, status, realname]) self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, target=channel, server=server, status=status) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] - if channel not in self._who: + if channel not in self._tempWho: return - callbacks, info = self._who[channel] + callbacks, info = self._tempWho[channel] for cb in callbacks: cb.callback((channel, info)) - del self._who[channel] + del self._tempWho[channel] def got_who(self, whoinfo): userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1]) @@ -382,7 +384,8 @@ class IRCBot(IRCClient): self.channels.append(channel) self.names(channel).addCallback(self.got_names) if main.config["Toggles"]["Who"]: - self.who(channel).addCallback(self.got_who) + #self.who(channel).addCallback(self.got_who) + #self.who(channel) lc = LoopingCall(self.who, channel) self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] @@ -466,7 +469,7 @@ class IRCBotFactory(ReconnectingClientFactory): self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 def buildProtocol(self, addr): - if self.net == None: + if self.relay == None: entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) else: entry = IRCBot(self.net, self.num) diff --git a/main.py b/main.py index 28633ea..c9e45e3 100644 --- a/main.py +++ b/main.py @@ -39,14 +39,6 @@ lastMinuteSample = 0 hashKey = urandom(16) lastEvents = {} -def nets(): - if not "pool" in globals(): - return - networks = set() - for i in pool.keys(): - networks.add("".join([x for x in i if not x in digits])) - return networks - def liveNets(): networks = set() for i in IRCPool.keys(): diff --git a/modules/userinfo.py b/modules/userinfo.py index 29da9b0..e4035e8 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -13,7 +13,7 @@ def getWhoSingle(name, query): def getWho(query): result = {} - for i in main.nets(): + for i in main.network.keys(): f = getWhoSingle(i, query) if f: result[i] = f @@ -35,7 +35,7 @@ def getChanList(name, nick): def getChans(nick): result = {} - for i in main.nets(): + for i in main.network.keys(): f = getChansSingle(i, nick) if f: result[i] = f @@ -50,7 +50,7 @@ def getUsersSingle(name, nick): def getUsers(nick): result = {} - for i in main.nets(): + for i in main.network.keys(): f = getUsersSingle(i, nick) if f: result[i] = f @@ -61,7 +61,7 @@ def getNumWhoEntries(name): def getNumTotalWhoEntries(): total = 0 - for i in main.nets(): + for i in main.network.keys(): total += getNumWhoEntries(i) return total diff --git a/threshold b/threshold index c5a3cc3..9425e60 100755 --- a/threshold +++ b/threshold @@ -1,18 +1,21 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory -from sys import argv, stdout, stderr +import sys +from signal import signal, SIGINT #from twisted.python import log #from sys import stdout #log.startLogging(stdout) +from sys import stdout, stderr # Import again because we want to override from codecs import getwriter # fix printing odd shit to the terminal stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using # unicode quit messages for no reason import main - main.initMain() -if "--debug" in argv: # yes really +from utils.cleanup import handler +signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine +if "--debug" in sys.argv: # yes really main.config["Debug"] = True from utils.logging.log import * from utils.loaders.command_loader import loadCommands diff --git a/utils/cleanup.py b/utils/cleanup.py new file mode 100644 index 0000000..3b818f3 --- /dev/null +++ b/utils/cleanup.py @@ -0,0 +1,15 @@ +import main +from twisted.internet import reactor +from utils.logging.debug import debug +from utils.logging.log import * +import sys + +def handler(sig, frame): + log("Received SIGINT, cleaning up") + cleanup() + +def cleanup(): + debug("Flushing Redis database") + main.r.flushdb() + reactor.stop() + #sys.exit(1) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index dc28252..81e5c76 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -1,6 +1,6 @@ from os import listdir from importlib import reload -from sys import modules +import sys from utils.logging.debug import debug from utils.logging.log import * @@ -13,8 +13,8 @@ def loadSingle(commandName): className = commandName.capitalize()+"Command" try: if commandName in CommandMap.keys(): - reload(modules["commands."+commandName]) - CommandMap[commandName] = getattr(modules["commands."+commandName], className) + reload(sys.modules["commands."+commandName]) + CommandMap[commandName] = getattr(sys.modules["commands."+commandName], className) debug("Reloaded command: %s" % commandName) return "RELOAD" module = __import__('commands.%s' % commandName) From 355a80b19b100dcc67c20ff67587a66831e8e829 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 29 Sep 2019 14:57:36 +0100 Subject: [PATCH 142/394] Fix the all and allc commands so they work with the new data format --- commands/all.py | 17 +++++++++-------- commands/allc.py | 33 +++++++++++++-------------------- commands/auto.py | 12 +++++++++++- commands/cmd.py | 2 +- commands/disable.py | 2 +- commands/network.py | 4 ++++ core/bot.py | 6 +++--- modules/network.py | 4 +++- 8 files changed, 45 insertions(+), 35 deletions(-) diff --git a/commands/all.py b/commands/all.py index 10ec381..a6c377f 100644 --- a/commands/all.py +++ b/commands/all.py @@ -1,20 +1,21 @@ import main from core.bot import deliverRelayCommands -class All: +class AllCommand: def __init__(self, *args): self.all(*args) def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 2: - for i in main.pool.keys(): - relay = main.pool[i]["relay"] - network = main.pool[i]["network"] - alias = main.pool[i]["alias"] - commands = {spl[1]: [" ".join(spl[2:])]} - success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) - deliverRelayCommands(relay, commands, user=alias+"/"+network) + for i in main.network.keys(): + for x in main.network[i].relays.keys(): + num = main.network[i].relays[x]["id"] + net = main.network[i].relays[x]["net"] + alias = main.network[i].aliases[x]["nick"] + commands = {spl[1]: [" ".join(spl[2:])]} + success("Sending commands to relay %s as user %s" % (num, alias+"/"+net)) + deliverRelayCommands(num, commands, user=alias+"/"+net) return else: incUsage("all") diff --git a/commands/allc.py b/commands/allc.py index 9e662fe..41b1a48 100644 --- a/commands/allc.py +++ b/commands/allc.py @@ -1,7 +1,7 @@ import main from core.bot import deliverRelayCommands -class Allc: +class AllcCommand: def __init__(self, *args): self.allc(*args) @@ -10,21 +10,14 @@ class Allc: if length > 4: targets = [] if spl[1] == "network": - if spl[2] in main.network.keys(): - for i in main.pool.keys(): - if main.pool[i]["network"] == spl[2]: - targets.append(i) - else: - failure("No such network: %s" % spl[2]) - return + for i in main.network.keys(): + for x in main.network[i].relays.keys(): + if main.network[i].relays[x]["net"] == spl[2]: + targets.append((i, x)) elif spl[1] == "alias": - if spl[2] in main.alias.keys(): - for i in main.pool.keys(): - if main.pool[i]["alias"] == spl[2]: - targets.append(i) - else: - failure("No such alias: %s" % spl[2]) - return + for i in main.network.keys(): + [targets.append((i, x)) for x in main.network[i].aliases.keys() if + main.network[i].aliases[x]["nick"] == spl[2]] else: incUsage("allc") return @@ -32,12 +25,12 @@ class Allc: failure("No matches found: %s" % spl[2]) return for i in targets: - relay = main.pool[i]["relay"] - network = main.pool[i]["network"] - alias = main.pool[i]["alias"] + net = i[0] + num = i[1] + alias = main.network[net].aliases[num]["nick"] commands = {spl[3]: [" ".join(spl[4:])]} - success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) - deliverRelayCommands(relay, commands, user=alias+"/"+network) + success("Sending commands to relay %i as user %s" % (num, alias+"/"+net)) + deliverRelayCommands(num, commands, user=alias+"/"+net) return else: incUsage("allc") diff --git a/commands/auto.py b/commands/auto.py index 05e8d6c..90a2839 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -12,7 +12,7 @@ class AutoCommand: failure("No such network: %s" % spl[1]) return if not spl[2].isdigit(): - failure("Must be integer, not %s" % spl[2]) + failure("Must be a number, not %s" % spl[2]) return relayNum = int(spl[2]) id, alias = main.network[spl[1]].add_relay(relayNum) @@ -21,6 +21,16 @@ class AutoCommand: rtrn = provision.provisionRelay(relayNum, spl[1]) success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], rtrn)) return + elif length == 2: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + num, alias = main.network[spl[1]].add_relay() + success("Successfully created relay %s on network %s with alias %s" % (str(num), spl[1], alias)) + main.saveConf("network") + rtrn = provision.provisionRelay(num, spl[1]) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, rtrn)) + return else: incUsage("auto") diff --git a/commands/cmd.py b/commands/cmd.py index 858dc15..466b197 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -9,7 +9,7 @@ class CmdCommand: if authed: if length > 4: if not spl[1].isdigit(): - failure("Must be integer, not %s" % spl[1]) + failure("Must be a number, not %s" % spl[1]) return commands = {spl[3]: [" ".join(spl[4:])]} success("Sending commands to relay %s as user %s" % (spl[2], spl[3])) diff --git a/commands/disable.py b/commands/disable.py index 7abfec0..2da50f3 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -12,7 +12,7 @@ class DisableCommand: failure("No such network: %s" % spl[1]) return if not spl[2].isdigit(): - failure("Must be integer, not %s" % spl[2]) + failure("Must be a number, not %s" % spl[2]) return relayNum = int(spl[2]) name = spl[1]+spl[2] diff --git a/commands/network.py b/commands/network.py index b9cfbfd..33d404c 100644 --- a/commands/network.py +++ b/commands/network.py @@ -1,6 +1,7 @@ import main from yaml import dump from modules.network import Network +from string import digits class NetworkCommand: def __init__(self, *args): @@ -19,6 +20,9 @@ class NetworkCommand: if not spl[5].lower() in ["ssl", "plain"]: failure("Security must be ssl or plain, not %s" % spl[5]) return + if set(spl[2]).intersection(set(digits)): + failure("Network name cannot contain numbers") + return if not spl[6].lower() in ["sasl", "ns", "none"]: failure("Auth must be sasl, ns or none, not %s" % spl[5]) return diff --git a/core/bot.py b/core/bot.py index 466a2a4..952293e 100644 --- a/core/bot.py +++ b/core/bot.py @@ -469,11 +469,11 @@ class IRCBotFactory(ReconnectingClientFactory): self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 def buildProtocol(self, addr): - if self.relay == None: - entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) - else: + if self.relay == False: entry = IRCBot(self.net, self.num) main.IRCPool[self.name] = entry + else: + entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) self.client = entry return entry diff --git a/modules/network.py b/modules/network.py index b6f0cfc..094a0a5 100644 --- a/modules/network.py +++ b/modules/network.py @@ -25,11 +25,13 @@ class Network: self.last += 1 num = self.last self.relays[num] = { - "enabled": False, + "enabled": main.config["ConnectOnCreate"], "net": self.net, "id": num } self.aliases[num] = alias.generate_alias() + if main.config["ConnectOnCreate"]: + self.start_bot(num) return num, self.aliases[num]["nick"] def delete_relay(self, id): From 32309ecec26eb4a53fa1bec1704e322bfc2a85fe Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 29 Sep 2019 22:45:16 +0100 Subject: [PATCH 143/394] Change alias definitions to be global, so aliases can be reused across different networks --- commands/alias.py | 12 ++++++++++++ commands/all.py | 2 +- commands/allc.py | 7 ++++--- commands/auto.py | 6 +++--- commands/disable.py | 2 +- commands/enable.py | 2 +- commands/mod.py | 31 +++++++++++++++++-------------- conf/example/alias.json | 1 + conf/help.json | 1 + core/bot.py | 4 ++-- main.py | 7 +++++++ modules/alias.py | 5 ++++- modules/network.py | 15 ++++++++++----- modules/provision.py | 4 +++- utils/{getrelay.py => get.py} | 2 +- 15 files changed, 68 insertions(+), 33 deletions(-) create mode 100644 commands/alias.py create mode 100644 conf/example/alias.json rename utils/{getrelay.py => get.py} (93%) diff --git a/commands/alias.py b/commands/alias.py new file mode 100644 index 0000000..1c01f0d --- /dev/null +++ b/commands/alias.py @@ -0,0 +1,12 @@ +import main +from yaml import dump + +class AliasCommand: + def __init__(self, *args): + self.alias(*args) + + def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + info(dump(main.alias)) + else: + incUsage(None) diff --git a/commands/all.py b/commands/all.py index a6c377f..e59a9fd 100644 --- a/commands/all.py +++ b/commands/all.py @@ -12,7 +12,7 @@ class AllCommand: for x in main.network[i].relays.keys(): num = main.network[i].relays[x]["id"] net = main.network[i].relays[x]["net"] - alias = main.network[i].aliases[x]["nick"] + alias = main.alias[x]["nick"] commands = {spl[1]: [" ".join(spl[2:])]} success("Sending commands to relay %s as user %s" % (num, alias+"/"+net)) deliverRelayCommands(num, commands, user=alias+"/"+net) diff --git a/commands/allc.py b/commands/allc.py index 41b1a48..5d1d7cb 100644 --- a/commands/allc.py +++ b/commands/allc.py @@ -16,8 +16,9 @@ class AllcCommand: targets.append((i, x)) elif spl[1] == "alias": for i in main.network.keys(): - [targets.append((i, x)) for x in main.network[i].aliases.keys() if - main.network[i].aliases[x]["nick"] == spl[2]] + [targets.append((i, x)) for x in main.alias.keys() if + main.alias[x]["nick"] == spl[2] and + x in main.network[i].aliases.keys()] else: incUsage("allc") return @@ -27,7 +28,7 @@ class AllcCommand: for i in targets: net = i[0] num = i[1] - alias = main.network[net].aliases[num]["nick"] + alias = main.alias[num]["nick"] commands = {spl[3]: [" ".join(spl[4:])]} success("Sending commands to relay %i as user %s" % (num, alias+"/"+net)) deliverRelayCommands(num, commands, user=alias+"/"+net) diff --git a/commands/auto.py b/commands/auto.py index 90a2839..7adf0cb 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -15,8 +15,8 @@ class AutoCommand: failure("Must be a number, not %s" % spl[2]) return relayNum = int(spl[2]) - id, alias = main.network[spl[1]].add_relay(relayNum) - success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[1], alias)) + num, alias = main.network[spl[1]].add_relay(relayNum) + success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) main.saveConf("network") rtrn = provision.provisionRelay(relayNum, spl[1]) success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], rtrn)) @@ -26,7 +26,7 @@ class AutoCommand: failure("No such network: %s" % spl[1]) return num, alias = main.network[spl[1]].add_relay() - success("Successfully created relay %s on network %s with alias %s" % (str(num), spl[1], alias)) + success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) main.saveConf("network") rtrn = provision.provisionRelay(num, spl[1]) success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, rtrn)) diff --git a/commands/disable.py b/commands/disable.py index 2da50f3..2e1caeb 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -22,7 +22,7 @@ class DisableCommand: failure("No such relay: %s in network %s" % (spl[2], spl[1])) return main.network[spl[1]].relays[relayNum]["enabled"] = False - user = main.network[spl[1]].aliases[relayNum]["nick"] + user = main.alias[relayNum]["nick"] network = spl[1] relay = main.network[spl[1]].relays[relayNum] commands = {"status": ["Disconnect"]} diff --git a/commands/enable.py b/commands/enable.py index 0e88dce..af645cd 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -19,7 +19,7 @@ class EnableCommand: return main.network[spl[1]].relays[int(spl[2])]["enabled"] = True - user = main.network[spl[1]].aliases[int(spl[2])]["nick"] + user = main.alias[int(spl[2])]["nick"] network = spl[1] commands = {"status": ["Connect"]} deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network) diff --git a/commands/mod.py b/commands/mod.py index 611ab9b..0df1ada 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -22,21 +22,24 @@ class ModCommand: main.saveConf("network") success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return + # Find a better way to do this + #elif length == 6: + # if not spl[1] in main.network.keys(): + # failure("Network does not exist: %s" % spl[1]) + # return + # if not spl[3].isdigit(): + # failure("Must be a number, not %s" % spl[3]) + # return + # if not int(spl[3]) in main.network[spl[1]].relays.keys(): + # failure("Relay/alias does not exist: %s" % spl[3]) + # return - elif length == 6: - if not spl[1] in main.network.keys(): - failure("Network does not exist: %s" % spl[1]) - return - if not int(spl[3]) in main.network[spl[1]].relays.keys(): - failure("Relay/alias does not exist: %s" % spl[3]) - return - - try: - x = getattr(main.network[spl[1]], spl[2]) - x[spl[3]] = spl[4] - except e: - failure("Something went wrong.") - return + # try: + # x = getattr(main.network[spl[1]], spl[2]) + # x[spl[3]] = spl[4] + # except Exception as err: + # failure("Error: %s" % err) + # return else: incUsage("mod") diff --git a/conf/example/alias.json b/conf/example/alias.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/alias.json @@ -0,0 +1 @@ +{} diff --git a/conf/help.json b/conf/help.json index 34eca54..0a7ebff 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,6 +20,7 @@ "users": "users [ ...]", "relay": "relay [] []", "network": "network [
]", + "alias": "alias", "auto": "auto ", "cmd": "cmd ", "token": "token [] []", diff --git a/core/bot.py b/core/bot.py index 952293e..d078ca1 100644 --- a/core/bot.py +++ b/core/bot.py @@ -14,7 +14,7 @@ from modules import monitor from core.relay import sendRelayNotification from utils.dedup import dedup -from utils.getrelay import getRelay +from utils.get import getRelay import main from utils.logging.log import * @@ -99,7 +99,7 @@ class IRCBot(IRCClient): self.num = num self.buffer = "" self.name = net + str(num) - alias = main.network[self.net].aliases[num] + alias = main.alias[num] relay = main.network[self.net].relays[num] self.nickname = alias["nick"] self.realname = alias["realname"] diff --git a/main.py b/main.py index c9e45e3..e7e3a6c 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ filemap = { "monitor": ["monitor.json", "monitoring database", "json"], "tokens": ["tokens.json", "authentication tokens", "json"], "aliasdata": ["aliasdata.json", "data for alias generation", "json"], + "alias": ["alias.json", "provisioned alias data", "json"], # Binary (pickle) configs "network": ["network.dat", "network list", "pickle"] @@ -59,6 +60,12 @@ def loadConf(var): if filemap[var][2] == "json": with open(configPath+filemap[var][0], "r") as f: globals()[var] = json.load(f) + if var == "alias": + # This is a hack to convert all the keys into integers since JSON + # turns them into strings... + # Dammit Jason! + global alias + alias = {int(x):y for x, y in alias.items()} elif filemap[var][2] == "pickle": try: with open(configPath+filemap[var][0], "rb") as f: diff --git a/modules/alias.py b/modules/alias.py index 0248088..e05950d 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -2,6 +2,9 @@ import main import random import re +def generate_password(): + return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) + def generate_alias(): nick = random.choice(main.aliasdata["stubs"]) rand = random.randint(1, 2) @@ -60,6 +63,6 @@ def generate_alias(): if rand == 3 or rand == 4: realname = realname.capitalize() - password = "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) + password = generate_password() return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password} diff --git a/modules/network.py b/modules/network.py index 094a0a5..066b41f 100644 --- a/modules/network.py +++ b/modules/network.py @@ -6,7 +6,7 @@ from twisted.internet import reactor from core.bot import IRCBot, IRCBotFactory import main from utils.logging.log import * -from utils.getrelay import getRelay +from utils.get import getRelay class Network: def __init__(self, net, host, port, security, auth): @@ -29,14 +29,19 @@ class Network: "net": self.net, "id": num } - self.aliases[num] = alias.generate_alias() - if main.config["ConnectOnCreate"]: - self.start_bot(num) - return num, self.aliases[num]["nick"] + password = alias.generate_password() + if not num in main.alias.keys(): + main.alias[num] = alias.generate_alias() + main.saveConf("alias") + self.aliases[num] = {"password": password} + #if main.config["ConnectOnCreate"]: -- Done in provision + # self.start_bot(num) + return num, main.alias[num]["nick"] def delete_relay(self, id): del self.relays[id] del self.aliases[id] + #del main.alias[id] - Aliases are global per num, so don't delete them! def start_bot(self, num): # a single name is given to relays in the backend diff --git a/modules/provision.py b/modules/provision.py index d2aad53..af623e1 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -57,8 +57,10 @@ def provisionRelayForNetwork(num, alias, network): return def provisionRelay(num, network): - aliasObj = main.network[network].aliases[num] + aliasObj = main.alias[num] alias = aliasObj["nick"] provisionUserData(num, *aliasObj.values()) provisionRelayForNetwork(num, alias, network) + if main.config["ConnectOnCreate"]: + main.network[network].start_bot(num) return alias diff --git a/utils/getrelay.py b/utils/get.py similarity index 93% rename from utils/getrelay.py rename to utils/get.py index 043ce44..46e9edf 100644 --- a/utils/getrelay.py +++ b/utils/get.py @@ -9,4 +9,4 @@ def getRelay(num): port = int(port) except ValueError: return False - return [host, port] + return (host, port) From b19dfcc113a0445177b903d25b43b28ef6284d78 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 29 Sep 2019 23:55:22 +0100 Subject: [PATCH 144/394] Add alias.json to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 60fe20c..4e98897 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ conf/counters.json conf/monitor.json conf/tokens.json conf/network.dat +conf/alias.json conf/dist.sh env/ From 0f31d7f5e21088a46bf1cf2f5f96ff40f157cc06 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 2 Oct 2019 13:46:02 +0100 Subject: [PATCH 145/394] Implement admall command to run commands as the administrative relay user --- commands/admall.py | 22 ++++++++++++++++++++++ conf/help.json | 1 + 2 files changed, 23 insertions(+) create mode 100644 commands/admall.py diff --git a/commands/admall.py b/commands/admall.py new file mode 100644 index 0000000..cbbf854 --- /dev/null +++ b/commands/admall.py @@ -0,0 +1,22 @@ +import main +from core.bot import deliverRelayCommands + +class AdmallCommand: + def __init__(self, *args): + self.admall(*args) + + def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 2: + for i in main.network.keys(): + for x in main.network[i].relays.keys(): + num = main.network[i].relays[x]["id"] + commands = {spl[1]: [" ".join(spl[2:])]} + success("Sending commands to relay %s" % (num)) + deliverRelayCommands(num, commands) + return + else: + incUsage("admall") + return + else: + incUsage(None) diff --git a/conf/help.json b/conf/help.json index 0a7ebff..37655a9 100644 --- a/conf/help.json +++ b/conf/help.json @@ -26,5 +26,6 @@ "token": "token [] []", "all": "all ", "allc": "allc <(network)|(alias)> ", + "admall": "admall ", "swho": "swho []" } From a3b81f8849e9e20816ddc69d47442e76485e26cf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 2 Oct 2019 20:26:05 +0100 Subject: [PATCH 146/394] Implement more automated provisioning of aliases and relays, and remove networks cleanly --- commands/alias.py | 46 ++++++++++++++++++++++++++++++++++++++++++++- commands/auto.py | 20 +++++++++++++++----- commands/disable.py | 2 +- commands/network.py | 1 + conf/help.json | 2 +- modules/network.py | 15 ++++++++++++++- 6 files changed, 77 insertions(+), 9 deletions(-) diff --git a/commands/alias.py b/commands/alias.py index 1c01f0d..b501c36 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -1,5 +1,6 @@ import main from yaml import dump +from modules import alias class AliasCommand: def __init__(self, *args): @@ -7,6 +8,49 @@ class AliasCommand: def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - info(dump(main.alias)) + if length == 1: + info(dump(main.alias)) + return + elif length == 2: + if spl[1] == "add": + nextNum = max(main.alias.keys())+1 + main.alias[nextNum] = alias.generate_alias() + success("Generated new alias: %i" % nextNum) + main.saveConf("alias") + return + else: + incUsage("alias") + return + elif length == 3: + if spl[1] == "add": + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return + num = int(spl[2]) + for i in range(num): + nextNum = max(main.alias.keys())+1 + main.alias[nextNum] = alias.generate_alias() + success("Generated new alias: %i" % nextNum) + main.saveConf("alias") + return + elif spl[1] == "del": + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return + num = int(spl[2]) + failed = False + for i in main.network.keys(): + if num in main.network[i].aliases.keys(): + failure("Alias in use by %s" % i) + failed = True + if failed: + return + del main.alias[num] + success("Removed alias: %i" % num) + main.saveConf("alias") + return + else: + incUsage("alias") + return else: incUsage(None) diff --git a/commands/auto.py b/commands/auto.py index 7adf0cb..8832bb7 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -25,13 +25,23 @@ class AutoCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - num, alias = main.network[spl[1]].add_relay() - success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) + for i in main.alias.keys(): + print("num", i) + num, alias = main.network[spl[1]].add_relay(i) + success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) + rtrn = provision.provisionRelay(num, spl[1]) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, rtrn)) + main.saveConf("network") + return + elif length == 1: + for i in main.network.keys(): + for x in main.alias.keys(): + num, alias = main.network[i].add_relay(x) + success("Successfully created relay %i on network %s with alias %s" % (num, i, alias)) + rtrn = provision.provisionRelay(num, i) + success("Started provisioning network %s on relay %s for alias %s" % (i, num, rtrn)) main.saveConf("network") - rtrn = provision.provisionRelay(num, spl[1]) - success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, rtrn)) return - else: incUsage("auto") return diff --git a/commands/disable.py b/commands/disable.py index 2e1caeb..62b42df 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -32,7 +32,7 @@ class DisableCommand: if name in main.FactoryPool.keys(): main.FactoryPool[name].stopTrying() main.ReactorPool[name].disconnect() - if spl[1] in main.IRCPool.keys(): + if name in main.IRCPool.keys(): del main.IRCPool[name] del main.ReactorPool[name] del main.FactoryPool[name] diff --git a/commands/network.py b/commands/network.py index 33d404c..e88a905 100644 --- a/commands/network.py +++ b/commands/network.py @@ -38,6 +38,7 @@ class NetworkCommand: elif length == 3: if spl[1] == "del": if spl[2] in main.network.keys(): + main.network[spl[2]].seppuku() # ;( del main.network[spl[2]] success("Successfully removed network: %s" % spl[2]) main.saveConf("network") diff --git a/conf/help.json b/conf/help.json index 37655a9..1e00114 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,7 +20,7 @@ "users": "users [ ...]", "relay": "relay [] []", "network": "network [
]", - "alias": "alias", + "alias": "alias []", "auto": "auto ", "cmd": "cmd ", "token": "token [] []", diff --git a/modules/network.py b/modules/network.py index 066b41f..e4eefb3 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,7 +1,7 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory import json -import modules.alias as alias +from modules import alias from twisted.internet import reactor from core.bot import IRCBot, IRCBotFactory import main @@ -43,6 +43,19 @@ class Network: del self.aliases[id] #del main.alias[id] - Aliases are global per num, so don't delete them! + def seppuku(self): + # Removes all bots in preperation for deletion + for i in self.relays.keys(): + name = self.net+str(i) + if name in main.ReactorPool.keys(): + if name in main.FactoryPool.keys(): + main.FactoryPool[name].stopTrying() + main.ReactorPool[name].disconnect() + if name in main.IRCPool.keys(): + del main.IRCPool[name] + del main.ReactorPool[name] + del main.FactoryPool[name] + def start_bot(self, num): # a single name is given to relays in the backend # e.g. freenode1 for the first relay on freenode network From d35f96de87da9ec82e66f836f04597def58a5907 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 2 Oct 2019 20:45:28 +0100 Subject: [PATCH 147/394] Error checking on alias removal and clean up when removing relays --- commands/alias.py | 18 +++++++----------- conf/help.json | 5 +++-- modules/network.py | 20 ++++++++++++-------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/commands/alias.py b/commands/alias.py index b501c36..baccad5 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -11,16 +11,6 @@ class AliasCommand: if length == 1: info(dump(main.alias)) return - elif length == 2: - if spl[1] == "add": - nextNum = max(main.alias.keys())+1 - main.alias[nextNum] = alias.generate_alias() - success("Generated new alias: %i" % nextNum) - main.saveConf("alias") - return - else: - incUsage("alias") - return elif length == 3: if spl[1] == "add": if not spl[2].isdigit(): @@ -28,7 +18,10 @@ class AliasCommand: return num = int(spl[2]) for i in range(num): - nextNum = max(main.alias.keys())+1 + if len(main.alias.keys()) == 0: + nextNum = 1 + else: + nextNum = max(main.alias.keys())+1 main.alias[nextNum] = alias.generate_alias() success("Generated new alias: %i" % nextNum) main.saveConf("alias") @@ -38,6 +31,9 @@ class AliasCommand: failure("Must be a number, not %s" % spl[2]) return num = int(spl[2]) + if not num in main.alias.keys(): + failure("No such alias: %i" % num) + return failed = False for i in main.network.keys(): if num in main.network[i].aliases.keys(): diff --git a/conf/help.json b/conf/help.json index 1e00114..5730e73 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,12 +20,13 @@ "users": "users [ ...]", "relay": "relay [] []", "network": "network [
]", - "alias": "alias []", + "alias": "alias [] []", "auto": "auto ", "cmd": "cmd ", "token": "token [] []", "all": "all ", "allc": "allc <(network)|(alias)> ", "admall": "admall ", - "swho": "swho []" + "swho": "swho []", + "exec": "exec " } diff --git a/modules/network.py b/modules/network.py index e4eefb3..6b45880 100644 --- a/modules/network.py +++ b/modules/network.py @@ -38,14 +38,8 @@ class Network: # self.start_bot(num) return num, main.alias[num]["nick"] - def delete_relay(self, id): - del self.relays[id] - del self.aliases[id] - #del main.alias[id] - Aliases are global per num, so don't delete them! - - def seppuku(self): - # Removes all bots in preperation for deletion - for i in self.relays.keys(): + def killAliases(self, aliasList): + for i in aliasList: name = self.net+str(i) if name in main.ReactorPool.keys(): if name in main.FactoryPool.keys(): @@ -56,6 +50,16 @@ class Network: del main.ReactorPool[name] del main.FactoryPool[name] + def delete_relay(self, id): + del self.relays[id] + del self.aliases[id] + #del main.alias[id] - Aliases are global per num, so don't delete them! + self.killAliases([id]) + + def seppuku(self): + # Removes all bots in preperation for deletion + self.killAliases(self.relay.keys()) + def start_bot(self, num): # a single name is given to relays in the backend # e.g. freenode1 for the first relay on freenode network From 872d08be3ed8c4568a9d29dceff5f2270a55b00b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 2 Oct 2019 20:46:00 +0100 Subject: [PATCH 148/394] Implement exec command for running raw Python code --- commands/exec.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 commands/exec.py diff --git a/commands/exec.py b/commands/exec.py new file mode 100644 index 0000000..50080f1 --- /dev/null +++ b/commands/exec.py @@ -0,0 +1,16 @@ +import main + +class ExecCommand: + def __init__(self, *args): + self.exec(*args) + + def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 1: + info(str(exec(" ".join(spl[1:])))) + return + else: + incUsage("exec") + return + else: + incUsage(None) From 89894287b3b0b2404d2e126a3229cac6dc2021dd Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 2 Oct 2019 21:25:15 +0100 Subject: [PATCH 149/394] Add error handling to exec command and fix minor bug in cleaning up relays --- commands/exec.py | 7 ++++++- core/bot.py | 2 ++ modules/network.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/commands/exec.py b/commands/exec.py index 50080f1..c199aa8 100644 --- a/commands/exec.py +++ b/commands/exec.py @@ -7,7 +7,12 @@ class ExecCommand: def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 1: - info(str(exec(" ".join(spl[1:])))) + try: + rtrn = exec(" ".join(spl[1:])) + except Exception as err: + failure(str(err)) + return + info(str(rtrn)) return else: incUsage("exec") diff --git a/core/bot.py b/core/bot.py index d078ca1..122b9b0 100644 --- a/core/bot.py +++ b/core/bot.py @@ -223,6 +223,8 @@ class IRCBot(IRCClient): self.setNick(self._attemptedNick) def irc_ERR_PASSWDMISMATCH(self, prefix, params): + print(locals()) + print(globals()) log("%s: password mismatch" % self.name) sendAll("%s: password mismatch" % self.name) diff --git a/modules/network.py b/modules/network.py index 6b45880..fbcfaa4 100644 --- a/modules/network.py +++ b/modules/network.py @@ -58,7 +58,7 @@ class Network: def seppuku(self): # Removes all bots in preperation for deletion - self.killAliases(self.relay.keys()) + self.killAliases(self.relays.keys()) def start_bot(self, num): # a single name is given to relays in the backend From ddadeb617caa6cf86952faf92bb3e9e54be4f564 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 3 Oct 2019 18:02:00 +0100 Subject: [PATCH 150/394] Change message to msg in relay output and in functions, include name in connection notifications instead of net --- core/bot.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/core/bot.py b/core/bot.py index 122b9b0..5737a85 100644 --- a/core/bot.py +++ b/core/bot.py @@ -141,8 +141,8 @@ class IRCBot(IRCClient): cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]: # del cast["muser"] - if set(["nick", "ident", "host", "message"]).issubset(set(cast)): - if "message" in cast.keys(): + if set(["nick", "ident", "host", "msg"]).issubset(set(cast)): + if "msg" in cast.keys(): if cast["ident"] == "znc" and cast["host"] == "znc.in": cast["type"] = "znc" cast["name"] = self.name @@ -175,9 +175,9 @@ class IRCBot(IRCClient): castDup["name"] = self.name if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly self.event(**castDup) # don't tell anyone else - if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries - if not cast["message"] == None: - if self.nickname.lower() in cast["message"].lower(): + if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries + if not cast["msg"] == None: + if self.nickname.lower() in cast["msg"].lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "highlight" @@ -190,13 +190,13 @@ class IRCBot(IRCClient): monitor.event(self.name, cast) def privmsg(self, user, channel, msg): - self.event(type="msg", muser=user, target=channel, message=msg) + self.event(type="msg", muser=user, target=channel, msg=msg) def noticed(self, user, channel, msg): - self.event(type="notice", muser=user, target=channel, message=msg) + self.event(type="notice", muser=user, target=channel, msg=msg) def action(self, user, channel, msg): - self.event(type="action", muser=user, target=channel, message=msg) + self.event(type="action", muser=user, target=channel, msg=msg) def get(self, var): try: @@ -379,7 +379,8 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True log("signed on: %s" % self.name) - self.event(type="conn", status="connected") + #self.event(type="conn", status="connected") + sendRelayNotification({"type": "conn", "name": self.name, "status": "signedon"}) def joined(self, channel): if not channel in self.channels: From 78e4d6bd66a3456b60f09e5f801d673126449cef Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 5 Oct 2019 00:51:00 +0100 Subject: [PATCH 151/394] Rename 'message' to 'msg' and 'target' to 'channel' --- conf/help.json | 2 +- core/bot.py | 30 +++++++++++++++--------------- modules/monitor.py | 18 +++++++++--------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/conf/help.json b/conf/help.json index 5730e73..e9d55be 100644 --- a/conf/help.json +++ b/conf/help.json @@ -21,7 +21,7 @@ "relay": "relay [] []", "network": "network [
]", "alias": "alias [] []", - "auto": "auto ", + "auto": "auto [] []", "cmd": "cmd ", "token": "token [] []", "all": "all ", diff --git a/core/bot.py b/core/bot.py index 5737a85..5e20f86 100644 --- a/core/bot.py +++ b/core/bot.py @@ -149,10 +149,10 @@ class IRCBot(IRCClient): del cast["nick"] del cast["ident"] del cast["host"] - del cast["target"] + del cast["channel"] if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: - if "target" in cast.keys() and not cast["type"] == "mode": # don't handle modes here - if cast["target"].lower() == self.nickname.lower(): # as they are target == nickname + if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here + if cast["channel"].lower() == self.nickname.lower(): # as they are channel == nickname #castDup = deepcopy(cast) # however modes are not queries! cast["mtype"] = cast["type"] cast["type"] = "query" @@ -173,7 +173,7 @@ class IRCBot(IRCClient): castDup["mtype"] = cast["type"] castDup["type"] = "self" castDup["name"] = self.name - if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly + if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly self.event(**castDup) # don't tell anyone else if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries if not cast["msg"] == None: @@ -190,13 +190,13 @@ class IRCBot(IRCClient): monitor.event(self.name, cast) def privmsg(self, user, channel, msg): - self.event(type="msg", muser=user, target=channel, msg=msg) + self.event(type="msg", muser=user, channel=channel, msg=msg) def noticed(self, user, channel, msg): - self.event(type="notice", muser=user, target=channel, msg=msg) + self.event(type="notice", muser=user, channel=channel, msg=msg) def action(self, user, channel, msg): - self.event(type="action", muser=user, target=channel, msg=msg) + self.event(type="action", muser=user, channel=channel, msg=msg) def get(self, var): try: @@ -251,7 +251,7 @@ class IRCBot(IRCClient): return n = self._tempWho[channel][1] n.append([nick, nick, host, server, status, realname]) - self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, target=channel, server=server, status=status) + self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -407,14 +407,14 @@ class IRCBot(IRCClient): #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so def left(self, user, channel, message): # even if they saw it, they wouldn't react - self.event(type="part", muser=user, target=channel, message=message) + self.event(type="part", muser=user, channel=channel, message=message) self.botLeft(channel) def userJoined(self, user, channel): - self.event(type="join", muser=user, target=channel) + self.event(type="join", muser=user, channel=channel) def userLeft(self, user, channel, message): - self.event(type="part", muser=user, target=channel, message=message) + self.event(type="part", muser=user, channel=channel, message=message) def userQuit(self, user, quitMessage): self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage}) @@ -422,7 +422,7 @@ class IRCBot(IRCClient): def userKicked(self, kickee, channel, kicker, message): if kickee.lower() == self.nickname.lower(): self.botLeft(channel) - self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee) + self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee) def chanlessEvent(self, cast): cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) @@ -437,20 +437,20 @@ class IRCBot(IRCClient): # ones we have common with them realChans = set(chans).intersection(set(self.channels)) for i in realChans: - cast["target"] = i + cast["channel"] = i self.event(**cast) def userRenamed(self, oldname, newname): self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname}) def topicUpdated(self, user, channel, newTopic): - self.event(type="topic", muser=user, target=channel, message= newTopic) + self.event(type="topic", muser=user, channel=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a) + self.event(type="mode", muser=user, channel=channel, modes=m, status=toset, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): diff --git a/modules/monitor.py b/modules/monitor.py index 3913c6f..0335800 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -54,8 +54,8 @@ def magicFunction(A, B): return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen - if not "target" in c.keys(): - c["target"] = None + if not "channel" in c.keys(): + c["channel"] = None if dedup(numName, c): return @@ -68,13 +68,13 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) elif c["type"] == "kick": userinfo.editUser(c["name"], c["muser"]) - userinfo.delUserByNick(c["name"], c["target"], c["user"]) + userinfo.delUserByNick(c["name"], c["channel"], c["user"]) elif c["type"] == "quit": userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) elif c["type"] == "join": - userinfo.addUser(c["name"], c["target"], c["nick"], c["muser"]) + userinfo.addUser(c["name"], c["channel"], c["nick"], c["muser"]) elif c["type"] == "part": - userinfo.delUser(c["name"], c["target"], c["nick"], c["muser"]) + userinfo.delUser(c["name"], c["channel"], c["nick"], c["muser"]) if "mtype" in c.keys(): if c["mtype"] == "nick": @@ -85,7 +85,7 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe sendRelayNotification(c) # only monitors below - monitorGroups = testNetTarget(c["name"], c["target"]) + monitorGroups = testNetTarget(c["name"], c["channel"]) if monitorGroups == False: return for monitorGroup in monitorGroups: @@ -95,11 +95,11 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): if isinstance(main.monitor[monitorGroup]["send"][i], bool): - sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "errdeliv"}) + sendRelayNotification({"type": "err", "name": name, "channel": channel, "message": c, "reason": "errdeliv"}) continue if not i in main.pool.keys(): - sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noname"}) + sendRelayNotification({"type": "err", "name": name, "channel": channel, "message": c, "reason": "noname"}) if not i in main.IRCPool.keys(): - sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noinstance"}) + sendRelayNotification({"type": "err", "name": name, "channel": channel, "message": c, "reason": "noinstance"}) for x in main.monitor[monitorGroup]["send"][i]: main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["name"], c)) From 5eda50af13b953521d1c717bf1ec5f759ba5d1c7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 5 Oct 2019 18:13:04 +0100 Subject: [PATCH 152/394] Use net and num instead of name in relay output --- core/bot.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/core/bot.py b/core/bot.py index 5e20f86..8c33855 100644 --- a/core/bot.py +++ b/core/bot.py @@ -145,7 +145,7 @@ class IRCBot(IRCClient): if "msg" in cast.keys(): if cast["ident"] == "znc" and cast["host"] == "znc.in": cast["type"] = "znc" - cast["name"] = self.name + cast["num"] = self.num del cast["nick"] del cast["ident"] del cast["host"] @@ -156,7 +156,7 @@ class IRCBot(IRCClient): #castDup = deepcopy(cast) # however modes are not queries! cast["mtype"] = cast["type"] cast["type"] = "query" - cast["name"] = self.name + cast["num"] = self.num #self.event(**castDup) # Don't call self.event for this one because queries are not events on a # channel, but we still want to see them @@ -165,14 +165,14 @@ class IRCBot(IRCClient): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" - castDup["name"] = self.name + cast["num"] = self.num self.event(**castDup) if "nick" in cast.keys(): if cast["nick"].lower() == self.nickname.lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" - castDup["name"] = self.name + cast["num"] = self.num if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly self.event(**castDup) # don't tell anyone else if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries @@ -181,13 +181,13 @@ class IRCBot(IRCClient): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "highlight" - castDup["name"] = self.name + cast["num"] = self.num self.event(**castDup) - if not "name" in cast.keys(): - cast["name"] = self.net + if not "net" in cast.keys(): + cast["net"] = self.net counters.event(self.net, cast["type"]) - monitor.event(self.name, cast) + monitor.event(self.net, cast) def privmsg(self, user, channel, msg): self.event(type="msg", muser=user, channel=channel, msg=msg) @@ -225,8 +225,8 @@ class IRCBot(IRCClient): def irc_ERR_PASSWDMISMATCH(self, prefix, params): print(locals()) print(globals()) - log("%s: password mismatch" % self.name) - sendAll("%s: password mismatch" % self.name) + log("%s - %i: password mismatch" % (self.net, self.num)) + sendAll("%s - %i: password mismatch" % (self.net, self.num)) def _who(self, channel): d = Deferred() @@ -378,9 +378,9 @@ class IRCBot(IRCClient): def signedOn(self): self.connected = True - log("signed on: %s" % self.name) + log("signed on: %s - %i" % (self.net, self.num)) #self.event(type="conn", status="connected") - sendRelayNotification({"type": "conn", "name": self.name, "status": "signedon"}) + sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"}) def joined(self, channel): if not channel in self.channels: @@ -426,7 +426,9 @@ class IRCBot(IRCClient): def chanlessEvent(self, cast): cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) - if dedup(self.name, cast): + if dedup(self.name, cast): # Needs to be kept self.name until the dedup + # function is converted to the new net, num + # format return # stop right there sir! chans = userinfo.getChanList(self.net, cast["nick"]) if chans == None: @@ -456,10 +458,10 @@ class IRCBotFactory(ReconnectingClientFactory): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): if net == None: self.num = num - self.name = "Relay to %i" % num + self.name = "relay - %i" % num self.relay = True else: - self.name = net + str(num) + self.name = net+str(num) self.num = num self.net = net self.relay = False @@ -488,10 +490,10 @@ class IRCBotFactory(ReconnectingClientFactory): self.client.connected = False self.client.channels = [] error = reason.getErrorMessage() - log("%s: connection lost: %s" % (self.name, error)) + log("%s - %i: connection lost: %s" % (self.net, self.num, error)) if not self.relay: - sendAll("%s: connection lost: %s" % (self.name, error)) - sendRelayNotification({"type": "conn", "name": self.name, "status": "lost", "message": error}) + sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) + sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -500,10 +502,10 @@ class IRCBotFactory(ReconnectingClientFactory): self.client.connected = False self.client.channels = [] error = reason.getErrorMessage() - log("%s: connection failed: %s" % (self.name, error)) + log("%s - %i: connection failed: %s" % (self.net, self.num, error)) if not self.relay: - sendAll("%s: connection failed: %s" % (self.name, error)) - sendRelayNotification({"type": "conn", "name": self.name, "status": "failed", "message": error}) + sendAll("%s -%s: connection failed: %s" % (self.net, self.num, error)) + sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) From f0fff7c9589d38c53b735b1f77acd5725639e93d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 5 Oct 2019 18:20:51 +0100 Subject: [PATCH 153/394] Implement sorting relay output by custom keys --- modules/monitor.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/modules/monitor.py b/modules/monitor.py index 0335800..d55c1c5 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -7,6 +7,10 @@ from core.relay import sendRelayNotification from modules import userinfo from utils.dedup import dedup +order = ["type", "net", "num", "channel", "msg", "nick", + "ident", "host", "mtype", "user", "modes", "modeargs" + "realname", "server", "status"] + def testNetTarget(name, target): called = False for i in main.monitor.keys(): @@ -63,29 +67,29 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe # need to check if this was received from a relay # in which case, do not do this if c["type"] in ["msg", "notice", "action", "topic", "mode"]: - userinfo.editUser(c["name"], c["muser"]) + userinfo.editUser(c["net"], c["muser"]) elif c["type"] == "nick": - userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) elif c["type"] == "kick": - userinfo.editUser(c["name"], c["muser"]) - userinfo.delUserByNick(c["name"], c["channel"], c["user"]) + userinfo.editUser(c["net"], c["muser"]) + userinfo.delUserByNick(c["net"], c["channel"], c["user"]) elif c["type"] == "quit": - userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) + userinfo.delUserByNetwork(c["net"], c["nick"], c["muser"]) elif c["type"] == "join": - userinfo.addUser(c["name"], c["channel"], c["nick"], c["muser"]) + userinfo.addUser(c["net"], c["channel"], c["nick"], c["muser"]) elif c["type"] == "part": - userinfo.delUser(c["name"], c["channel"], c["nick"], c["muser"]) + userinfo.delUser(c["net"], c["channel"], c["nick"], c["muser"]) if "mtype" in c.keys(): if c["mtype"] == "nick": - userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) if "muser" in c.keys(): del c["muser"] - sendRelayNotification(c) + sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order # only monitors below - monitorGroups = testNetTarget(c["name"], c["channel"]) + monitorGroups = testNetTarget(c["net"], c["channel"]) if monitorGroups == False: return for monitorGroup in monitorGroups: @@ -95,11 +99,11 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): if isinstance(main.monitor[monitorGroup]["send"][i], bool): - sendRelayNotification({"type": "err", "name": name, "channel": channel, "message": c, "reason": "errdeliv"}) + sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "errdeliv"}) continue if not i in main.pool.keys(): - sendRelayNotification({"type": "err", "name": name, "channel": channel, "message": c, "reason": "noname"}) + sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noname"}) if not i in main.IRCPool.keys(): - sendRelayNotification({"type": "err", "name": name, "channel": channel, "message": c, "reason": "noinstance"}) + sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noinstance"}) for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["name"], c)) + main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["net"], c)) From aa54759337ce27b5af1e850be06191d24bfa61d3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 5 Oct 2019 18:22:14 +0100 Subject: [PATCH 154/394] Additional error checking on user record deletion --- modules/userinfo.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index e4035e8..e5cc764 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -108,14 +108,17 @@ def addUser(name, channel, nick, user): p.sadd(chanspace, channel) p.execute() -def delUser(name, channel, nick, user): +def delUser(name, channel, nick, user): gnamespace, namespace, chanspace = getNamespace(name, channel, nick) p = main.r.pipeline() channels = main.r.smembers(chanspace) p.srem(namespace, nick) if channels == {channel.encode()}: p.delete(chanspace) - p.srem(gnamespace, user) + if user: + p.srem(gnamespace, user) + else: + warn("Attempt to delete nonexistent user: %s" % user) else: p.srem(chanspace, channel) p.execute() From 15b394bd7938cba61b2da6567730e63667058ba0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 6 Oct 2019 21:10:44 +0100 Subject: [PATCH 155/394] Implement requesting a LIST and parsing the output --- core/bot.py | 53 +++++++++++++++++++++++++++++++++++---------- modules/chankeep.py | 1 + 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 modules/chankeep.py diff --git a/core/bot.py b/core/bot.py index 8c33855..7ac0820 100644 --- a/core/bot.py +++ b/core/bot.py @@ -112,10 +112,12 @@ class IRCBot(IRCClient): self.versionEnv = None self.sourceURL = None - self._tempWho = {} - self._getWho = {} + self._getWho = {} # LoopingCall objects -- needed to be able to stop them - self._names = {} + self._tempWho = {} # Temporary storage for gathering WHO info + self._tempNames = {} # Temporary storage for gathering NAMES info + self._tempList = ([], []) # Temporary storage for gathering LIST info + self.listOngoing = False def parsen(self, user): step = user.split("!") @@ -276,28 +278,28 @@ class IRCBot(IRCClient): def names(self, channel): d = Deferred() - if channel not in self._names: - self._names[channel] = ([], []) - self._names[channel][0].append(d) + if channel not in self._tempNames: + self._tempNames[channel] = ([], []) + self._tempNames[channel][0].append(d) self.sendLine("NAMES %s" % channel) return d def irc_RPL_NAMREPLY(self, prefix, params): channel = params[2] nicklist = params[3].split(' ') - if channel not in self._names: + if channel not in self._tempNames: return - n = self._names[channel][1] + n = self._tempNames[channel][1] n.append(nicklist) def irc_RPL_ENDOFNAMES(self, prefix, params): channel = params[1] - if channel not in self._names: + if channel not in self._tempNames: return - callbacks, namelist = self._names[channel] + callbacks, namelist = self._tempNames[channel] for cb in callbacks: cb.callback((channel, namelist)) - del self._names[channel] + del self._tempNames[channel] def got_names(self, nicklist): newNicklist = [] @@ -308,6 +310,35 @@ class IRCBot(IRCClient): newNicklist.append(f) userinfo.initialNames(self.net, nicklist[0], newNicklist) + def _list(self): + d = Deferred() + self._tempList = ([], []) + self._tempList[0].append(d) + self.sendLine("LIST") + return d + + def list(self): + if not self.listOngoing: + self._list().addCallback(self.got_list) + self.listOngoing = True + + def irc_RPL_LIST(self, prefix, params): + channel = params[1] + users = params[2] + topic = params[3] + self._tempList[1].append([channel, users, topic]) + + def irc_RPL_LISTEND(self, prefix, params): + callbacks, info = self._tempList + for cb in callbacks: + cb.callback((info)) + self.listOngoing = False + self._tempList[0].clear() + self._tempList[1].clear() + + def got_list(self, listinfo): + print("have list", listinfo) + #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): """ diff --git a/modules/chankeep.py b/modules/chankeep.py new file mode 100644 index 0000000..8ee9bae --- /dev/null +++ b/modules/chankeep.py @@ -0,0 +1 @@ +import main From 06d3dd4d7e3f790aab8e071e136a5034ba585ad9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 18:15:23 +0100 Subject: [PATCH 156/394] Implement storing analytics on a LIST response --- core/bot.py | 25 ++++++++++++++++++++++--- modules/chankeep.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ modules/network.py | 1 + 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/core/bot.py b/core/bot.py index 7ac0820..5c3dc9b 100644 --- a/core/bot.py +++ b/core/bot.py @@ -11,6 +11,7 @@ from copy import deepcopy from modules import userinfo from modules import counters from modules import monitor +from modules import chankeep from core.relay import sendRelayNotification from utils.dedup import dedup @@ -119,6 +120,8 @@ class IRCBot(IRCClient): self._tempList = ([], []) # Temporary storage for gathering LIST info self.listOngoing = False + self.chanLimit = 0 + def parsen(self, user): step = user.split("!") nick = step[0] @@ -314,13 +317,15 @@ class IRCBot(IRCClient): d = Deferred() self._tempList = ([], []) self._tempList[0].append(d) - self.sendLine("LIST") + self.sendLine("LIST >0") return d def list(self): if not self.listOngoing: self._list().addCallback(self.got_list) - self.listOngoing = True + + def irc_RPL_LISTSTART(self, prefix, params): + self.listOngoing = True def irc_RPL_LIST(self, prefix, params): channel = params[1] @@ -337,7 +342,20 @@ class IRCBot(IRCClient): self._tempList[1].clear() def got_list(self, listinfo): - print("have list", listinfo) + chankeep.initialList(self.net, self.num, listinfo, self.chanLimit) + + def isupport(self, options): + for i in options: + if i.startswith("CHANLIMIT"): + if "#" in i: + split = i.split("#") + if len(split) >= 2: + chanLimit = split[-1].replace(":", "") + try: + self.chanLimit = int(chanLimit) + return + except TypeError: + error("Invalid CHANLIMIT: %s" % i) #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): @@ -489,6 +507,7 @@ class IRCBotFactory(ReconnectingClientFactory): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): if net == None: self.num = num + self.net = None self.name = "relay - %i" % num self.relay = True else: diff --git a/modules/chankeep.py b/modules/chankeep.py index 8ee9bae..0f18e9b 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -1 +1,45 @@ import main +from utils.logging.log import * + +# make this into a threaded function +def initialList(net, num, listinfo, chanLimit): + #listinfo = sorted(listinfo, key=lambda x: xdd[0]) + listLength = len(listinfo) + cumul = 0 + try: + cumul += sum(int(i[1]) for i in listinfo) + except TypeError: + warn("Bad LIST data received from %s - %i" % (net, num)) + return + mean = cumul/listLength + sigLength = 0 + insigLength = 0 + sigCumul = 0 + insigCumul = 0 + for i in listinfo: + if int(i[1]) > mean: + sigLength += 1 + sigCumul += int(i[1]) + elif int(i[1]) < mean: + insigLength += 1 + insigCumul += int(i[1]) + + if not net in main.network.keys(): + warn("Cannot write list info - no network entry for %s" % net) + return + main.network[net].list["mean"] = mean + main.network[net].list["total"] = listLength + main.network[net].list["sigtotal"] = sigLength + main.network[net].list["insigtotal"] = insigLength + main.network[net].list["sigperc"] = sigLength/listLength*100 + main.network[net].list["insigperc"] = insigLength/listLength*100 + main.network[net].list["cumul"] = cumul + main.network[net].list["sigcumul"] = sigCumul + main.network[net].list["insigcumul"] = insigCumul + main.network[net].list["relay"] = listLength/chanLimit + main.network[net].list["sigrelay"] = sigLength/chanLimit + main.network[net].list["insigrelay"] = insigLength/chanLimit + + # Rounding + main.network[net].list = {x: round(y, 4) for x, y in main.network[net].list.items()} + main.saveConf("network") diff --git a/modules/network.py b/modules/network.py index fbcfaa4..a958cc1 100644 --- a/modules/network.py +++ b/modules/network.py @@ -19,6 +19,7 @@ class Network: self.last = 0 self.relays = {} self.aliases = {} + self.list = {} def add_relay(self, num=None): if not num: From c92e61807617a4e277c99c7c5a52f7bfb3c70d9c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 20:53:10 +0100 Subject: [PATCH 157/394] Remove the del command --- commands/del.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 commands/del.py diff --git a/commands/del.py b/commands/del.py deleted file mode 100644 index e1108ad..0000000 --- a/commands/del.py +++ /dev/null @@ -1,37 +0,0 @@ -import main - -class DelCommand: - def __init__(self, *args): - self.delete(*args) - - def delete(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - if authed: - if length == 3: - if not spl[1] in main.network.keys(): - failure("No such network: %s" % spl[1]) - return - if not spl[2].isdigit(): - failure("Must be integer, not %s" % spl[2]) - return - if not int(spl[2]) in main.network[spl[1]].relays.keys(): - failure("No such relay: %s in network %s" % (spl[2], spl[1])) - return - - main.network[spl[1]].delete_relay(int(spl[2])) - name = spl[1]+spl[2] - if name in main.ReactorPool.keys(): - if name in main.FactoryPool.keys(): - main.FactoryPool[name].stopTrying() - main.ReactorPool[name].disconnect() - if name in main.IRCPool.keys(): - del main.IRCPool[name] - del main.ReactorPool[name] - del main.FactoryPool[name] - success("Successfully removed bot: %s" % spl[1]) - main.saveConf("network") - return - else: - incUsage("del") - return - else: - incUsage(None) From a027a0b4f6c21c60d307695df54c9520afee9724 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 20:53:39 +0100 Subject: [PATCH 158/394] Fix getting user records by nickname Use -1 to refer to unlimited records instead of an arbitrarily large number. --- modules/userinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index e5cc764..a569c42 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -132,7 +132,7 @@ def escape(text): def getUserByNick(name, nick): gnamespace = "live.who.%s" % name - usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=9999999) + usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1) if usermatch[1] == []: return False else: From da14a86e70fdb814c2aaf6489f7834f34a322ec7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 21:00:57 +0100 Subject: [PATCH 159/394] Fix getting LIST output to work with more networks * Parse ISUPPORT properly if more than one channel prefix is supported * Retry getting LIST another way is the response is empty --- core/bot.py | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/core/bot.py b/core/bot.py index 5c3dc9b..8089d5e 100644 --- a/core/bot.py +++ b/core/bot.py @@ -19,6 +19,7 @@ from utils.get import getRelay import main from utils.logging.log import * +from utils.logging.debug import * from utils.logging.send import * from twisted.internet.ssl import DefaultOpenSSLContextFactory @@ -119,8 +120,9 @@ class IRCBot(IRCClient): self._tempNames = {} # Temporary storage for gathering NAMES info self._tempList = ([], []) # Temporary storage for gathering LIST info self.listOngoing = False + self.listRetried = False - self.chanLimit = 0 + self.chanlimit = 0 def parsen(self, user): step = user.split("!") @@ -313,16 +315,19 @@ class IRCBot(IRCClient): newNicklist.append(f) userinfo.initialNames(self.net, nicklist[0], newNicklist) - def _list(self): + def _list(self, noargs): d = Deferred() self._tempList = ([], []) self._tempList[0].append(d) - self.sendLine("LIST >0") + if noargs: + self.sendLine("LIST") + else: + self.sendLine("LIST >0") return d - def list(self): + def list(self, noargs=False): if not self.listOngoing: - self._list().addCallback(self.got_list) + self._list(noargs).addCallback(self.got_list) def irc_RPL_LISTSTART(self, prefix, params): self.listOngoing = True @@ -335,27 +340,45 @@ class IRCBot(IRCClient): def irc_RPL_LISTEND(self, prefix, params): callbacks, info = self._tempList + self.listOngoing = False for cb in callbacks: cb.callback((info)) - self.listOngoing = False + noResults = False + if len(self._tempList[1]) == 0: + noResults = True self._tempList[0].clear() self._tempList[1].clear() + if noResults: + if not self.listRetried: + self.list(True) + else: + warn("List still empty after retry: %s - %i" % (net, num)) + self.listRetried = False + return + else: + self.listRetried = False def got_list(self, listinfo): - chankeep.initialList(self.net, self.num, listinfo, self.chanLimit) + if len(listinfo) == 0: # probably ngircd not supporting LIST >0 + return + + chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) + + def irc_unknown(self, prefix, command, params): + debug("Unknown message: %s - %s - %s" % (prefix, command, params)) def isupport(self, options): for i in options: if i.startswith("CHANLIMIT"): - if "#" in i: - split = i.split("#") + if ":" in i: + split = i.split(":") if len(split) >= 2: - chanLimit = split[-1].replace(":", "") + chanlimit = split[1] try: - self.chanLimit = int(chanLimit) + self.chanlimit = int(chanlimit) return except TypeError: - error("Invalid CHANLIMIT: %s" % i) + warn("Invalid CHANLIMIT: %s" % i) #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): From 7e90080a2c6db8d7b4c809965093cae48fa09d6e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 21:07:54 +0100 Subject: [PATCH 160/394] Set up Redis database which isn't cleared on quit --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index e7e3a6c..6bb26ad 100644 --- a/main.py +++ b/main.py @@ -80,8 +80,9 @@ def initConf(): loadConf(i) def initMain(): - global r + global r, g initConf() - r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) + r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit + g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent From 33cb173db6bc57b3e3dd42c6fa5640e6b1da5f56 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 21:10:42 +0100 Subject: [PATCH 161/394] Switch to using Redis for LIST storage --- modules/chankeep.py | 91 +++++++++++++++++++++++++++++++-------------- modules/network.py | 3 +- 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 0f18e9b..0ed89ac 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -1,45 +1,82 @@ import main from utils.logging.log import * +from utils.logging.debug import * +from copy import deepcopy +from twisted.internet.threads import deferToThread -# make this into a threaded function -def initialList(net, num, listinfo, chanLimit): +def provisionInstances(net, relaysNeeded): + #num, alias = + pass + +def purgeRecords(net): + base = "list.%s" % net + p = main.g.pipeline() + existingChans = main.g.smembers("list."+net) + for i in existingChans: + p.delete(base+"."+i.decode("utf-8")) + p.execute() + +def _nukeNetwork(net): + purgeRecords(net) + p = main.g.pipeline() + p.delete("analytics.list."+net) + p.delete("list."+net) + p.execute() + +def nukeNetwork(net): + deferToThread(_nukeNetwork, net) + +def _initialList(net, num, listinfo, chanlimit): #listinfo = sorted(listinfo, key=lambda x: xdd[0]) - listLength = len(listinfo) + listlength = len(listinfo) cumul = 0 try: cumul += sum(int(i[1]) for i in listinfo) except TypeError: warn("Bad LIST data received from %s - %i" % (net, num)) return - mean = cumul/listLength - sigLength = 0 - insigLength = 0 - sigCumul = 0 - insigCumul = 0 + mean = round(cumul/listlength, 2) + siglength = 0 + insiglength = 0 + sigcumul = 0 + insigcumul = 0 for i in listinfo: if int(i[1]) > mean: - sigLength += 1 - sigCumul += int(i[1]) + siglength += 1 + sigcumul += int(i[1]) elif int(i[1]) < mean: - insigLength += 1 - insigCumul += int(i[1]) + insiglength += 1 + insigcumul += int(i[1]) if not net in main.network.keys(): warn("Cannot write list info - no network entry for %s" % net) return - main.network[net].list["mean"] = mean - main.network[net].list["total"] = listLength - main.network[net].list["sigtotal"] = sigLength - main.network[net].list["insigtotal"] = insigLength - main.network[net].list["sigperc"] = sigLength/listLength*100 - main.network[net].list["insigperc"] = insigLength/listLength*100 - main.network[net].list["cumul"] = cumul - main.network[net].list["sigcumul"] = sigCumul - main.network[net].list["insigcumul"] = insigCumul - main.network[net].list["relay"] = listLength/chanLimit - main.network[net].list["sigrelay"] = sigLength/chanLimit - main.network[net].list["insigrelay"] = insigLength/chanLimit + sigrelay = round(siglength/chanlimit, 2) + netbase = "list.%s" % net + abase = "analytics.list.%s" % net + p = main.g.pipeline() + p.hset(abase, "mean", mean) + p.hset(abase, "total", listlength) + p.hset(abase, "sigtotal", siglength) + p.hset(abase, "insigtotal", insiglength) + p.hset(abase, "sigperc", round(siglength/listlength*100, 2)) + p.hset(abase, "insigperc", round(insiglength/listlength*100, 2)) + p.hset(abase, "cumul", cumul) + p.hset(abase, "sigcumul", sigcumul) + p.hset(abase, "insigcumul", insigcumul) + p.hset(abase, "relay", round(listlength/chanlimit, 2)) + p.hset(abase, "sigrelay", sigrelay) + p.hset(abase, "insigrelay", round(insiglength/chanlimit, 2)) + # Purge existing records before writing + purgeRecords(net) + for i in listinfo: + p.rpush(netbase+"."+i[0], i[1]) + p.rpush(netbase+"."+i[0], i[2]) + p.sadd(netbase, i[0]) + + p.execute() + debug("List parsing completed on %s" % net) + +def initialList(net, num, listinfo, chanlimit): + deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) - # Rounding - main.network[net].list = {x: round(y, 4) for x, y in main.network[net].list.items()} - main.saveConf("network") diff --git a/modules/network.py b/modules/network.py index a958cc1..3758460 100644 --- a/modules/network.py +++ b/modules/network.py @@ -2,6 +2,7 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory import json from modules import alias +from modules.chankeep import nukeNetwork from twisted.internet import reactor from core.bot import IRCBot, IRCBotFactory import main @@ -19,7 +20,6 @@ class Network: self.last = 0 self.relays = {} self.aliases = {} - self.list = {} def add_relay(self, num=None): if not num: @@ -60,6 +60,7 @@ class Network: def seppuku(self): # Removes all bots in preperation for deletion self.killAliases(self.relays.keys()) + nukeNetwork(self.net) def start_bot(self, num): # a single name is given to relays in the backend From 31c9e6479023c8699efc404fd24260cd672dcf5f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 21:11:04 +0100 Subject: [PATCH 162/394] Implement function to set up multiple relays --- modules/provision.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/provision.py b/modules/provision.py index af623e1..41d669c 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -64,3 +64,10 @@ def provisionRelay(num, network): if main.config["ConnectOnCreate"]: main.network[network].start_bot(num) return alias + +def provisionMultipleRelays(net, relaysNeeded): + for i in range(1, relaysNeeded): + num, alias = main.network[net].add_relay() + provisionRelay(num, net) + main.saveConf("network") + From f9619a5bc4761f8f2a53303c34f6d40543eb5417 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 8 Oct 2019 21:12:06 +0100 Subject: [PATCH 163/394] Remove help entry for del --- conf/help.json | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/help.json b/conf/help.json index e9d55be..3c41b22 100644 --- a/conf/help.json +++ b/conf/help.json @@ -1,7 +1,6 @@ { "pass": "pass ", "logout": "logout", - "del": "del ", "mod": "mod [] []", "who": "who ", "join": "join []", From 324af04de034ede38173acd494eab60c72f59d8e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 11 Oct 2019 13:02:39 +0100 Subject: [PATCH 164/394] Don't return the same thing twice The add_relay and provisionRelay functions both returned the alias. Only use the output from one function. --- commands/auto.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/commands/auto.py b/commands/auto.py index 8832bb7..9dde92c 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -18,19 +18,18 @@ class AutoCommand: num, alias = main.network[spl[1]].add_relay(relayNum) success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) main.saveConf("network") - rtrn = provision.provisionRelay(relayNum, spl[1]) - success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], rtrn)) + provision.provisionRelay(relayNum, spl[1]) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], alias)) return elif length == 2: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return for i in main.alias.keys(): - print("num", i) num, alias = main.network[spl[1]].add_relay(i) success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) - rtrn = provision.provisionRelay(num, spl[1]) - success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, rtrn)) + provision.provisionRelay(num, spl[1]) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias)) main.saveConf("network") return elif length == 1: @@ -38,8 +37,8 @@ class AutoCommand: for x in main.alias.keys(): num, alias = main.network[i].add_relay(x) success("Successfully created relay %i on network %s with alias %s" % (num, i, alias)) - rtrn = provision.provisionRelay(num, i) - success("Started provisioning network %s on relay %s for alias %s" % (i, num, rtrn)) + provision.provisionRelay(num, i) + success("Started provisioning network %s on relay %s for alias %s" % (i, num, alias)) main.saveConf("network") return else: From c3d0cb04b64eb425f798628c02d182c34fb8d1b2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 11 Oct 2019 13:04:58 +0100 Subject: [PATCH 165/394] Pass all arguments from debug into print Allows for multi-argument debug() statements without ugly + or %s operators. --- utils/logging/debug.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/logging/debug.py b/utils/logging/debug.py index 5a7b067..4b05602 100644 --- a/utils/logging/debug.py +++ b/utils/logging/debug.py @@ -1,7 +1,7 @@ import main # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main # to read the setting -def debug(data): +def debug(*data): if main.config["Debug"]: - print("[DEBUG]", data) + print("[DEBUG]", *data) From 7a6e3338c0b89c6a8973539e87fc9a741ace7996 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 11 Oct 2019 13:07:57 +0100 Subject: [PATCH 166/394] Implement ChanKeep joining functions * Low-key channel joining with incrementally increasing delay * Spin up needed instances to be able to cover a certain channel space * Fix provisioning functions to prevent race conditions with lots of relays being created at once * Tweakable switchover from covering all channels to only covering channels with more users than the mean of the cumulative user count --- conf/example/config.json | 4 +++ conf/help.json | 1 + core/bot.py | 63 +++++++++++++++++++++++++++++---------- main.py | 1 + modules/chankeep.py | 64 ++++++++++++++++++++++++++++++++++++---- modules/network.py | 6 ++-- modules/provision.py | 24 +++++++-------- 7 files changed, 126 insertions(+), 37 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 0e6117d..7026833 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -22,6 +22,10 @@ "User": "sir", "Password": "sir" }, + "ChanKeep": { + "MaxRelay": 30, + "SigSwitch": 20 + }, "Dist": { "Enabled": true, "SendOutput": false, diff --git a/conf/help.json b/conf/help.json index 3c41b22..0f561c4 100644 --- a/conf/help.json +++ b/conf/help.json @@ -27,5 +27,6 @@ "allc": "allc <(network)|(alias)> ", "admall": "admall ", "swho": "swho []", + "list": "list ", "exec": "exec " } diff --git a/core/bot.py b/core/bot.py index 8089d5e..c8c35fd 100644 --- a/core/bot.py +++ b/core/bot.py @@ -1,8 +1,8 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall, deferLater -from twisted.internet import reactor +from twisted.internet.task import LoopingCall +from twisted.internet import reactor, task from string import digits from random import randint @@ -25,7 +25,6 @@ from utils.logging.send import * from twisted.internet.ssl import DefaultOpenSSLContextFactory def deliverRelayCommands(num, relayCommands, user=None, stage2=None): - # where relay is a dictionary extracted from the Network object keyFN = main.certPath+main.config["Key"] certFN = main.certPath+main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), @@ -53,6 +52,7 @@ class IRCRelay(IRCClient): self.relayCommands = relayCommands self.num = num self.stage2 = stage2 + self.loop = None def parsen(self, user): step = user.split("!") @@ -68,6 +68,10 @@ class IRCRelay(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) + if "does not exist" in msg or "doesn't exist" in msg: + error("ZNC issue:", msg) + if "Unable to load" in msg: + error("ZNC issue:", msg) if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]: nick = nick[1:] if nick in self.relayCommands.keys(): @@ -77,20 +81,27 @@ class IRCRelay(IRCClient): log("%s: relay password mismatch" % self.num) sendAll("%s: relay password mismatch" % self.num) - def signedOn(self): - self.connected = True - log("signed on as a relay: %s" % self.num) - #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares - for i in self.relayCommands.keys(): - for x in self.relayCommands[i]: - self.msg(main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) + def sendStage2(self): if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []] if not len(self.stage2) == 0: user = self.stage2[0].pop(0) commands = self.stage2[0].pop(0) del self.stage2[0] deliverRelayCommands(self.num, commands, user, self.stage2) - deferLater(reactor, 1, self.transport.loseConnection) + + def signedOn(self): + self.connected = True + log("signed on as a relay: %s" % self.num) + #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares + sleeptime = 0 + increment = 0.8 + for i in self.relayCommands.keys(): + for x in self.relayCommands[i]: + reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) + sleeptime += increment + increment += 0.8 + reactor.callLater(sleeptime, self.sendStage2) + reactor.callLater(sleeptime+5, self.transport.loseConnection) return class IRCBot(IRCClient): @@ -140,6 +151,26 @@ class IRCBot(IRCClient): return [nick, ident, host] + def joinChannels(self, channels): + sleeptime = 0.0 + increment = 0.8 + for i in channels: + if not i in self.channels: + debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds") + reactor.callLater(sleeptime, self.join, i) + sleeptime += increment + if sleeptime == 10: + sleeptime = 0.0 + increment = 0.7 + increment += 0.1 + else: + print("Already on %s, skipping." % i) + + def checkChannels(self): + if self.net in main.TempChan.keys(): + if self.num in main.TempChan[self.net].keys(): + self.joinChannels(main.TempChan[self.net][self.num]) + def event(self, **cast): for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over if cast[i] == "": # a dictionary that changes length with each iteration @@ -157,6 +188,8 @@ class IRCBot(IRCClient): del cast["ident"] del cast["host"] del cast["channel"] + if "Disconnected from IRC" in cast["msg"]: + self.connected = False if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here if cast["channel"].lower() == self.nickname.lower(): # as they are channel == nickname @@ -230,8 +263,6 @@ class IRCBot(IRCClient): self.setNick(self._attemptedNick) def irc_ERR_PASSWDMISMATCH(self, prefix, params): - print(locals()) - print(globals()) log("%s - %i: password mismatch" % (self.net, self.num)) sendAll("%s - %i: password mismatch" % (self.net, self.num)) @@ -364,9 +395,6 @@ class IRCBot(IRCClient): chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) - def irc_unknown(self, prefix, command, params): - debug("Unknown message: %s - %s - %s" % (prefix, command, params)) - def isupport(self, options): for i in options: if i.startswith("CHANLIMIT"): @@ -379,6 +407,9 @@ class IRCBot(IRCClient): return except TypeError: warn("Invalid CHANLIMIT: %s" % i) + if self.num == 1: # Only one instance should do a list, so + self.list() # why not this one? :P + self.checkChannels() #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): diff --git a/main.py b/main.py index 6bb26ad..1e2a767 100644 --- a/main.py +++ b/main.py @@ -28,6 +28,7 @@ relayConnections = {} IRCPool = {} ReactorPool = {} FactoryPool = {} +TempChan = {} MonitorPool = [] diff --git a/modules/chankeep.py b/modules/chankeep.py index 0ed89ac..0248bb0 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -2,11 +2,61 @@ import main from utils.logging.log import * from utils.logging.debug import * from copy import deepcopy +from math import ceil +from modules.provision import provisionMultipleRelays from twisted.internet.threads import deferToThread +from numpy import array_split -def provisionInstances(net, relaysNeeded): - #num, alias = - pass +def allRelaysActive(net): + relayNum = len(main.network[net].relays.keys()) + existNum = 0 + for i in main.network[net].relays.keys(): + name = net+str(i) + if name in main.IRCPool.keys(): + existNum += 1 + if existNum == relayNum: + return True + return False + +def populateChans(net, clist, relay): + divided = array_split(clist, relay) + for i in range(0, len(divided)): + if net in main.TempChan.keys(): + main.TempChan[net][i] = divided[i] + else: + main.TempChan[net] = {i: divided[i]} + +def notifyJoin(net): + for i in main.network[net].relays.keys(): + name = net+str(i) + if name in main.IRCPool.keys(): + main.IRCPool[name].checkChannels() + +def keepChannels(net, listinfo, mean, sigrelay, relay): + #print("list", listinfo) + #print("sigrelay", sigrelay) + #print("cur", len(main.network[net].relays.keys())) + if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels + coverAll = True + elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels + coverAll = False + if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: + error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) + return + if coverAll: + needed = relay-len(main.network[net].relays.keys()) + flist = [i[0] for i in listinfo] + populateChans(net, flist, relay) + else: + needed = sigrelay-len(main.network[net].relays.keys()) + siglist = [i[0] for i in listinfo if int(i[1]) > mean] + populateChans(net, siglist, sigrelay) + notifyJoin(net) + if needed > 0: + provisionMultipleRelays(net, needed) + + #print("coverall", coverAll) + #print("needed", needed) def purgeRecords(net): base = "list.%s" % net @@ -51,7 +101,8 @@ def _initialList(net, num, listinfo, chanlimit): if not net in main.network.keys(): warn("Cannot write list info - no network entry for %s" % net) return - sigrelay = round(siglength/chanlimit, 2) + sigrelay = ceil(siglength/chanlimit) + relay = ceil(listlength/chanlimit) netbase = "list.%s" % net abase = "analytics.list.%s" % net p = main.g.pipeline() @@ -64,9 +115,9 @@ def _initialList(net, num, listinfo, chanlimit): p.hset(abase, "cumul", cumul) p.hset(abase, "sigcumul", sigcumul) p.hset(abase, "insigcumul", insigcumul) - p.hset(abase, "relay", round(listlength/chanlimit, 2)) + p.hset(abase, "relay", relay) p.hset(abase, "sigrelay", sigrelay) - p.hset(abase, "insigrelay", round(insiglength/chanlimit, 2)) + p.hset(abase, "insigrelay", ceil(insiglength/chanlimit)) # Purge existing records before writing purgeRecords(net) for i in listinfo: @@ -76,6 +127,7 @@ def _initialList(net, num, listinfo, chanlimit): p.execute() debug("List parsing completed on %s" % net) + keepChannels(net, listinfo, mean, sigrelay, relay) def initialList(net, num, listinfo, chanlimit): deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) diff --git a/modules/network.py b/modules/network.py index 3758460..9e235a1 100644 --- a/modules/network.py +++ b/modules/network.py @@ -17,14 +17,16 @@ class Network: self.security = security self.auth = auth - self.last = 0 + self.last = 1 self.relays = {} self.aliases = {} def add_relay(self, num=None): if not num: - self.last += 1 num = self.last + self.last += 1 + elif num == self.last: + self.last += 1 self.relays[num] = { "enabled": main.config["ConnectOnCreate"], "net": self.net, diff --git a/modules/provision.py b/modules/provision.py index 41d669c..d55635e 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,8 +1,10 @@ import main from core.bot import deliverRelayCommands from utils.logging.log import * +from twisted.internet import reactor -def provisionUserData(num, nick, altnick, ident, realname, unused): # last field is password, which we don't want to inherit here, but still want to use * expansion, so this is a bit of a hack +def provisionUserData(num, nick, altnick, ident, realname, unused): # last field is password, + # which we don't want to inherit here, but still want to use * expansion, so this is a bit of a hack commands = {} commands["controlpanel"] = [] commands["controlpanel"].append("AddUser %s %s" % (nick, main.config["Relay"]["Password"])) @@ -16,7 +18,7 @@ def provisionUserData(num, nick, altnick, ident, realname, unused): # last field def provisionNetworkData(num, nick, network, host, port, security, auth, password): commands = {} stage2commands = {} - stage3commands = {} + stage2commands["status"] = [] commands["controlpanel"] = [] commands["controlpanel"].append("AddNetwork %s %s" % (nick, network)) if security == "ssl": @@ -25,26 +27,21 @@ def provisionNetworkData(num, nick, network, host, port, security, auth, passwor elif security == "plain": commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port)) if auth == "sasl": - stage2commands["status"] = [] stage2commands["sasl"] = [] stage2commands["status"].append("LoadMod sasl") stage2commands["sasl"].append("Mechanism plain") stage2commands["sasl"].append("Set %s %s" % (nick, password)) elif auth == "ns": - stage2commands["status"] = [] stage2commands["nickserv"] = [] stage2commands["status"].append("LoadMod nickserv") stage2commands["nickserv"].append("Set %s" % password) if not main.config["ConnectOnCreate"]: - stage3commands["status"] = [] - stage3commands["status"].append("Disconnect") + stage2commands["status"].append("Disconnect") if main.config["Toggles"]["CycleChans"]: - stage2commands["status"] = [] stage2commands["status"].append("LoadMod disconkick") stage2commands["status"].append("LoadMod chansaver") deliverRelayCommands(num, commands, - stage2=[[nick+"/"+network, stage2commands], - [nick+"/"+network, stage3commands]]) + stage2=[[nick+"/"+network, stage2commands]]) return def provisionRelayForNetwork(num, alias, network): @@ -60,14 +57,15 @@ def provisionRelay(num, network): aliasObj = main.alias[num] alias = aliasObj["nick"] provisionUserData(num, *aliasObj.values()) - provisionRelayForNetwork(num, alias, network) + reactor.callLater(5, provisionRelayForNetwork, num, alias, network) if main.config["ConnectOnCreate"]: - main.network[network].start_bot(num) - return alias + reactor.callLater(10, main.network[network].start_bot, num) + return def provisionMultipleRelays(net, relaysNeeded): - for i in range(1, relaysNeeded): + for i in range(relaysNeeded): num, alias = main.network[net].add_relay() + print(relaysNeeded, "for", net, ":", num, alias) provisionRelay(num, net) main.saveConf("network") From 0321651c200f1d9ecb5e63caa52317defa63f874 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 12 Oct 2019 21:05:55 +0100 Subject: [PATCH 167/394] Implement fair channel allocation in ChanKeep * Allocate channels to relays only if they have free space based on their chanlimit value * Minify channels by removing ones that are already covered before passing them off to be joined --- commands/list.py | 26 +++++++++++ core/bot.py | 6 ++- modules/chankeep.py | 101 +++++++++++++++++++++++++++++++------------ modules/provision.py | 1 - 4 files changed, 103 insertions(+), 31 deletions(-) create mode 100644 commands/list.py diff --git a/commands/list.py b/commands/list.py new file mode 100644 index 0000000..44cd366 --- /dev/null +++ b/commands/list.py @@ -0,0 +1,26 @@ +import main + +class ListCommand: + def __init__(self, *args): + self.list(*args) + + def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 2: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + if not 1 in main.network[spl[1]].relays.keys(): + failure("Network has no first instance") + return + if not spl[1]+"1" in main.IRCPool.keys(): + failure("No IRC instance: %s - 1" % spl[1]) + return + main.IRCPool[spl[1]+"1"].list() + success("Requested list with first instance of %s" % spl[1]) + return + else: + incUsage("list") + return + else: + incUsage(None) diff --git a/core/bot.py b/core/bot.py index c8c35fd..fe46749 100644 --- a/core/bot.py +++ b/core/bot.py @@ -164,12 +164,15 @@ class IRCBot(IRCClient): increment = 0.7 increment += 0.1 else: - print("Already on %s, skipping." % i) + error("%s - %i - Cannot join channel we are already on - %s" % (self.net, self.num, i)) def checkChannels(self): if self.net in main.TempChan.keys(): if self.num in main.TempChan[self.net].keys(): self.joinChannels(main.TempChan[self.net][self.num]) + del main.TempChan[self.net][self.num] + if not main.TempChan[self.net]: + del main.TempChan[self.net] def event(self, **cast): for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over @@ -392,7 +395,6 @@ class IRCBot(IRCClient): def got_list(self, listinfo): if len(listinfo) == 0: # probably ngircd not supporting LIST >0 return - chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) def isupport(self, options): diff --git a/modules/chankeep.py b/modules/chankeep.py index 0248bb0..2db206d 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -13,18 +13,46 @@ def allRelaysActive(net): for i in main.network[net].relays.keys(): name = net+str(i) if name in main.IRCPool.keys(): - existNum += 1 + if main.IRCPool[name].connected: + existNum += 1 if existNum == relayNum: return True return False +def getChanFree(net): + chanfree = {} + for i in main.network[net].relays.keys(): + name = net+str(i) + chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels) + return chanfree + +def emptyChanAllocate(net, flist, relay): + chanfree = getChanFree(net) + allocated = {} + toalloc = len(flist) + if toalloc > sum(chanfree.values()): + error("Too many channels to allocate for %s - this is probably a bug" % net) + return False + for i in chanfree.keys(): + for x in range(chanfree[i]): + if not len(flist): + break + if i in allocated.keys(): + allocated[i].append(flist.pop()) + else: + allocated[i] = [flist.pop()] + return allocated + def populateChans(net, clist, relay): - divided = array_split(clist, relay) - for i in range(0, len(divided)): + #divided = array_split(clist, relay) + allocated = emptyChanAllocate(net, clist, relay) + if not allocated: + return + for i in allocated.keys(): if net in main.TempChan.keys(): - main.TempChan[net][i] = divided[i] + main.TempChan[net][i] = allocated[i] else: - main.TempChan[net] = {i: divided[i]} + main.TempChan[net] = {i: allocated[i]} def notifyJoin(net): for i in main.network[net].relays.keys(): @@ -32,10 +60,28 @@ def notifyJoin(net): if name in main.IRCPool.keys(): main.IRCPool[name].checkChannels() +def minifyChans(net, listinfo): + if not allRelaysActive(net): + error("All relays for %s are not active, cannot minify list") + return False + for i in main.network[net].relays.keys(): + name = net+str(i) + for x in main.IRCPool[name].channels: + for y in listinfo: + if y[0] == x: + listinfo.remove(y) + if not listinfo: + log("We're on all the channels we want to be on, dropping LIST") + return False + return listinfo + def keepChannels(net, listinfo, mean, sigrelay, relay): #print("list", listinfo) #print("sigrelay", sigrelay) #print("cur", len(main.network[net].relays.keys())) + listinfo = minifyChans(net, listinfo) + if not listinfo: + return if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels coverAll = True elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels @@ -58,23 +104,24 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): #print("coverall", coverAll) #print("needed", needed) -def purgeRecords(net): - base = "list.%s" % net - p = main.g.pipeline() - existingChans = main.g.smembers("list."+net) - for i in existingChans: - p.delete(base+"."+i.decode("utf-8")) - p.execute() - -def _nukeNetwork(net): - purgeRecords(net) - p = main.g.pipeline() - p.delete("analytics.list."+net) - p.delete("list."+net) - p.execute() +#def purgeRecords(net): +# base = "list.%s" % net +# p = main.g.pipeline() +# existingChans = main.g.smembers("list."+net) +# for i in existingChans: +# p.delete(base+"."+i.decode("utf-8")) +# p.execute() def nukeNetwork(net): - deferToThread(_nukeNetwork, net) + #purgeRecords(net) + #p = main.g.pipeline() + main.g.delete("analytics.list."+net) + #p.delete("list."+net) + #p.execute() + +#def nukeNetwork(net): +# deferToThread(_nukeNetwork, net) + def _initialList(net, num, listinfo, chanlimit): #listinfo = sorted(listinfo, key=lambda x: xdd[0]) @@ -98,9 +145,6 @@ def _initialList(net, num, listinfo, chanlimit): insiglength += 1 insigcumul += int(i[1]) - if not net in main.network.keys(): - warn("Cannot write list info - no network entry for %s" % net) - return sigrelay = ceil(siglength/chanlimit) relay = ceil(listlength/chanlimit) netbase = "list.%s" % net @@ -118,12 +162,13 @@ def _initialList(net, num, listinfo, chanlimit): p.hset(abase, "relay", relay) p.hset(abase, "sigrelay", sigrelay) p.hset(abase, "insigrelay", ceil(insiglength/chanlimit)) + # Purge existing records before writing - purgeRecords(net) - for i in listinfo: - p.rpush(netbase+"."+i[0], i[1]) - p.rpush(netbase+"."+i[0], i[2]) - p.sadd(netbase, i[0]) + #purgeRecords(net) + #for i in listinfo: + # p.rpush(netbase+"."+i[0], i[1]) + # p.rpush(netbase+"."+i[0], i[2]) + # p.sadd(netbase, i[0]) p.execute() debug("List parsing completed on %s" % net) diff --git a/modules/provision.py b/modules/provision.py index d55635e..3b99cfe 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -65,7 +65,6 @@ def provisionRelay(num, network): def provisionMultipleRelays(net, relaysNeeded): for i in range(relaysNeeded): num, alias = main.network[net].add_relay() - print(relaysNeeded, "for", net, ":", num, alias) provisionRelay(num, net) main.saveConf("network") From 6ad6d6dc50d8e78b54f1c80eeea1bed24170d6e8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 12 Oct 2019 21:40:50 +0100 Subject: [PATCH 168/394] Fix channel allocation when relays are provisioned Fix a bug where the channel allocation algorithm failed due to necessary relays not having been provisioned yet. Passed the newly created relay numbers to the allocation function and assumed their CHANMAX would be the same as all other relays for the same network. --- modules/chankeep.py | 40 ++++++++++++++++++++++++++-------------- modules/provision.py | 3 +++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 2db206d..43a3d32 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -5,7 +5,6 @@ from copy import deepcopy from math import ceil from modules.provision import provisionMultipleRelays from twisted.internet.threads import deferToThread -from numpy import array_split def allRelaysActive(net): relayNum = len(main.network[net].relays.keys()) @@ -19,22 +18,35 @@ def allRelaysActive(net): return True return False -def getChanFree(net): +def getChanFree(net, new): chanfree = {} + chanlimits = set() for i in main.network[net].relays.keys(): + if i in new: + continue name = net+str(i) chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels) - return chanfree + chanlimits.add(main.IRCPool[name].chanlimit) -def emptyChanAllocate(net, flist, relay): - chanfree = getChanFree(net) + if not len(chanlimits) == 1: + error("Network %s has servers with different CHANMAX values" % net) + return False + return (chanfree, chanlimits.pop()) + +def emptyChanAllocate(net, flist, relay, new): + chanfree = getChanFree(net, new) + if not chanfree: + return + for i in new: + chanfree[0][i] = chanfree[1] + print("chanfree", chanfree) allocated = {} toalloc = len(flist) - if toalloc > sum(chanfree.values()): + if toalloc > sum(chanfree[0].values()): error("Too many channels to allocate for %s - this is probably a bug" % net) return False - for i in chanfree.keys(): - for x in range(chanfree[i]): + for i in chanfree[0].keys(): + for x in range(chanfree[0][i]): if not len(flist): break if i in allocated.keys(): @@ -43,9 +55,9 @@ def emptyChanAllocate(net, flist, relay): allocated[i] = [flist.pop()] return allocated -def populateChans(net, clist, relay): +def populateChans(net, clist, relay, new): #divided = array_split(clist, relay) - allocated = emptyChanAllocate(net, clist, relay) + allocated = emptyChanAllocate(net, clist, relay, new) if not allocated: return for i in allocated.keys(): @@ -91,15 +103,15 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): return if coverAll: needed = relay-len(main.network[net].relays.keys()) + newNums = provisionMultipleRelays(net, needed) flist = [i[0] for i in listinfo] - populateChans(net, flist, relay) + populateChans(net, flist, relay, newNums) else: needed = sigrelay-len(main.network[net].relays.keys()) + newNums = provisionMultipleRelays(net, needed) siglist = [i[0] for i in listinfo if int(i[1]) > mean] - populateChans(net, siglist, sigrelay) + populateChans(net, siglist, sigrelay, newNums) notifyJoin(net) - if needed > 0: - provisionMultipleRelays(net, needed) #print("coverall", coverAll) #print("needed", needed) diff --git a/modules/provision.py b/modules/provision.py index 3b99cfe..f15d4c6 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -63,8 +63,11 @@ def provisionRelay(num, network): return def provisionMultipleRelays(net, relaysNeeded): + numsProvisioned = [] for i in range(relaysNeeded): num, alias = main.network[net].add_relay() + numsProvisioned.append(num) provisionRelay(num, net) main.saveConf("network") + return numsProvisioned From b97ebe43ab4ca6b1ff290bfbf0c86aa92695068f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 13 Oct 2019 12:37:01 +0100 Subject: [PATCH 169/394] Only start one relay with the auto command We only need one initial relay, as ChanKeep will automatically add as many as it needs when receiving a LIST response. --- commands/auto.py | 42 ++++++++++++++++-------------------------- conf/help.json | 2 +- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/commands/auto.py b/commands/auto.py index 9dde92c..29bd3a8 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -7,38 +7,28 @@ class AutoCommand: def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 3: - if not spl[1] in main.network.keys(): - failure("No such network: %s" % spl[1]) - return - if not spl[2].isdigit(): - failure("Must be a number, not %s" % spl[2]) - return - relayNum = int(spl[2]) - num, alias = main.network[spl[1]].add_relay(relayNum) - success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) + if length == 1: + for i in main.network.keys(): + if 1 in main.network[i].relays.keys(): + info("Skipping %s - first relay exists" % i) + else: + num, alias = main.network[i].add_relay(1) + success("Successfully created first relay on network %s with alias %s" % (i, alias)) + provision.provisionRelay(num, i) + success("Started provisioning network %s on first relay for alias %s" % (i, alias)) main.saveConf("network") - provision.provisionRelay(relayNum, spl[1]) - success("Started provisioning network %s on relay %s for alias %s" % (spl[1], spl[2], alias)) return elif length == 2: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - for i in main.alias.keys(): - num, alias = main.network[spl[1]].add_relay(i) - success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) - provision.provisionRelay(num, spl[1]) - success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias)) - main.saveConf("network") - return - elif length == 1: - for i in main.network.keys(): - for x in main.alias.keys(): - num, alias = main.network[i].add_relay(x) - success("Successfully created relay %i on network %s with alias %s" % (num, i, alias)) - provision.provisionRelay(num, i) - success("Started provisioning network %s on relay %s for alias %s" % (i, num, alias)) + if 1 in main.network[spl[1]].relays.keys(): + failure("First relay exists on %s" % spl[1]) + return + num, alias = main.network[spl[1]].add_relay(1) + success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) + provision.provisionRelay(num, spl[1]) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias)) main.saveConf("network") return else: diff --git a/conf/help.json b/conf/help.json index 0f561c4..f97448f 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,7 +20,7 @@ "relay": "relay [] []", "network": "network [
]", "alias": "alias [] []", - "auto": "auto [] []", + "auto": "auto []", "cmd": "cmd ", "token": "token [] []", "all": "all ", From 5777ef0cfe3921bca6171f35b378fa611d36c12f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 13 Oct 2019 12:38:44 +0100 Subject: [PATCH 170/394] Improve ZNC error message detection --- core/bot.py | 7 +++---- main.py | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/bot.py b/core/bot.py index fe46749..74e52f2 100644 --- a/core/bot.py +++ b/core/bot.py @@ -68,10 +68,9 @@ class IRCRelay(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) - if "does not exist" in msg or "doesn't exist" in msg: - error("ZNC issue:", msg) - if "Unable to load" in msg: - error("ZNC issue:", msg) + for i in main.ZNCErrors: + if i in msg: + error("ZNC issue:", msg) if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]: nick = nick[1:] if nick in self.relayCommands.keys(): diff --git a/main.py b/main.py index 1e2a767..bba9753 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,11 @@ from os import urandom from utils.logging.log import * +ZNCErrors = ["Error:", + "Unable to load", + "does not exist", + "doesn't exist"] + configPath = "conf/" certPath = "cert/" From a64765121a8b5b88923b1584553d420491f230ad Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 13 Oct 2019 12:40:16 +0100 Subject: [PATCH 171/394] Allow multiple arguments for the logging functions --- utils/logging/log.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/logging/log.py b/utils/logging/log.py index 6bb6a99..67b2148 100644 --- a/utils/logging/log.py +++ b/utils/logging/log.py @@ -1,8 +1,8 @@ -def log(data): - print("[LOG]", data) +def log(*data): + print("[LOG]", *data) -def warn(data): - print("[WARNING]", data) +def warn(*data): + print("[WARNING]", *data) -def error(data): - print("[ERROR]", data) +def error(*data): + print("[ERROR]", *data) From f34de8940ff7d8bb2b7517d1559188886cfca04c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 17 Oct 2019 20:19:35 +0100 Subject: [PATCH 172/394] Improve performance in userinfo * Implement a nick -> user mapping, preventing a superfluous SSCAN on the entire dataset for when networks are disconnected * Use one thread for all channels when a network instance is disconnected, instead of one thread per channel * Made returns comprising of only a list into tuples --- core/bot.py | 41 ++++------------- modules/userinfo.py | 108 ++++++++++++++++++++++++++------------------ utils/parsing.py | 15 ++++++ 3 files changed, 88 insertions(+), 76 deletions(-) create mode 100644 utils/parsing.py diff --git a/core/bot.py b/core/bot.py index 74e52f2..73e18f8 100644 --- a/core/bot.py +++ b/core/bot.py @@ -21,6 +21,7 @@ import main from utils.logging.log import * from utils.logging.debug import * from utils.logging.send import * +from utils.parsing import parsen from twisted.internet.ssl import DefaultOpenSSLContextFactory @@ -54,20 +55,8 @@ class IRCRelay(IRCClient): self.stage2 = stage2 self.loop = None - def parsen(self, user): - step = user.split("!") - nick = step[0] - if len(step) == 2: - step2 = step[1].split("@") - ident, host = step2 - else: - ident = nick - host = nick - - return [nick, ident, host] - def privmsg(self, user, channel, msg): - nick, ident, host = self.parsen(user) + nick, ident, host = parsen(user) for i in main.ZNCErrors: if i in msg: error("ZNC issue:", msg) @@ -134,22 +123,6 @@ class IRCBot(IRCClient): self.chanlimit = 0 - def parsen(self, user): - step = user.split("!") - nick = step[0] - if len(step) == 2: - step2 = step[1].split("@") - if len(step2) == 2: - ident, host = step2 - else: - ident = nick - host = nick - else: - ident = nick - host = nick - - return [nick, ident, host] - def joinChannels(self, channels): sleeptime = 0.0 increment = 0.8 @@ -178,7 +151,9 @@ class IRCBot(IRCClient): if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] if "muser" in cast.keys(): - cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) + cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) + # additional checks here to see if it's a server -- irc.example.net + # discard if it is #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]: # del cast["muser"] if set(["nick", "ident", "host", "msg"]).issubset(set(cast)): @@ -507,7 +482,7 @@ class IRCBot(IRCClient): lc = self._getWho[channel] lc.stop() del self._getWho[channel] - userinfo.delChannel(self.net, channel) # < we do not need to deduplicate this + userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so def left(self, user, channel, message): # even if they saw it, they wouldn't react @@ -529,7 +504,7 @@ class IRCBot(IRCClient): self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee) def chanlessEvent(self, cast): - cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) + cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) if dedup(self.name, cast): # Needs to be kept self.name until the dedup # function is converted to the new net, num # format @@ -590,7 +565,7 @@ class IRCBotFactory(ReconnectingClientFactory): def clientConnectionLost(self, connector, reason): if not self.relay: - userinfo.delNetwork(self.net, self.client.channels) + userinfo.delChannels(self.net, self.client.channels) if not self.client == None: self.client.connected = False self.client.channels = [] diff --git a/modules/userinfo.py b/modules/userinfo.py index a569c42..c8c46cb 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -4,12 +4,13 @@ from string import digits import main from utils.logging.log import * from utils.logging.debug import debug +from utils.parsing import parsen def getWhoSingle(name, query): - result = main.r.sscan("live.who."+name, 0, query, count=9999999) + result = main.r.sscan("live.who."+name, 0, query, count=-1) if result[1] == []: return None - return [i.decode() for i in result[1]] + return (i.decode() for i in result[1]) def getWho(query): result = {} @@ -24,14 +25,14 @@ def getChansSingle(name, nick): result = main.r.sinter(*nick) if len(result) == 0: return None - return [i.decode() for i in result] + return (i.decode() for i in result) def getChanList(name, nick): chanspace = "live.chan."+name+"."+nick result = main.r.smembers(chanspace) if len(result) == 0: return None - return [i.decode() for i in result] + return (i.decode() for i in result) def getChans(nick): result = {} @@ -42,11 +43,11 @@ def getChans(nick): return result def getUsersSingle(name, nick): - nick = ["live.who."+name+"."+i for i in nick] + nick = ("live.who."+name+"."+i for i in nick) result = main.r.sinter(*nick) if len(result) == 0: return None - return [i.decode() for i in result] + return (i.decode() for i in result) def getUsers(nick): result = {} @@ -69,13 +70,17 @@ def getNamespace(name, channel, nick): gnamespace = "live.who.%s" % name namespace = "live.who.%s.%s" % (name, channel) chanspace = "live.chan.%s.%s" % (name, nick) - return [gnamespace, namespace, chanspace] + mapspace = "live.map.%s" % name + return (gnamespace, namespace, chanspace, mapspace) def _initialUsers(name, channel, users): gnamespace = "live.who.%s" % name + mapspace = "live.map.%s" % name p = main.r.pipeline() for i in users: - p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2]) + user = i[0]+"!"+i[1]+"@"+i[2] + p.hset(mapspace, i[0], user) + p.sadd(gnamespace, user) p.execute() def initialUsers(name, channel, users): @@ -98,29 +103,36 @@ def initialNames(name, channel, names): def editUser(name, user): gnamespace = "live.who.%s" % name - main.r.sadd(gnamespace, user) + mapspace = "live.map.%s" % name + parsed = parsen(user) + p = main.r.pipeline() + p.sadd(gnamespace, user) + p.hset(mapspace, parsed[0], user) # add nick -> user mapping + p.execute() def addUser(name, channel, nick, user): - gnamespace, namespace, chanspace = getNamespace(name, channel, nick) + gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick) p = main.r.pipeline() p.sadd(gnamespace, user) p.sadd(namespace, nick) p.sadd(chanspace, channel) + p.hset(mapspace, nick, user) p.execute() def delUser(name, channel, nick, user): - gnamespace, namespace, chanspace = getNamespace(name, channel, nick) + gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick) p = main.r.pipeline() channels = main.r.smembers(chanspace) p.srem(namespace, nick) - if channels == {channel.encode()}: - p.delete(chanspace) + if channels == {channel.encode()}: # can we only see them on this channel? + p.delete(chanspace) # remove channel tracking entry + p.hdel(mapspace, nick) # remove nick mapping entry if user: - p.srem(gnamespace, user) + p.srem(gnamespace, user) # remove global userinfo entry else: warn("Attempt to delete nonexistent user: %s" % user) else: - p.srem(chanspace, channel) + p.srem(chanspace, channel) # keep up - remove the channel from their list p.execute() def escape(text): @@ -131,7 +143,14 @@ def escape(text): return text def getUserByNick(name, nick): - gnamespace = "live.who.%s" % name + gnamespace = "live.who.%s" % name # "nick": "nick!ident@host" + mapspace = "live.map.%s" % name + if main.r.hexists(mapspace, nick): + return main.r.hget(mapspace, nick) + else: + warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) + #return Falsedd + # legacy code below - remove when map is reliable usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1) if usermatch[1] == []: return False @@ -140,12 +159,13 @@ def getUserByNick(name, nick): user = usermatch[1][0] return user else: - warn("Entry doesn't exist: %s on %s" % (nick, gnamespace)) + warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace)) return False def renameUser(name, oldnick, olduser, newnick, newuser): gnamespace = "live.who.%s" % name chanspace = "live.chan.%s.%s" % (name, oldnick) + mapspace = "live.map.%s" % name newchanspace = "live.chan.%s.%s" % (name, newnick) p = main.r.pipeline() p.srem(gnamespace, olduser) @@ -154,51 +174,53 @@ def renameUser(name, oldnick, olduser, newnick, newuser): i = i.decode() p.srem("live.who."+name+"."+i, oldnick) p.sadd("live.who."+name+"."+i, newnick) + p.hdel(mapspace, oldnick) + p.hset(mapspace, newnick, newuser) if main.r.exists(chanspace): p.rename(chanspace, newchanspace) else: warn("Key doesn't exist: %s" % chanspace) p.execute() -def delUserByNick(name, channel, nick): +def delUserByNick(name, channel, nick): # kick user = getUserByNick(name, nick) delUser(name, channel, nick, user) -def delUserByNetwork(name, nick, user): +def delUserByNetwork(name, nick, user): # quit gnamespace = "live.who.%s" % name chanspace = "live.chan.%s.%s" % (name, nick) + mapspace = "live.chan.%s" % name p = main.r.pipeline() p.srem(gnamespace, user) for i in main.r.smembers(chanspace): p.srem("live.who."+name+"."+i.decode(), nick) p.delete(chanspace) + p.hdel(mapspace, nick) p.execute() -def _delChannel(name, channel): # This function is extremely expensive, look to replace - gnamespace = "live.who.%s" % name - namespace = "live.who.%s.%s" % (name, channel) +def _delChannels(net, channels): + gnamespace = "live.who.%s" % net + mapspace = "live.map.%s" % net p = main.r.pipeline() - for i in main.r.smembers(namespace): - user = getUserByNick(name, i.decode()) - if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}: - if user: - p.srem(gnamespace, user) - - p.delete("live.chan."+name+"."+i.decode()) - else: - p.srem("live.chan."+name+"."+i.decode(), channel) - p.delete(namespace) + for channel in channels: + namespace = "live.who.%s.%s" % (net, channel) + for i in main.r.smembers(namespace): + nick = i.decode() + #user = getUserByNick(net, nick) -- far too many function calls + user = main.r.hget(mapspace, nick) + if not user: + warn("User lookup failed: %s on %s" % (nick, net)) + if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}: + if user: + p.srem(gnamespace, user) + p.delete("live.chan."+net+"."+nick) + p.hdel(mapspace, nick) # remove map entry + else: + p.srem("live.chan."+net+"."+nick, channel) + p.delete(namespace) p.execute() - return [name, channel] -def delChannel(name, channel): - debug("Purging channel %s for %s" % (channel, name)) - d = deferToThread(_delChannel, name, channel) +def delChannels(net, channels): + debug("Purging channel %s for %s" % (", ".join(channels), net)) + d = deferToThread(_delChannels, net, channels) #d.addCallback(testCallback) - -def delNetwork(name, channels): - debug("Purging channels for %s" % name) - for i in channels: - delChannel(name, i) - #log("Finished purging channels for %s" % name) - return diff --git a/utils/parsing.py b/utils/parsing.py new file mode 100644 index 0000000..fb4f588 --- /dev/null +++ b/utils/parsing.py @@ -0,0 +1,15 @@ +def parsen(user): + step = user.split("!") + nick = step[0] + if len(step) == 2: + step2 = step[1].split("@") + if len(step2) == 2: + ident, host = step2 + else: + ident = nick + host = nick + else: + ident = nick + host = nick + + return (nick, ident, host) From b4fa747853360ed23ad3345df29290b12026be0a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 20 Oct 2019 16:44:33 +0100 Subject: [PATCH 173/394] Fix LIST handling and message parsing * Always use simple LIST syntax if it succeeded once after a failed complex query * Reject asking for a LIST twice * Quickly discard any ISUPPORT messages that don't contain things we need to use * Detect the server name and drop any messages from the server --- core/bot.py | 86 +++++++++++++++++++++++++++++++++------------ modules/chankeep.py | 2 -- modules/userinfo.py | 2 +- 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/core/bot.py b/core/bot.py index 73e18f8..e2df83e 100644 --- a/core/bot.py +++ b/core/bot.py @@ -106,22 +106,26 @@ class IRCBot(IRCClient): self.realname = alias["realname"] self.username = alias["nick"]+"/"+relay["net"] self.password = main.config["Relay"]["Password"] - self.userinfo = None - self.fingerReply = None - self.versionName = None - self.versionNum = None - self.versionEnv = None - self.sourceURL = None + self.userinfo = None # + self.fingerReply = None # + self.versionName = None # don't tell anyone information about us + self.versionNum = None # + self.versionEnv = None # + self.sourceURL = None # self._getWho = {} # LoopingCall objects -- needed to be able to stop them - self._tempWho = {} # Temporary storage for gathering WHO info - self._tempNames = {} # Temporary storage for gathering NAMES info - self._tempList = ([], []) # Temporary storage for gathering LIST info - self.listOngoing = False - self.listRetried = False + self._tempWho = {} # temporary storage for gathering WHO info + self._tempNames = {} # temporary storage for gathering NAMES info + self._tempList = ([], []) # temporary storage for gathering LIST info + self.listOngoing = False # we are currently receiving a LIST + self.listRetried = False # we asked and got nothing so asked again + self.listAttempted = False # we asked for a list + self.listSimple = False # after asking again we got the list, so + # use the simple syntax from now on self.chanlimit = 0 + self.servername = None def joinChannels(self, channels): sleeptime = 0.0 @@ -151,11 +155,13 @@ class IRCBot(IRCClient): if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] if "muser" in cast.keys(): + if cast["muser"] == self.servername: + return + if "channel" in cast.keys(): + if cast["channel"] == "*": + return + if not {"nick", "ident", "host"}.issubset(set(cast.keys())): cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) - # additional checks here to see if it's a server -- irc.example.net - # discard if it is - #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]: - # del cast["muser"] if set(["nick", "ident", "host", "msg"]).issubset(set(cast)): if "msg" in cast.keys(): if cast["ident"] == "znc" and cast["host"] == "znc.in": @@ -166,6 +172,7 @@ class IRCBot(IRCClient): del cast["host"] del cast["channel"] if "Disconnected from IRC" in cast["msg"]: + log("ZNC disconnected on %s - %i" (self.net, self.num)) self.connected = False if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here @@ -323,10 +330,16 @@ class IRCBot(IRCClient): newNicklist.append(f) userinfo.initialNames(self.net, nicklist[0], newNicklist) + def myInfo(self, servername, version, umodes, cmodes): + self.servername = servername + def _list(self, noargs): d = Deferred() self._tempList = ([], []) self._tempList[0].append(d) + if self.listSimple: + self.sendLine("LIST") + return d if noargs: self.sendLine("LIST") else: @@ -334,10 +347,19 @@ class IRCBot(IRCClient): return d def list(self, noargs=False): + if not self.listAttempted: + self.listAttempted = True + else: + debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num)) + return if not self.listOngoing: self._list(noargs).addCallback(self.got_list) + else: + debug("List request dropped, already ongoing - %s - %i" % (self.net, self.num)) + return def irc_RPL_LISTSTART(self, prefix, params): + self.listAttempted = False self.listOngoing = True def irc_RPL_LIST(self, prefix, params): @@ -359,12 +381,16 @@ class IRCBot(IRCClient): if noResults: if not self.listRetried: self.list(True) + self.listRetried = True else: warn("List still empty after retry: %s - %i" % (net, num)) self.listRetried = False return else: - self.listRetried = False + if self.listRetried: + self.listRetried = False + debug("List received after retry - defaulting to simple list syntax") + self.listSimple = True def got_list(self, listinfo): if len(listinfo) == 0: # probably ngircd not supporting LIST >0 @@ -372,19 +398,33 @@ class IRCBot(IRCClient): chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) def isupport(self, options): - for i in options: + interested = ("CHANLIMIT", "MAXCHANNELS") + if not any((x for x in options if any(y in x for y in interested))): + return # check if any of interested is in any of options, some networks + chanlimit = None # call isupport() more than once, so discard swiftly anything + for i in options: # we don't care about if i.startswith("CHANLIMIT"): if ":" in i: split = i.split(":") if len(split) >= 2: chanlimit = split[1] - try: - self.chanlimit = int(chanlimit) - return - except TypeError: - warn("Invalid CHANLIMIT: %s" % i) + break + elif i.startswith("MAXCHANNELS"): + if "=" in i: + split = i.split("=") + if len(split) == 2: + chanlimit = split[1] + break + if chanlimit: + try: + self.chanlimit = int(chanlimit) + except TypeError: + warn("Invalid chanlimit: %s" % i) if self.num == 1: # Only one instance should do a list, so - self.list() # why not this one? :P + if self.chanlimit: + self.list() # why not this one? :P + else: + debug("Aborting LIST due to bad chanlimit") self.checkChannels() #twisted sucks so i have to do this to actually get the user info diff --git a/modules/chankeep.py b/modules/chankeep.py index 43a3d32..d9dd94c 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -27,7 +27,6 @@ def getChanFree(net, new): name = net+str(i) chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels) chanlimits.add(main.IRCPool[name].chanlimit) - if not len(chanlimits) == 1: error("Network %s has servers with different CHANMAX values" % net) return False @@ -39,7 +38,6 @@ def emptyChanAllocate(net, flist, relay, new): return for i in new: chanfree[0][i] = chanfree[1] - print("chanfree", chanfree) allocated = {} toalloc = len(flist) if toalloc > sum(chanfree[0].values()): diff --git a/modules/userinfo.py b/modules/userinfo.py index c8c46cb..b903e80 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -7,7 +7,7 @@ from utils.logging.debug import debug from utils.parsing import parsen def getWhoSingle(name, query): - result = main.r.sscan("live.who."+name, 0, query, count=-1) + result = main.r.sscan("live.who."+name, 0, query, count=999999) if result[1] == []: return None return (i.decode() for i in result[1]) From 7ffb6125aa7256eddcf03fd88f8340073ffa1d4f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 31 Oct 2019 15:44:59 +0000 Subject: [PATCH 174/394] Fix various bugs uncovered by the LIST system * Work around Twisted's broken handling of spaces * Work around Twisted's broken line decoding * Don't run signedOn twice for relays * Improved detection of whether the endpoint is connected to ZNC * Delay a LIST until all configured relays are online * Discard a LIST if there are no callbacks for it * Get rid of some double-negative ternary blocks --- core/bot.py | 163 ++++++++++++++++++++++++++++++++------------ modules/chankeep.py | 7 +- 2 files changed, 123 insertions(+), 47 deletions(-) diff --git a/core/bot.py b/core/bot.py index e2df83e..1b37f7c 100644 --- a/core/bot.py +++ b/core/bot.py @@ -3,7 +3,9 @@ from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall from twisted.internet import reactor, task +from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage +import sys from string import digits from random import randint from copy import deepcopy @@ -36,9 +38,33 @@ def deliverRelayCommands(num, relayCommands, user=None, stage2=None): port, bot, contextFactory) +def parsemsg(s): + """ + Breaks a message from an IRC server into its prefix, command, and + arguments. + @param s: The message to break. + @type s: L{bytes} + @return: A tuple of (prefix, command, args). + @rtype: L{tuple} + """ + prefix = '' + trailing = [] + if not s: + raise IRCBadMessage("Empty line.") + if s[0:1] == ':': + prefix, s = s[1:].split(' ', 1) + if s.find(' :') != -1: + s, trailing = s.split(' :', 1) + args = s.split(' ') + args.append(trailing) + else: + args = s.split(' ') + command = args.pop(0) + return prefix, command, args + class IRCRelay(IRCClient): def __init__(self, num, relayCommands, user, stage2): - self.connected = False + self.isconnected = False self.buffer = "" if user == None: self.user = main.config["Relay"]["User"] @@ -78,23 +104,24 @@ class IRCRelay(IRCClient): deliverRelayCommands(self.num, commands, user, self.stage2) def signedOn(self): - self.connected = True - log("signed on as a relay: %s" % self.num) - #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares - sleeptime = 0 - increment = 0.8 - for i in self.relayCommands.keys(): - for x in self.relayCommands[i]: - reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) - sleeptime += increment - increment += 0.8 - reactor.callLater(sleeptime, self.sendStage2) - reactor.callLater(sleeptime+5, self.transport.loseConnection) - return + if not self.isconnected: + self.isconnected = True + log("signed on as a relay: %s" % self.num) + #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares + sleeptime = 0 + increment = 0.8 + for i in self.relayCommands.keys(): + for x in self.relayCommands[i]: + reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) + sleeptime += increment + increment += 0.8 + reactor.callLater(sleeptime, self.sendStage2) + reactor.callLater(sleeptime+5, self.transport.loseConnection) + return class IRCBot(IRCClient): def __init__(self, net, num): - self.connected = False + self.isconnected = False self.channels = [] self.net = net self.num = num @@ -121,12 +148,26 @@ class IRCBot(IRCClient): self.listOngoing = False # we are currently receiving a LIST self.listRetried = False # we asked and got nothing so asked again self.listAttempted = False # we asked for a list - self.listSimple = False # after asking again we got the list, so - # use the simple syntax from now on - + self.listSimple = False # after asking again we got the list, so use the simple + # syntax from now on + self.wantList = False # we want to send a LIST, but not all relays are active yet self.chanlimit = 0 self.servername = None + def lineReceived(self, line): + if bytes != str and isinstance(line, bytes): + # decode bytes from transport to unicode + line = line.decode("utf-8", "replace") + + line = lowDequote(line) + try: + prefix, command, params = parsemsg(line) + if command in numeric_to_symbolic: + command = numeric_to_symbolic[command] + self.handleCommand(command, prefix, params) + except IRCBadMessage: + self.badMessage(line, *sys.exc_info()) + def joinChannels(self, channels): sleeptime = 0.0 increment = 0.8 @@ -172,8 +213,11 @@ class IRCBot(IRCClient): del cast["host"] del cast["channel"] if "Disconnected from IRC" in cast["msg"]: - log("ZNC disconnected on %s - %i" (self.net, self.num)) - self.connected = False + log("ZNC disconnected on %s - %i" % (self.net, self.num)) + self.isconnected = False + if "Connected!" in cast["msg"]: + log("ZNC connected on %s - %i" % (self.net, self.num)) + self.isconnected = True if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here if cast["channel"].lower() == self.nickname.lower(): # as they are channel == nickname @@ -339,28 +383,38 @@ class IRCBot(IRCClient): self._tempList[0].append(d) if self.listSimple: self.sendLine("LIST") - return d + return d # return early if we know what to do + if noargs: self.sendLine("LIST") else: self.sendLine("LIST >0") return d - def list(self, noargs=False): - if not self.listAttempted: - self.listAttempted = True - else: + def list(self, noargs=False, nocheck=False): + if self.listAttempted: debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num)) return - if not self.listOngoing: - self._list(noargs).addCallback(self.got_list) else: - debug("List request dropped, already ongoing - %s - %i" % (self.net, self.num)) + self.listAttempted = True + if self.listOngoing: + debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)) return + else: + if nocheck: + allRelays = True # override the system - if this is + else: # specified, we already did this + allRelays = chankeep.allRelaysActive(self.net) + if not allRelays: + self.wantList = True + debug("Not all relays were active for LIST request") + return + self._list(noargs).addCallback(self.got_list) def irc_RPL_LISTSTART(self, prefix, params): self.listAttempted = False self.listOngoing = True + self.wantList = False def irc_RPL_LIST(self, prefix, params): channel = params[1] @@ -369,6 +423,11 @@ class IRCBot(IRCClient): self._tempList[1].append([channel, users, topic]) def irc_RPL_LISTEND(self, prefix, params): + if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there + debug("We didn't ask for this LIST, discarding") + self._tempList[0].clear() + self._tempList[1].clear() + return callbacks, info = self._tempList self.listOngoing = False for cb in callbacks: @@ -379,13 +438,13 @@ class IRCBot(IRCClient): self._tempList[0].clear() self._tempList[1].clear() if noResults: - if not self.listRetried: - self.list(True) - self.listRetried = True - else: - warn("List still empty after retry: %s - %i" % (net, num)) + if self.listRetried: + warn("LIST still empty after retry: %s - %i" % (net, num)) self.listRetried = False return + else: + self.list(True) + self.listRetried = True else: if self.listRetried: self.listRetried = False @@ -402,7 +461,10 @@ class IRCBot(IRCClient): if not any((x for x in options if any(y in x for y in interested))): return # check if any of interested is in any of options, some networks chanlimit = None # call isupport() more than once, so discard swiftly anything - for i in options: # we don't care about + if not self.isconnected: # we don't care about + log("endpoint connected: %s - %i" % (self.net, self.num)) + self.isconnected = True + for i in options: if i.startswith("CHANLIMIT"): if ":" in i: split = i.split(":") @@ -415,14 +477,28 @@ class IRCBot(IRCClient): if len(split) == 2: chanlimit = split[1] break - if chanlimit: - try: - self.chanlimit = int(chanlimit) - except TypeError: - warn("Invalid chanlimit: %s" % i) - if self.num == 1: # Only one instance should do a list, so + print("chanlimit", chanlimit) + try: + self.chanlimit = int(chanlimit) + except TypeError: + warn("Invalid chanlimit: %s" % i) + if self.chanlimit == 0: + self.chanlimit = 200 # don't take the piss if it's unlimited + allRelays = chankeep.allRelaysActive(self.net) + print(self.net, self.num, allRelays) + if allRelays: + for i in main.network.keys(): + for x in main.network[i].relays.keys(): + name = i+str(x) + if main.IRCPool[name].wantList == True: + main.IRCPool[name].list(nocheck=True) + debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) + if self.num == 1: # Only one instance should do a list if self.chanlimit: - self.list() # why not this one? :P + if allRelays: + self.list() + else: + self.wantList = True else: debug("Aborting LIST due to bad chanlimit") self.checkChannels() @@ -496,7 +572,6 @@ class IRCBot(IRCClient): #END hacks def signedOn(self): - self.connected = True log("signed on: %s - %i" % (self.net, self.num)) #self.event(type="conn", status="connected") sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"}) @@ -607,7 +682,7 @@ class IRCBotFactory(ReconnectingClientFactory): if not self.relay: userinfo.delChannels(self.net, self.client.channels) if not self.client == None: - self.client.connected = False + self.client.isconnected = False self.client.channels = [] error = reason.getErrorMessage() log("%s - %i: connection lost: %s" % (self.net, self.num, error)) @@ -619,7 +694,7 @@ class IRCBotFactory(ReconnectingClientFactory): def clientConnectionFailed(self, connector, reason): if not self.client == None: - self.client.connected = False + self.client.isconnected = False self.client.channels = [] error = reason.getErrorMessage() log("%s - %i: connection failed: %s" % (self.net, self.num, error)) diff --git a/modules/chankeep.py b/modules/chankeep.py index d9dd94c..6e174ca 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -12,7 +12,7 @@ def allRelaysActive(net): for i in main.network[net].relays.keys(): name = net+str(i) if name in main.IRCPool.keys(): - if main.IRCPool[name].connected: + if main.IRCPool[name].isconnected: existNum += 1 if existNum == relayNum: return True @@ -28,7 +28,8 @@ def getChanFree(net, new): chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels) chanlimits.add(main.IRCPool[name].chanlimit) if not len(chanlimits) == 1: - error("Network %s has servers with different CHANMAX values" % net) + error("Network %s has servers with different CHANLIMIT values" % net) + print(chanlimits) return False return (chanfree, chanlimits.pop()) @@ -72,7 +73,7 @@ def notifyJoin(net): def minifyChans(net, listinfo): if not allRelaysActive(net): - error("All relays for %s are not active, cannot minify list") + error("All relays for %s are not active, cannot minify list" % net) return False for i in main.network[net].relays.keys(): name = net+str(i) From a20fcdb8fc7b9b601c15103c300de04882ded06e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 31 Oct 2019 15:54:07 +0000 Subject: [PATCH 175/394] Remove some debugging code from ChanKeep --- modules/chankeep.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 6e174ca..83edcef 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -29,7 +29,6 @@ def getChanFree(net, new): chanlimits.add(main.IRCPool[name].chanlimit) if not len(chanlimits) == 1: error("Network %s has servers with different CHANLIMIT values" % net) - print(chanlimits) return False return (chanfree, chanlimits.pop()) @@ -87,9 +86,6 @@ def minifyChans(net, listinfo): return listinfo def keepChannels(net, listinfo, mean, sigrelay, relay): - #print("list", listinfo) - #print("sigrelay", sigrelay) - #print("cur", len(main.network[net].relays.keys())) listinfo = minifyChans(net, listinfo) if not listinfo: return @@ -112,17 +108,6 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): populateChans(net, siglist, sigrelay, newNums) notifyJoin(net) - #print("coverall", coverAll) - #print("needed", needed) - -#def purgeRecords(net): -# base = "list.%s" % net -# p = main.g.pipeline() -# existingChans = main.g.smembers("list."+net) -# for i in existingChans: -# p.delete(base+"."+i.decode("utf-8")) -# p.execute() - def nukeNetwork(net): #purgeRecords(net) #p = main.g.pipeline() @@ -135,7 +120,6 @@ def nukeNetwork(net): def _initialList(net, num, listinfo, chanlimit): - #listinfo = sorted(listinfo, key=lambda x: xdd[0]) listlength = len(listinfo) cumul = 0 try: From 9c4ea94ebdc885a6e288eb1536e0313f713e51be Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 17 Nov 2019 18:52:47 +0000 Subject: [PATCH 176/394] Add requirements file --- requirements.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..11a3e06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +asn1crypto==0.24.0 +attrs==19.1.0 +Automat==0.7.0 +cffi==1.12.3 +constantly==15.1.0 +cryptography==2.7 +csiphash==0.0.5 +hyperlink==19.0.0 +idna==2.8 +incremental==17.5.0 +numpy==1.17.2 +pyasn1==0.4.6 +pyasn1-modules==0.2.6 +pycparser==2.19 +PyHamcrest==1.9.0 +pyOpenSSL==19.0.0 +PyYAML==5.1.2 +redis==3.3.8 +service-identity==18.1.0 +six==1.12.0 +Twisted==19.7.0 +zope.interface==4.6.0 From e3e522ad1e49b99c013884e507b47b56c228cbb1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 17 Nov 2019 19:09:17 +0000 Subject: [PATCH 177/394] Add requirements --- requirements.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..11a3e06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +asn1crypto==0.24.0 +attrs==19.1.0 +Automat==0.7.0 +cffi==1.12.3 +constantly==15.1.0 +cryptography==2.7 +csiphash==0.0.5 +hyperlink==19.0.0 +idna==2.8 +incremental==17.5.0 +numpy==1.17.2 +pyasn1==0.4.6 +pyasn1-modules==0.2.6 +pycparser==2.19 +PyHamcrest==1.9.0 +pyOpenSSL==19.0.0 +PyYAML==5.1.2 +redis==3.3.8 +service-identity==18.1.0 +six==1.12.0 +Twisted==19.7.0 +zope.interface==4.6.0 From 06903d872e016c031da558955e1d431ff9cdbc04 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 7 Dec 2019 16:35:29 +0000 Subject: [PATCH 178/394] Add more comments and remove obsolete code --- commands/logout.py | 2 -- core/bot.py | 46 ++++++++++------------------------------------ core/server.py | 2 -- main.py | 23 ++++++++++++++++++++--- 4 files changed, 30 insertions(+), 43 deletions(-) diff --git a/commands/logout.py b/commands/logout.py index fca76d6..d1dea67 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -7,8 +7,6 @@ class LogoutCommand: def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: obj.authed = False - if obj.addr in main.MonitorPool: - main.MonitorPool.remove(obj.addr) success("Logged out") return else: diff --git a/core/bot.py b/core/bot.py index 1b37f7c..879a9a1 100644 --- a/core/bot.py +++ b/core/bot.py @@ -38,6 +38,7 @@ def deliverRelayCommands(num, relayCommands, user=None, stage2=None): port, bot, contextFactory) +# Copied from the Twisted source so we can fix a bug def parsemsg(s): """ Breaks a message from an IRC server into its prefix, command, and @@ -55,10 +56,10 @@ def parsemsg(s): prefix, s = s[1:].split(' ', 1) if s.find(' :') != -1: s, trailing = s.split(' :', 1) - args = s.split(' ') + args = s.split(' ') # Twisted bug fixed by adding an argument to split() args.append(trailing) else: - args = s.split(' ') + args = s.split(' ') # And again command = args.pop(0) return prefix, command, args @@ -107,7 +108,6 @@ class IRCRelay(IRCClient): if not self.isconnected: self.isconnected = True log("signed on as a relay: %s" % self.num) - #sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares sleeptime = 0 increment = 0.8 for i in self.relayCommands.keys(): @@ -135,7 +135,7 @@ class IRCBot(IRCClient): self.password = main.config["Relay"]["Password"] self.userinfo = None # self.fingerReply = None # - self.versionName = None # don't tell anyone information about us + self.versionName = None # Don't give out information self.versionNum = None # self.versionEnv = None # self.sourceURL = None # @@ -369,7 +369,7 @@ class IRCBot(IRCClient): newNicklist = [] for i in nicklist[1]: for x in i: - f = self.sanit(x) # need to store this as well, or potentially just do not remove it... + f = self.sanit(x) if f: newNicklist.append(f) userinfo.initialNames(self.net, nicklist[0], newNicklist) @@ -477,15 +477,13 @@ class IRCBot(IRCClient): if len(split) == 2: chanlimit = split[1] break - print("chanlimit", chanlimit) try: self.chanlimit = int(chanlimit) except TypeError: warn("Invalid chanlimit: %s" % i) if self.chanlimit == 0: - self.chanlimit = 200 # don't take the piss if it's unlimited + self.chanlimit = 200 # don't take the piss if it's not limited allRelays = chankeep.allRelaysActive(self.net) - print(self.net, self.num, allRelays) if allRelays: for i in main.network.keys(): for x in main.network[i].relays.keys(): @@ -503,11 +501,10 @@ class IRCBot(IRCClient): debug("Aborting LIST due to bad chanlimit") self.checkChannels() - #twisted sucks so i have to do this to actually get the user info + # We need to override these functions as Twisted discards + # the hostname and other useful information in the functions + # that these call by default def irc_JOIN(self, prefix, params): - """ - Called when a user joins a channel. - """ nick = prefix.split('!')[0] channel = params[-1] if nick == self.nickname: @@ -516,9 +513,6 @@ class IRCBot(IRCClient): self.userJoined(prefix, channel) def irc_PART(self, prefix, params): - """ - Called when a user leaves a channel. - """ nick = prefix.split('!')[0] channel = params[0] if len(params) >= 2: @@ -531,19 +525,10 @@ class IRCBot(IRCClient): self.userLeft(prefix, channel, message) def irc_QUIT(self, prefix, params): - """ - Called when a user has quit. - """ nick = prefix.split('!')[0] - #if nick == self.nickname: - #self.botQuit(prefix, params[0]) - #else: self.userQuit(prefix, params[0]) def irc_NICK(self, prefix, params): - """ - Called when a user changes their nickname. - """ nick = prefix.split('!', 1)[0] if nick == self.nickname: self.nickChanged(prefix, params[0]) @@ -551,10 +536,6 @@ class IRCBot(IRCClient): self.userRenamed(prefix, params[0]) def irc_KICK(self, prefix, params): - """ - Called when a user is kicked from a channel. - """ - #kicker = prefix.split('!')[0] channel = params[0] kicked = params[1] message = params[-1] @@ -562,18 +543,13 @@ class IRCBot(IRCClient): self.userKicked(kicked, channel, prefix, message) def irc_TOPIC(self, prefix, params): - """ - Someone in the channel set the topic. - """ - #user = prefix.split('!')[0] channel = params[0] newtopic = params[1] self.topicUpdated(prefix, channel, newtopic) - #END hacks + # End of Twisted hackery def signedOn(self): log("signed on: %s - %i" % (self.net, self.num)) - #self.event(type="conn", status="connected") sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"}) def joined(self, channel): @@ -581,8 +557,6 @@ class IRCBot(IRCClient): self.channels.append(channel) self.names(channel).addCallback(self.got_names) if main.config["Toggles"]["Who"]: - #self.who(channel).addCallback(self.got_who) - #self.who(channel) lc = LoopingCall(self.who, channel) self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] diff --git a/core/server.py b/core/server.py index 30ca24d..e3f125a 100644 --- a/core/server.py +++ b/core/server.py @@ -38,8 +38,6 @@ class Server(Protocol): del main.connections[self.addr] else: warn("Tried to remove a non-existant connection.") - if self.addr in main.MonitorPool: - main.MonitorPool.remove(self.addr) class ServerFactory(Factory): def buildProtocol(self, addr): diff --git a/main.py b/main.py index bba9753..6b29325 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,7 @@ from os import urandom from utils.logging.log import * +# List of errors ZNC can give us ZNCErrors = ["Error:", "Unable to load", "does not exist", @@ -28,17 +29,32 @@ filemap = { "network": ["network.dat", "network list", "pickle"] } +# Connections to the plain-text interface connections = {} +# Connections to the JSON interface relayConnections = {} + +# Mapping of network names to Protocol (IRCClient) instances IRCPool = {} + +# Mapping of network names to Reactor instances +# Needed for calling .disconnect() ReactorPool = {} + +# Mapping of network names to Factory instances +# Needed for calling .stopTrying() FactoryPool = {} + +# Temporary store for channels allocated after a LIST +# Will get purged as the instances fire up and pop() from +# their respective keys in here TempChan = {} -MonitorPool = [] - +# Mapping of command names to their functions CommandMap = {} +# Incremented by 1 for each event reaching modules.counters.event() +# and cloned into lastMinuteSample every minute runningSample = 0 lastMinuteSample = 0 @@ -46,6 +62,7 @@ lastMinuteSample = 0 hashKey = urandom(16) lastEvents = {} +# Get networks that are currently online and dedupliate def liveNets(): networks = set() for i in IRCPool.keys(): @@ -67,7 +84,7 @@ def loadConf(var): with open(configPath+filemap[var][0], "r") as f: globals()[var] = json.load(f) if var == "alias": - # This is a hack to convert all the keys into integers since JSON + # This is a workaround to convert all the keys into integers since JSON # turns them into strings... # Dammit Jason! global alias From 97a25334aafe87c919357102204e056e15e426af Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Dec 2019 17:50:38 +0000 Subject: [PATCH 179/394] Add IRC network definitions --- conf/irc.json | 8 ++++++++ main.py | 1 + 2 files changed, 9 insertions(+) create mode 100644 conf/irc.json diff --git a/conf/irc.json b/conf/irc.json new file mode 100644 index 0000000..c7c5bb9 --- /dev/null +++ b/conf/irc.json @@ -0,0 +1,8 @@ +{ + "_": { + "register": true, + "authentity": "NickServ", + "authcmd": "REGISTER {password} {email}", + "tokenregex": "" + } +} diff --git a/main.py b/main.py index 6b29325..681c755 100644 --- a/main.py +++ b/main.py @@ -24,6 +24,7 @@ filemap = { "tokens": ["tokens.json", "authentication tokens", "json"], "aliasdata": ["aliasdata.json", "data for alias generation", "json"], "alias": ["alias.json", "provisioned alias data", "json"], + "irc": ["irc.json", "IRC network definitions", "json"], # Binary (pickle) configs "network": ["network.dat", "network list", "pickle"] From f4e5d248d523e4d41f6ccd72b4c7a03954049a26 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 28 Dec 2019 17:51:03 +0000 Subject: [PATCH 180/394] Separate provisioning into user and auth info --- modules/provision.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/provision.py b/modules/provision.py index f15d4c6..59dda84 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -13,7 +13,19 @@ def provisionUserData(num, nick, altnick, ident, realname, unused): # last field commands["controlpanel"].append("Set Ident %s %s" % (nick, ident)) commands["controlpanel"].append("Set RealName %s %s" % (nick, realname)) deliverRelayCommands(num, commands) - return + +def provisionAuthenticationData(num, nick, network, security, auth, password): + commands = {} + if auth == "sasl": + commands["sasl"] = [] + commands["status"].append("LoadMod sasl") + commands["sasl"].append("Mechanism plain") + commands["sasl"].append("Set %s %s" % (nick, password)) + elif auth == "ns": + commands["nickserv"] = [] + commands["status"].append("LoadMod nickserv") + commands["nickserv"].append("Set %s" % password) + deliverRelayCommands(num, commands, user=nick+"/"+network) def provisionNetworkData(num, nick, network, host, port, security, auth, password): commands = {} @@ -26,6 +38,7 @@ def provisionNetworkData(num, nick, network, host, port, security, auth, passwor commands["controlpanel"].append("AddServer %s %s %s +%s" % (nick, network, host, port)) elif security == "plain": commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port)) + # Remove below and use provisionAuthenticationData() when registration implemented if auth == "sasl": stage2commands["sasl"] = [] stage2commands["status"].append("LoadMod sasl") @@ -35,6 +48,7 @@ def provisionNetworkData(num, nick, network, host, port, security, auth, passwor stage2commands["nickserv"] = [] stage2commands["status"].append("LoadMod nickserv") stage2commands["nickserv"].append("Set %s" % password) + # End if not main.config["ConnectOnCreate"]: stage2commands["status"].append("Disconnect") if main.config["Toggles"]["CycleChans"]: @@ -42,7 +56,6 @@ def provisionNetworkData(num, nick, network, host, port, security, auth, passwor stage2commands["status"].append("LoadMod chansaver") deliverRelayCommands(num, commands, stage2=[[nick+"/"+network, stage2commands]]) - return def provisionRelayForNetwork(num, alias, network): provisionNetworkData(num, alias, network, @@ -51,7 +64,6 @@ def provisionRelayForNetwork(num, alias, network): main.network[network].security, main.network[network].auth, main.network[network].aliases[num]["password"]) - return def provisionRelay(num, network): aliasObj = main.alias[num] @@ -60,7 +72,6 @@ def provisionRelay(num, network): reactor.callLater(5, provisionRelayForNetwork, num, alias, network) if main.config["ConnectOnCreate"]: reactor.callLater(10, main.network[network].start_bot, num) - return def provisionMultipleRelays(net, relaysNeeded): numsProvisioned = [] From 690bf93676eec8fd84136360d7473b49aa0a27c5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 21 Apr 2020 23:32:17 +0100 Subject: [PATCH 181/394] Fix variable scope in LIST error handling --- core/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index 879a9a1..2ba40ff 100644 --- a/core/bot.py +++ b/core/bot.py @@ -439,7 +439,7 @@ class IRCBot(IRCClient): self._tempList[1].clear() if noResults: if self.listRetried: - warn("LIST still empty after retry: %s - %i" % (net, num)) + warn("LIST still empty after retry: %s - %i" % (self.net, self.num)) self.listRetried = False return else: From 1ac1061348122c095886cebca9382c01a2c16672 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 30 May 2020 21:35:50 +0100 Subject: [PATCH 182/394] Add irc.json to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4e98897..2216062 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,6 @@ conf/monitor.json conf/tokens.json conf/network.dat conf/alias.json +conf/irc.json conf/dist.sh env/ From d99c3c394fe34ee4d7d1ca73d02c79b1a31a080d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 30 May 2020 21:37:22 +0100 Subject: [PATCH 183/394] Restructure provisioning into fewer functions --- modules/provision.py | 61 +++++++++++++++----------------------------- 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/modules/provision.py b/modules/provision.py index 59dda84..2e0fab5 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -3,19 +3,33 @@ from core.bot import deliverRelayCommands from utils.logging.log import * from twisted.internet import reactor -def provisionUserData(num, nick, altnick, ident, realname, unused): # last field is password, - # which we don't want to inherit here, but still want to use * expansion, so this is a bit of a hack +def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security, auth, password): commands = {} + stage2commands = {} + stage2commands["status"] = [] commands["controlpanel"] = [] commands["controlpanel"].append("AddUser %s %s" % (nick, main.config["Relay"]["Password"])) + commands["controlpanel"].append("AddNetwork %s %s" % (nick, network)) commands["controlpanel"].append("Set Nick %s %s" % (nick, nick)) commands["controlpanel"].append("Set Altnick %s %s" % (nick, altnick)) commands["controlpanel"].append("Set Ident %s %s" % (nick, ident)) commands["controlpanel"].append("Set RealName %s %s" % (nick, realname)) - deliverRelayCommands(num, commands) + if security == "ssl": + commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (nick, network)) # Don't judge me + commands["controlpanel"].append("AddServer %s %s %s +%s" % (nick, network, host, port)) + elif security == "plain": + commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port)) + if not main.config["ConnectOnCreate"]: + stage2commands["status"].append("Disconnect") + if main.config["Toggles"]["CycleChans"]: + stage2commands["status"].append("LoadMod disconkick") + stage2commands["status"].append("LoadMod chansaver") + deliverRelayCommands(num, commands, + stage2=[[nick+"/"+network, stage2commands]]) def provisionAuthenticationData(num, nick, network, security, auth, password): commands = {} + commands["status"] = [] if auth == "sasl": commands["sasl"] = [] commands["status"].append("LoadMod sasl") @@ -27,49 +41,16 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): commands["nickserv"].append("Set %s" % password) deliverRelayCommands(num, commands, user=nick+"/"+network) -def provisionNetworkData(num, nick, network, host, port, security, auth, password): - commands = {} - stage2commands = {} - stage2commands["status"] = [] - commands["controlpanel"] = [] - commands["controlpanel"].append("AddNetwork %s %s" % (nick, network)) - if security == "ssl": - commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (nick, network)) # Don't judge me - commands["controlpanel"].append("AddServer %s %s %s +%s" % (nick, network, host, port)) - elif security == "plain": - commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port)) - # Remove below and use provisionAuthenticationData() when registration implemented - if auth == "sasl": - stage2commands["sasl"] = [] - stage2commands["status"].append("LoadMod sasl") - stage2commands["sasl"].append("Mechanism plain") - stage2commands["sasl"].append("Set %s %s" % (nick, password)) - elif auth == "ns": - stage2commands["nickserv"] = [] - stage2commands["status"].append("LoadMod nickserv") - stage2commands["nickserv"].append("Set %s" % password) - # End - if not main.config["ConnectOnCreate"]: - stage2commands["status"].append("Disconnect") - if main.config["Toggles"]["CycleChans"]: - stage2commands["status"].append("LoadMod disconkick") - stage2commands["status"].append("LoadMod chansaver") - deliverRelayCommands(num, commands, - stage2=[[nick+"/"+network, stage2commands]]) -def provisionRelayForNetwork(num, alias, network): - provisionNetworkData(num, alias, network, +def provisionRelay(num, network): # provision user and network data + aliasObj = main.alias[num] + alias = aliasObj["nick"] + provisionUserNetworkData(num, *aliasObj.values(), network, main.network[network].host, main.network[network].port, main.network[network].security, main.network[network].auth, main.network[network].aliases[num]["password"]) - -def provisionRelay(num, network): - aliasObj = main.alias[num] - alias = aliasObj["nick"] - provisionUserData(num, *aliasObj.values()) - reactor.callLater(5, provisionRelayForNetwork, num, alias, network) if main.config["ConnectOnCreate"]: reactor.callLater(10, main.network[network].start_bot, num) From a3cdb35e0526edce7ffc1b1137544ee050da470b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 30 May 2020 21:40:10 +0100 Subject: [PATCH 184/394] Implement registration and confirmation of nicks --- commands/confirm.py | 27 +++++++++++++++++++++ commands/reg.py | 27 +++++++++++++++++++++ conf/example/config.json | 1 + conf/example/irc.json | 9 +++++++ conf/help.json | 6 +++-- core/bot.py | 27 ++++++++++++++------- modules/alias.py | 4 +--- modules/network.py | 11 ++++++++- modules/regproc.py | 52 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 commands/confirm.py create mode 100644 commands/reg.py create mode 100644 conf/example/irc.json create mode 100644 modules/regproc.py diff --git a/commands/confirm.py b/commands/confirm.py new file mode 100644 index 0000000..93af4d0 --- /dev/null +++ b/commands/confirm.py @@ -0,0 +1,27 @@ +import main +from modules import regproc + +class ConfirmCommand: + def __init__(self, *args): + self.confirm(*args) + + def confirm(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 4: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("No such relay on %s: %s" % (spl[2], spl[1])) + return + regproc.confirmAccount(spl[1], int(spl[2]), spl[3]) + success("Requested confirmation on %s - %s with token %s" % (spl[1], spl[2], spl[3])) + return + else: + incUsage("confirm") + return + else: + incUsage(None) diff --git a/commands/reg.py b/commands/reg.py new file mode 100644 index 0000000..406dda4 --- /dev/null +++ b/commands/reg.py @@ -0,0 +1,27 @@ +import main +from modules import regproc + +class RegCommand: + def __init__(self, *args): + self.reg(*args) + + def reg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 3: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return + if not int(spl[2]) in main.network[spl[1]].relays.keys(): + failure("No such relay on %s: %s" % (spl[2], spl[1])) + return + regproc.registerAccount(spl[1], int(spl[2])) + success("Requested registration on %s - %s" % (spl[1], spl[2])) + return + else: + incUsage("reg") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index 7026833..a3dfecc 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -15,6 +15,7 @@ "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, + "AutoReg": false, "Debug": false, "Relay": { "Host": "127.0.0.1", diff --git a/conf/example/irc.json b/conf/example/irc.json new file mode 100644 index 0000000..e624aa8 --- /dev/null +++ b/conf/example/irc.json @@ -0,0 +1,9 @@ +{ + "_": { + "register": true, + "entity": "NickServ", + "email": "{nickname}@example.com", + "register": "REGISTER {password} {email}", + "confirm": "CONFIRM {token}" + } +} diff --git a/conf/help.json b/conf/help.json index f97448f..1b44f76 100644 --- a/conf/help.json +++ b/conf/help.json @@ -13,7 +13,7 @@ "load": "load <(file)|list|all>", "dist": "dist", "loadmod": "loadmod ", - "msg": "msg ", + "msg": "msg ", "mon": "mon -h", "chans": "chans [ ...]", "users": "users [ ...]", @@ -28,5 +28,7 @@ "admall": "admall ", "swho": "swho []", "list": "list ", - "exec": "exec " + "exec": "exec ", + "reg": "reg ", + "confirm": "confirm " } diff --git a/core/bot.py b/core/bot.py index 2ba40ff..90b5baf 100644 --- a/core/bot.py +++ b/core/bot.py @@ -14,6 +14,7 @@ from modules import userinfo from modules import counters from modules import monitor from modules import chankeep +from modules import regproc from core.relay import sendRelayNotification from utils.dedup import dedup @@ -97,12 +98,12 @@ class IRCRelay(IRCClient): sendAll("%s: relay password mismatch" % self.num) def sendStage2(self): - if not self.stage2 == None: # [["user", {"sasl": ["message1", "message2"]}], []] - if not len(self.stage2) == 0: - user = self.stage2[0].pop(0) - commands = self.stage2[0].pop(0) - del self.stage2[0] - deliverRelayCommands(self.num, commands, user, self.stage2) + # [["user", {"sasl": ["message1", "message2"]}], []] + if not len(self.stage2) == 0: + user = self.stage2[0].pop(0) + commands = self.stage2[0].pop(0) + del self.stage2[0] + deliverRelayCommands(self.num, commands, user, self.stage2) def signedOn(self): if not self.isconnected: @@ -115,7 +116,8 @@ class IRCRelay(IRCClient): reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) sleeptime += increment increment += 0.8 - reactor.callLater(sleeptime, self.sendStage2) + if not self.stage2 == None: + reactor.callLater(sleeptime, self.sendStage2) reactor.callLater(sleeptime+5, self.transport.loseConnection) return @@ -392,6 +394,9 @@ class IRCBot(IRCClient): return d def list(self, noargs=False, nocheck=False): + if not main.network[self.net].relays[self.num]["registered"]: + debug("Will not send LIST, unregistered: %s - %i" % (self.net, self.num)) + return if self.listAttempted: debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num)) return @@ -448,7 +453,7 @@ class IRCBot(IRCClient): else: if self.listRetried: self.listRetried = False - debug("List received after retry - defaulting to simple list syntax") + debug("List received after retry - defaulting to simple list syntax: %s - %i" % (self.net, self.num)) self.listSimple = True def got_list(self, listinfo): @@ -464,6 +469,10 @@ class IRCBot(IRCClient): if not self.isconnected: # we don't care about log("endpoint connected: %s - %i" % (self.net, self.num)) self.isconnected = True + if not main.network[self.net].relays[self.num]["registered"]: + if main.config["AutoReg"]: + regproc.registerAccount(self.net, self.num) + debug("Attempting to register: %s - %i" % (self.net, self.num)) for i in options: if i.startswith("CHANLIMIT"): if ":" in i: @@ -673,7 +682,7 @@ class IRCBotFactory(ReconnectingClientFactory): error = reason.getErrorMessage() log("%s - %i: connection failed: %s" % (self.net, self.num, error)) if not self.relay: - sendAll("%s -%s: connection failed: %s" % (self.net, self.num, error)) + sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/modules/alias.py b/modules/alias.py index e05950d..4a3949f 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -63,6 +63,4 @@ def generate_alias(): if rand == 3 or rand == 4: realname = realname.capitalize() - password = generate_password() - - return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "password": password} + return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname} diff --git a/modules/network.py b/modules/network.py index 9e235a1..5df2577 100644 --- a/modules/network.py +++ b/modules/network.py @@ -27,10 +27,19 @@ class Network: self.last += 1 elif num == self.last: self.last += 1 + registered = False + if self.net in main.irc.keys(): + if "register" in main.irc[self.net].keys(): + if not main.irc[self.net]["register"]: + registered = True + # Don't need to register if it's been disabled in definitions, + # so we'll pretend we already did + self.relays[num] = { "enabled": main.config["ConnectOnCreate"], "net": self.net, - "id": num + "id": num, + "registered": registered } password = alias.generate_password() if not num in main.alias.keys(): diff --git a/modules/regproc.py b/modules/regproc.py new file mode 100644 index 0000000..f1296bf --- /dev/null +++ b/modules/regproc.py @@ -0,0 +1,52 @@ +import main +from modules import provision +from utils.logging.log import * + +def registerAccount(net, num): + alias = main.alias[num] + nickname = alias["nick"] + username = nickname+"/"+net + password = main.network[net].aliases[num]["password"] + if net in main.irc.keys(): + inst = main.irc[net] + else: + inst = main.irc["_"] + + if not inst["register"]: + error("Cannot register for %s: function disabled" % (net)) + return False + entity = inst["entity"] + email = inst["email"] + cmd = inst["register"] + email = email.replace("{nickname}", nickname) + cmd = cmd.replace("{password}", password) + cmd = cmd.replace("{email}", email) + name = net+str(num) + main.IRCPool[name].msg(entity, cmd) + +def confirmAccount(net, num, token): + if net in main.irc.keys(): + inst = main.irc[net] + else: + inst = main.irc["_"] + entity = inst["entity"] + cmd = inst["confirm"] + cmd = cmd.replace("{token}", token) + name = net+str(num) + main.IRCPool[name].msg(entity, cmd) + enableAuthentication(net, num) + +def enableAuthentication(net, num): + obj = main.network[net] + nick = main.alias[num]["nick"] + security = obj.security + auth = obj.auth + password = obj.aliases[num]["password"] + + uname = main.alias[num]["nick"]+"/"+net + provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth + if obj.relays[num]["registered"]: + warn("Authentication already enabled: %s - %i" % (net, num)) + obj.relays[num]["registered"] = True + main.saveConf("network") + main.IRCPool[net+str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"]+"status", "Jump") From aec683cccec14b334943f2f04d30e21a7ae4f3c1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 30 May 2020 21:42:26 +0100 Subject: [PATCH 185/394] Remove leftover irc.json file --- conf/irc.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 conf/irc.json diff --git a/conf/irc.json b/conf/irc.json deleted file mode 100644 index c7c5bb9..0000000 --- a/conf/irc.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "_": { - "register": true, - "authentity": "NickServ", - "authcmd": "REGISTER {password} {email}", - "tokenregex": "" - } -} From efb9666b6aaa464a31369a036b3ec332b133909c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 12:32:12 +0100 Subject: [PATCH 186/394] Add confirm command Confirm command to check which relays need manual confirmation. --- commands/pending.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 commands/pending.py diff --git a/commands/pending.py b/commands/pending.py new file mode 100644 index 0000000..a245a52 --- /dev/null +++ b/commands/pending.py @@ -0,0 +1,31 @@ +import main + +class PendingCommand: + def __init__(self, *args): + self.pending(*args) + + def pending(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 1: + results = [] + for i in main.network.keys(): + for x in main.network[i].relays.keys(): + if not main.network[i].relays[x]["registered"]: + results.append("%s: confirm %s %s [code]" % (main.alias[x]["nick"], i, x)) + info("\n".join(results)) + return + elif length == 2: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + results = [] + for x in main.network[spl[1]].relays.keys(): + if not main.network[spl[1]].relays[x]["registered"]: + results.append("%s: confirm %s %s [code]" % (main.alias[x]["nick"], spl[1], x)) + info("\n".join(results)) + return + else: + incUsage("pending") + return + else: + incUsage(None) From 4f9ca6088b34a71a5ee02f77e609441162e0dab9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 13:08:00 +0100 Subject: [PATCH 187/394] Allow sending LIST to all networks at once --- commands/list.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/commands/list.py b/commands/list.py index 44cd366..3839914 100644 --- a/commands/list.py +++ b/commands/list.py @@ -6,7 +6,18 @@ class ListCommand: def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 2: + if length == 1: + for i in main.network.keys(): + if not 1 in main.network[i].relays.keys(): + info("Network has no first instance: %s" % i) + continue + if not i+"1" in main.IRCPool.keys(): + info("No IRC instance: %s - 1" % i) + continue + main.IRCPool[i+"1"].list() + success("Requested list with first instance of %s" % i) + return + elif length == 2: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return From 5c6b6263960fbca42f62352d125dcea55509ba88 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 13:09:58 +0100 Subject: [PATCH 188/394] Check registration status before joining channels Do not join channels if any relay for a network is unregistered. --- core/bot.py | 5 ++++- modules/chankeep.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index 90b5baf..90bf653 100644 --- a/core/bot.py +++ b/core/bot.py @@ -183,9 +183,12 @@ class IRCBot(IRCClient): increment = 0.7 increment += 0.1 else: - error("%s - %i - Cannot join channel we are already on - %s" % (self.net, self.num, i)) + error("%s - Cannot join channel we are already on: %s - %i" % (i, self.net, self.num)) def checkChannels(self): + if not chankeep.allRelaysActive(self.net): + debug("Skipping channel check as we have inactive relays: %s - %i" % (self.net, self.num)) + return if self.net in main.TempChan.keys(): if self.num in main.TempChan[self.net].keys(): self.joinChannels(main.TempChan[self.net][self.num]) diff --git a/modules/chankeep.py b/modules/chankeep.py index 83edcef..9c9aac3 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -12,7 +12,7 @@ def allRelaysActive(net): for i in main.network[net].relays.keys(): name = net+str(i) if name in main.IRCPool.keys(): - if main.IRCPool[name].isconnected: + if main.IRCPool[name].isconnected and main.network[net].relays[i]["registered"]: existNum += 1 if existNum == relayNum: return True From 81b045090400fcb3a4490e89cc95cb3576e8cc28 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 13:23:09 +0100 Subject: [PATCH 189/394] Function to select and merge IRC network defs --- modules/regproc.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/regproc.py b/modules/regproc.py index f1296bf..0b006b1 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -1,17 +1,26 @@ import main from modules import provision from utils.logging.log import * +from copy import deepcopy + +def selectInst(net): + if net in main.irc.keys(): + inst = deepcopy(main.irc[net]) + for i in main.irc["_"].keys(): + if not i in inst: + inst[i] = main.irc["_"][i] + else: + inst = main.irc["_"] + print(inst) + print("hello") + return inst def registerAccount(net, num): alias = main.alias[num] nickname = alias["nick"] username = nickname+"/"+net password = main.network[net].aliases[num]["password"] - if net in main.irc.keys(): - inst = main.irc[net] - else: - inst = main.irc["_"] - + inst = selectInst(net) if not inst["register"]: error("Cannot register for %s: function disabled" % (net)) return False @@ -25,10 +34,7 @@ def registerAccount(net, num): main.IRCPool[name].msg(entity, cmd) def confirmAccount(net, num, token): - if net in main.irc.keys(): - inst = main.irc[net] - else: - inst = main.irc["_"] + inst = selectInst(net) entity = inst["entity"] cmd = inst["confirm"] cmd = cmd.replace("{token}", token) From 5ee53ace4cad768a07ccd39ce9a2207733483bdb Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 13:44:34 +0100 Subject: [PATCH 190/394] Add additional error handling in user queries --- modules/userinfo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index b903e80..94a52c4 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -149,10 +149,11 @@ def getUserByNick(name, nick): return main.r.hget(mapspace, nick) else: warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) - #return Falsedd + #return False # legacy code below - remove when map is reliable usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1) if usermatch[1] == []: + warn("No matches found for user query: %s on %s" % (nick, name)) return False else: if len(usermatch[1]) == 1: @@ -184,6 +185,8 @@ def renameUser(name, oldnick, olduser, newnick, newuser): def delUserByNick(name, channel, nick): # kick user = getUserByNick(name, nick) + if not user: + return delUser(name, channel, nick, user) def delUserByNetwork(name, nick, user): # quit From 586a337ea48dc0cd0d0d3f5d368daf105a59ffde Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 16:40:51 +0100 Subject: [PATCH 191/394] Add help for pending command --- conf/help.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/help.json b/conf/help.json index 1b44f76..ede4b7d 100644 --- a/conf/help.json +++ b/conf/help.json @@ -30,5 +30,6 @@ "list": "list ", "exec": "exec ", "reg": "reg ", - "confirm": "confirm " + "confirm": "confirm ", + "pending": "pending []" } From 097f100ec51f018a6290a0c4eaaa19fbdd70b2a4 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 21:52:56 +0100 Subject: [PATCH 192/394] Implement authentication detection * pending command to see which instances have never authenticated * authcheck command to see which instances are not currently authenticated --- commands/authcheck.py | 36 ++++++++++++++++++++++++++++++++++++ conf/example/irc.json | 8 ++++++-- conf/help.json | 3 ++- core/bot.py | 21 ++++++++++++++++----- modules/chankeep.py | 2 +- modules/monitor.py | 2 ++ modules/regproc.py | 40 +++++++++++++++++++++++++++++++++------- 7 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 commands/authcheck.py diff --git a/commands/authcheck.py b/commands/authcheck.py new file mode 100644 index 0000000..8ed5575 --- /dev/null +++ b/commands/authcheck.py @@ -0,0 +1,36 @@ +import main + +class AuthcheckCommand: + def __init__(self, *args): + self.authcheck(*args) + + def authcheck(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 1: + results = [] + for i in main.IRCPool.keys(): + num = main.IRCPool[i].num + net = main.IRCPool[i].net + if not main.IRCPool[i].authenticated: + results.append("%s - %s: %s" % (net, num, main.alias[num]["nick"])) + info("\n".join(results)) + return + elif length == 2: + if not spl[1] in main.IRCPool.keys(): + failure("No such instance: %s" % spl[1]) + return + results = [] + for i in main.IRCPool.keys(): + num = main.IRCPool[i].num + net = main.IRCPool[i].net + if not net == spl[1]: + continue + if not main.IRCPool[i].authenticated: + results.append("%s - %s: %s" % (net, num, main.alias[num]["nick"])) + info("\n".join(results)) + return + else: + incUsage("authcheck") + return + else: + incUsage(None) diff --git a/conf/example/irc.json b/conf/example/irc.json index e624aa8..680eb58 100644 --- a/conf/example/irc.json +++ b/conf/example/irc.json @@ -2,8 +2,12 @@ "_": { "register": true, "entity": "NickServ", - "email": "{nickname}@example.com", + "email": "{nickname}.irc@domain.com", "register": "REGISTER {password} {email}", - "confirm": "CONFIRM {token}" + "confirm": "CONFIRM {token}", + "check": true, + "checktype": "mode", + "checkmode": "r", + "checkmsg": "Password accepted - you are now recognized." } } diff --git a/conf/help.json b/conf/help.json index ede4b7d..3940fa4 100644 --- a/conf/help.json +++ b/conf/help.json @@ -31,5 +31,6 @@ "exec": "exec ", "reg": "reg ", "confirm": "confirm ", - "pending": "pending []" + "pending": "pending []", + "authcheck": "authcheck []" } diff --git a/core/bot.py b/core/bot.py index 90bf653..8a374f8 100644 --- a/core/bot.py +++ b/core/bot.py @@ -108,7 +108,7 @@ class IRCRelay(IRCClient): def signedOn(self): if not self.isconnected: self.isconnected = True - log("signed on as a relay: %s" % self.num) + #log("signed on as a relay: %s" % self.num) sleeptime = 0 increment = 0.8 for i in self.relayCommands.keys(): @@ -124,6 +124,7 @@ class IRCRelay(IRCClient): class IRCBot(IRCClient): def __init__(self, net, num): self.isconnected = False + self.authenticated = False self.channels = [] self.net = net self.num = num @@ -156,6 +157,8 @@ class IRCBot(IRCClient): self.chanlimit = 0 self.servername = None + self._regAttempt = None + def lineReceived(self, line): if bytes != str and isinstance(line, bytes): # decode bytes from transport to unicode @@ -208,7 +211,7 @@ class IRCBot(IRCClient): return if not {"nick", "ident", "host"}.issubset(set(cast.keys())): cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) - if set(["nick", "ident", "host", "msg"]).issubset(set(cast)): + if {"nick", "ident", "host", "msg"}.issubset(set(cast)): if "msg" in cast.keys(): if cast["ident"] == "znc" and cast["host"] == "znc.in": cast["type"] = "znc" @@ -235,6 +238,7 @@ class IRCBot(IRCClient): # channel, but we still want to see them if "user" in cast.keys(): if cast["user"].lower() == self.nickname.lower(): + cast["num"] = self.num castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" @@ -242,6 +246,7 @@ class IRCBot(IRCClient): self.event(**castDup) if "nick" in cast.keys(): if cast["nick"].lower() == self.nickname.lower(): + cast["num"] = self.num castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" @@ -251,6 +256,7 @@ class IRCBot(IRCClient): if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries if not cast["msg"] == None: if self.nickname.lower() in cast["msg"].lower(): + cast["num"] = self.num castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "highlight" @@ -259,6 +265,9 @@ class IRCBot(IRCClient): if not "net" in cast.keys(): cast["net"] = self.net + if not "num" in cast.keys(): + print("no num", cast) + cast["num"] = self.num counters.event(self.net, cast["type"]) monitor.event(self.net, cast) @@ -474,8 +483,8 @@ class IRCBot(IRCClient): self.isconnected = True if not main.network[self.net].relays[self.num]["registered"]: if main.config["AutoReg"]: - regproc.registerAccount(self.net, self.num) - debug("Attempting to register: %s - %i" % (self.net, self.num)) + self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) + #regproc.registerAccount(self.net, self.num) for i in options: if i.startswith("CHANLIMIT"): if ":" in i: @@ -669,10 +678,11 @@ class IRCBotFactory(ReconnectingClientFactory): userinfo.delChannels(self.net, self.client.channels) if not self.client == None: self.client.isconnected = False + self.client.authenticated = False self.client.channels = [] error = reason.getErrorMessage() - log("%s - %i: connection lost: %s" % (self.net, self.num, error)) if not self.relay: + log("%s - %i: connection lost: %s" % (self.net, self.num, error)) sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error}) self.retry(connector) @@ -681,6 +691,7 @@ class IRCBotFactory(ReconnectingClientFactory): def clientConnectionFailed(self, connector, reason): if not self.client == None: self.client.isconnected = False + self.client.authenticated = False self.client.channels = [] error = reason.getErrorMessage() log("%s - %i: connection failed: %s" % (self.net, self.num, error)) diff --git a/modules/chankeep.py b/modules/chankeep.py index 9c9aac3..05e7c61 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -12,7 +12,7 @@ def allRelaysActive(net): for i in main.network[net].relays.keys(): name = net+str(i) if name in main.IRCPool.keys(): - if main.IRCPool[name].isconnected and main.network[net].relays[i]["registered"]: + if main.IRCPool[name].authenticated and main.network[net].relays[i]["registered"]: existNum += 1 if existNum == relayNum: return True diff --git a/modules/monitor.py b/modules/monitor.py index d55c1c5..fcc386b 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -5,6 +5,7 @@ from datetime import datetime import main from core.relay import sendRelayNotification from modules import userinfo +from modules import regproc from utils.dedup import dedup order = ["type", "net", "num", "channel", "msg", "nick", @@ -63,6 +64,7 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if dedup(numName, c): return + regproc.registerTest(c) # metadata scraping # need to check if this was received from a relay # in which case, do not do this diff --git a/modules/regproc.py b/modules/regproc.py index 0b006b1..1784d31 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -1,6 +1,7 @@ import main from modules import provision from utils.logging.log import * +from utils.logging.debug import * from copy import deepcopy def selectInst(net): @@ -11,11 +12,10 @@ def selectInst(net): inst[i] = main.irc["_"][i] else: inst = main.irc["_"] - print(inst) - print("hello") return inst def registerAccount(net, num): + debug("Attempting to register: %s - %i" % (net, num)) alias = main.alias[num] nickname = alias["nick"] username = nickname+"/"+net @@ -42,17 +42,43 @@ def confirmAccount(net, num, token): main.IRCPool[name].msg(entity, cmd) enableAuthentication(net, num) +def confirmRegistration(net, num): + obj = main.network[net] + name = net+str(num) + if name in main.IRCPool.keys(): + debug("Relay authenticated: %s - %i" %(net, num)) + main.IRCPool[name].authenticated = True + if obj.relays[num]["registered"]: + return + if name in main.IRCPool.keys(): + if main.IRCPool[name]._regAttempt: + main.IRCPool[name]._regAttempt.cancel() + obj.relays[num]["registered"] = True + main.saveConf("network") + def enableAuthentication(net, num): obj = main.network[net] nick = main.alias[num]["nick"] security = obj.security auth = obj.auth password = obj.aliases[num]["password"] - uname = main.alias[num]["nick"]+"/"+net provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth - if obj.relays[num]["registered"]: - warn("Authentication already enabled: %s - %i" % (net, num)) - obj.relays[num]["registered"] = True - main.saveConf("network") main.IRCPool[net+str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"]+"status", "Jump") + if selectInst(net)["check"] == False: + confirmRegistration(net, num) + +def registerTest(c): + inst = selectInst(c["net"]) + if inst["check"] == False: + return + if inst["checktype"] == "msg": + if c["type"] == "query": + if inst["checkmsg"] in c["msg"] and c["nick"] == inst["entity"]: + confirmRegistration(c["net"], c["num"]) + return + elif inst["checktype"] == "mode": + if c["type"] == "mode": + if inst["checkmode"] in c["modes"] and c["status"] == True: + confirmRegistration(c["net"], c["num"]) + return From 290e0b5f876cb05da7bec48bc602c04c296e947c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 31 May 2020 21:54:43 +0100 Subject: [PATCH 193/394] Fix syntax error in redis query --- modules/userinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index 94a52c4..706089e 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -151,7 +151,7 @@ def getUserByNick(name, nick): warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) #return False # legacy code below - remove when map is reliable - usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=-1) + usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=999999999) if usermatch[1] == []: warn("No matches found for user query: %s on %s" % (nick, name)) return False From 1640955e5cf6ad1f59a0a3d0edcd3cd4d2df11a9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 2 Jun 2020 21:34:15 +0100 Subject: [PATCH 194/394] Fix various bugs in the event system Squash many bugs in the event notification system and simplify the code. --- core/bot.py | 103 +++++++++++++++++++++++++++------------------ core/relay.py | 5 +-- modules/monitor.py | 6 +-- utils/dedup.py | 6 ++- 4 files changed, 71 insertions(+), 49 deletions(-) diff --git a/core/bot.py b/core/bot.py index 8a374f8..2fd7bfc 100644 --- a/core/bot.py +++ b/core/bot.py @@ -9,6 +9,7 @@ import sys from string import digits from random import randint from copy import deepcopy +from datetime import datetime from modules import userinfo from modules import counters @@ -200,73 +201,95 @@ class IRCBot(IRCClient): del main.TempChan[self.net] def event(self, **cast): + if not "time" in cast.keys(): + cast["time"] = str(datetime.now().isoformat()) + + # remove odd stuff for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] + # remove server stuff if "muser" in cast.keys(): if cast["muser"] == self.servername: return if "channel" in cast.keys(): if cast["channel"] == "*": return + ## + + # expand out the hostmask if not {"nick", "ident", "host"}.issubset(set(cast.keys())): cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) + + # handle ZNC stuff if {"nick", "ident", "host", "msg"}.issubset(set(cast)): - if "msg" in cast.keys(): - if cast["ident"] == "znc" and cast["host"] == "znc.in": - cast["type"] = "znc" - cast["num"] = self.num - del cast["nick"] - del cast["ident"] - del cast["host"] - del cast["channel"] - if "Disconnected from IRC" in cast["msg"]: - log("ZNC disconnected on %s - %i" % (self.net, self.num)) - self.isconnected = False - if "Connected!" in cast["msg"]: - log("ZNC connected on %s - %i" % (self.net, self.num)) - self.isconnected = True - if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: - if "channel" in cast.keys() and not cast["type"] == "mode": # don't handle modes here - if cast["channel"].lower() == self.nickname.lower(): # as they are channel == nickname - #castDup = deepcopy(cast) # however modes are not queries! - cast["mtype"] = cast["type"] - cast["type"] = "query" - cast["num"] = self.num - #self.event(**castDup) - # Don't call self.event for this one because queries are not events on a - # channel, but we still want to see them + if cast["ident"] == "znc" and cast["host"] == "znc.in": + cast["type"] = "znc" + cast["num"] = self.num + del cast["nick"] + del cast["ident"] + del cast["host"] + del cast["channel"] + if "Disconnected from IRC" in cast["msg"]: + log("ZNC disconnected on %s - %i" % (self.net, self.num)) + self.isconnected = False + if "Connected!" in cast["msg"]: + log("ZNC connected on %s - %i" % (self.net, self.num)) + self.isconnected = True + # + + # don't reprocess the same message twice + # if the type is in that list, it's already been here, don't run it again + if not cast["type"] in {"query", "self", "highlight", "znc", "who"}: + cast["num"] = self.num + if "channel" in cast.keys(): + if cast["type"] == "mode": + if self.nickname.lower() == cast["channel"].lower(): + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "self" + self.event(**castDup) + if cast["modearg"]: + if self.nickname.lower() == cast["modearg"].lower(): + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "self" + self.event(**castDup) + else: + if cast["channel"].lower() == self.nickname.lower(): + cast["mtype"] = cast["type"] + cast["type"] = "query" + #self.event(**castDup) + # Don't call self.event for this one because queries are not events on a + # channel, but we still want to see them + + # we have been kicked if "user" in cast.keys(): if cast["user"].lower() == self.nickname.lower(): - cast["num"] = self.num castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" - cast["num"] = self.num self.event(**castDup) + + # we sent a message/left/joined/kick someone/quit if "nick" in cast.keys(): if cast["nick"].lower() == self.nickname.lower(): - cast["num"] = self.num castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" - cast["num"] = self.num - if not cast["channel"].lower() == self.nickname.lower(): # modes has been set on us directly - self.event(**castDup) # don't tell anyone else - if "msg" in cast.keys() and not cast["type"] == "query": # Don't highlight queries + + # we have been mentioned in a msg/notice/action/part/quit/topic message + if "msg" in cast.keys(): # Don't highlight queries if not cast["msg"] == None: if self.nickname.lower() in cast["msg"].lower(): - cast["num"] = self.num castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "highlight" - cast["num"] = self.num self.event(**castDup) if not "net" in cast.keys(): cast["net"] = self.net if not "num" in cast.keys(): - print("no num", cast) cast["num"] = self.num counters.event(self.net, cast["type"]) monitor.event(self.net, cast) @@ -596,22 +619,22 @@ class IRCBot(IRCClient): #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so def left(self, user, channel, message): # even if they saw it, they wouldn't react - self.event(type="part", muser=user, channel=channel, message=message) + self.event(type="part", muser=user, channel=channel, msg=message) self.botLeft(channel) def userJoined(self, user, channel): self.event(type="join", muser=user, channel=channel) def userLeft(self, user, channel, message): - self.event(type="part", muser=user, channel=channel, message=message) + self.event(type="part", muser=user, channel=channel, msg=message) def userQuit(self, user, quitMessage): - self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage}) + self.chanlessEvent({"type": "quit", "muser": user, "msg": quitMessage}) def userKicked(self, kickee, channel, kicker, message): if kickee.lower() == self.nickname.lower(): self.botLeft(channel) - self.event(type="kick", muser=kicker, channel=channel, message=message, user=kickee) + self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee) def chanlessEvent(self, cast): cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) @@ -635,13 +658,13 @@ class IRCBot(IRCClient): self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname}) def topicUpdated(self, user, channel, newTopic): - self.event(type="topic", muser=user, channel=channel, message= newTopic) + self.event(type="topic", muser=user, channel=channel, msg=newTopic) def modeChanged(self, user, channel, toset, modes, args): argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", muser=user, channel=channel, modes=m, status=toset, modeargs=a) + self.event(type="mode", muser=user, channel=channel, mode=m, status=toset, modearg=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): diff --git a/core/relay.py b/core/relay.py index ef20155..a3bb8f9 100644 --- a/core/relay.py +++ b/core/relay.py @@ -1,7 +1,6 @@ from twisted.internet.protocol import Protocol, Factory, ClientFactory from json import dumps, loads from copy import deepcopy -from datetime import datetime import main from utils.logging.log import * @@ -136,6 +135,4 @@ def sendRelayNotification(cast): for i in main.relayConnections.keys(): if main.relayConnections[i].authed: if cast["type"] in main.relayConnections[i].subscriptions: - newCast = deepcopy(cast) - newCast["time"] = str(datetime.now().isoformat()) - main.relayConnections[i].send(dumps(newCast)) + main.relayConnections[i].send(dumps(cast)) diff --git a/modules/monitor.py b/modules/monitor.py index fcc386b..99c7e6f 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,6 +1,5 @@ from copy import deepcopy from json import dumps -from datetime import datetime import main from core.relay import sendRelayNotification @@ -9,8 +8,8 @@ from modules import regproc from utils.dedup import dedup order = ["type", "net", "num", "channel", "msg", "nick", - "ident", "host", "mtype", "user", "modes", "modeargs" - "realname", "server", "status"] + "ident", "host", "mtype", "user", "mode", "modearg", + "realname", "server", "status", "time"] def testNetTarget(name, target): called = False @@ -88,6 +87,7 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if "muser" in c.keys(): del c["muser"] + sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order # only monitors below diff --git a/utils/dedup.py b/utils/dedup.py index 03255e3..3545e68 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -1,11 +1,13 @@ from datetime import datetime from csiphash import siphash24 +from copy import deepcopy from json import dumps import main from utils.logging.debug import debug -def dedup(numName, c): - # deduplication +def dedup(numName, b): + c = deepcopy(b) + del c["time"] c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] From 2a9869d0f975b6fbe91c40891eb3537592cc2862 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 7 Jun 2020 15:31:43 +0100 Subject: [PATCH 195/394] Remove condition-based monitoring system --- .gitignore | 1 - commands/mon.py | 280 ------------------------------------ conf/example/masterbuf.json | 1 - conf/example/monitor.json | 1 - core/relay.py | 2 +- main.py | 1 - modules/monitor.py | 66 --------- 7 files changed, 1 insertion(+), 351 deletions(-) delete mode 100644 commands/mon.py delete mode 100644 conf/example/masterbuf.json delete mode 100644 conf/example/monitor.json diff --git a/.gitignore b/.gitignore index 2216062..9d56c45 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ __pycache__/ conf/config.json conf/wholist.json conf/counters.json -conf/monitor.json conf/tokens.json conf/network.dat conf/alias.json diff --git a/commands/mon.py b/commands/mon.py deleted file mode 100644 index 65e83d1..0000000 --- a/commands/mon.py +++ /dev/null @@ -1,280 +0,0 @@ -import main -import argparse -import sys -from io import StringIO -from yaml import dump - -class MonCommand: - def __init__(self, *args): - self.mon(*args) - - def setup_arguments(self, ArgumentParser): - self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.") - group1 = self.parser.add_mutually_exclusive_group(required=True) - group1.add_argument("-a", "--add", metavar="entry", dest="addEntry", help="Add an entry") - group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry") - group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries") - group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry") - - group2 = self.parser.add_mutually_exclusive_group() - group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing") - group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing") - - self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values") - - self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who") - self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching") - self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching") - self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching") - self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching") - self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who") - - self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)") - self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action") - self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick") - self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode") - self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode") - self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who") - self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who") - self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)") - - def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): - # We need to override the ArgumentParser class because - # it's only meant for CLI applications and exits - # after running, so we override the exit function - # to do nothing. We also need to do it here in order - # to catch the error messages and show them to the user. - # It sucks. I know. - if authed: - class ArgumentParser(argparse.ArgumentParser): - def exit(self, status=0, message=None): - if message: - failure(message) - self.setup_arguments(ArgumentParser) - if length == 1: - info("Incorrect usage, use mon -h for help") - return - old_stdout = sys.stdout - old_stderr = sys.stderr - my_stdout = sys.stdout = StringIO() - my_stderr = sys.stderr = StringIO() - try: - parsed = self.parser.parse_args(spl[1:]) - except: - return - sys.stdout = old_stdout - sys.stderr = old_stderr - stdout = my_stdout.getvalue() - stderr = my_stdout.getvalue() - if not stdout == "": - info(stdout) - elif not stderr == "": - failure(my_stdout.getvalue()) - return - my_stdout.close() - my_stderr.close() - - if parsed.addEntry: - if parsed.addEntry in main.monitor.keys(): - failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry) - return - cast = self.makeCast(parsed, failure, info) - if cast == False: - return - main.monitor[parsed.addEntry] = cast - main.saveConf("monitor") - success("Successfully created monitor group %s" % parsed.addEntry) - return - elif parsed.delEntry: - if not parsed.delEntry in main.monitor.keys(): - failure("No such monitor group: %s" % parsed.delEntry) - return - del main.monitor[parsed.delEntry] - main.saveConf("monitor") - success("Successfully removed monitor group: %s" % parsed.delEntry) - return - elif parsed.modEntry: - if not parsed.doAppend and not parsed.doRemove: - failure("Specify --append or --remove with --mod") - return - if not parsed.modEntry in main.monitor.keys(): - failure("No such monitor group: %s" % parsed.modEntry) - return - cast = self.makeCast(parsed, failure, info) - if cast == False: - return - if parsed.doAppend: - merged = self.addCast(main.monitor[parsed.modEntry], cast, info) - main.monitor[parsed.modEntry] = merged - elif parsed.doRemove: - merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info) - if merged == {}: - del main.monitor[parsed.modEntry] - info("Group %s deleted due to having no attributes" % parsed.modEntry) - main.saveConf("monitor") - return - main.monitor[parsed.modEntry] = merged - main.saveConf("monitor") - success("Successfully updated entry %s" % parsed.modEntry) - return - elif parsed.listEntry: - info(dump(main.monitor)) - return - - else: - incUsage(None) - - def dedup(self, data): - if not isinstance(data, bool): - return list(set(data)) - return data - - def parseNetworkFormat(self, lst, failure, info): - cast = {} - if lst == None: - return "nil" - if len(lst) == 0: - failure("List has no entries") - return False - elif len(lst) > 0: - if len(lst[0]) == 0: - failure("Nested list has no entries") - return False - for i in lst: - if not i[0] in main.pool.keys(): - failure("Name does not exist: %s" % i[0]) - return False - if i[0] in main.IRCPool.keys(): - for x in i[1:]: - if not x in main.IRCPool[i[0]].channels: - info("%s: Bot not on channel: %s" % (i[0], x)) - if len(i) == 1: - cast[i[0]] = True - else: - if i[0] in cast.keys(): - if not cast[i[0]] == True: - for x in self.dedup(i[1:]): - cast[i[0]].append(x) - else: - cast[i[0]] = self.dedup(i[1:]) - else: - cast[i[0]] = self.dedup(i[1:]) - for i in cast.keys(): - deduped = self.dedup(cast[i]) - cast[i] = deduped - return cast - - # Create or modify a monitor group magically - def makeCast(self, obj, failure, info): - validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"] - cast = {} - if not obj.specType == None: - types = self.dedup(obj.specType) - for i in types: - if not i in validTypes: - failure("Invalid type: %s" % i) - info("Available types: %s" % ", ".join(validTypes)) - return False - cast["type"] = types - if not obj.source == None: - sourceParse = self.parseNetworkFormat(obj.source, failure, info) - if not sourceParse: - return False - if not sourceParse == "nil": - cast["sources"] = sourceParse - - if not obj.send == None: - sendParse = self.parseNetworkFormat(obj.send, failure, info) - if not sendParse: - return False - if not sendParse == "nil": - cast["send"] = sendParse - - if not obj.message == None: - cast["message"] = [] - for i in obj.message: - cast["message"].append(" ".join(i)) - - if not obj.user == None: - cast["user"] = obj.user - if not obj.modes == None: - cast["modes"] = obj.modes - if not obj.modeargs == None: - cast["modeargs"] = obj.modeargs - if not obj.server == None: - cast["server"] = obj.server - if not obj.status == None: - cast["status"] = obj.status - if not obj.free == None: - cast["free"] = obj.free - if not obj.nick == None: - cast["nick"] = obj.nick - if not obj.ident == None: - cast["ident"] = obj.ident - if not obj.host == None: - cast["host"] = obj.host - if not obj.real == None: - cast["real"] = [] - for i in obj.real: - cast["real"].append(" ".join(i)) - if not obj.inside == None: - if obj.inside.lower() in ["yes", "true", "on"]: - cast["inside"] = True - elif obj.inside.lower() in ["no", "false", "off"]: - cast["inside"] = False - else: - failure("inside: Unknown operand: %s" % obj.inside) - return - return cast - - def subtractCast(self, source, patch, info): - for i in patch.keys(): - if i in source.keys(): - if isinstance(source[i], dict): - result = self.subtractCast(source[i], patch[i], info) - if result == {}: - info("Removing upper element: %s" % i) - del source[i] - continue - source[i] = result - continue - if isinstance(patch[i], bool): - del source[i] - info("Removing entire element: %s" % i) - continue - for x in patch[i]: - if isinstance(source[i], bool): - info("Attempt to remove %s from network-wide definition" % x) - continue - if x in source[i]: - source[i].remove(x) - if source[i] == []: - del source[i] - else: - info("Element %s not in source %s" % (x, i)) - else: - info("Non-matched key: %s" % i) - return source - - def addCast(self, source, patch, info): - for i in patch.keys(): - if i in source.keys(): - if isinstance(source[i], dict): - result = self.addCast(source[i], patch[i], info) - source[i] = result - continue - if isinstance(patch[i], bool): - source[i] = patch[i] - info("Overriding local element %s with network-wide definition" % i) - continue - for x in patch[i]: - if isinstance(source[i], bool): - source[i] = [] - info("Overriding element %s, previously set network-wide" % i) - if x in source[i]: - info("Element %s already in source %s" % (x, i)) - else: - source[i].append(x) - else: - source[i] = patch[i] - return source diff --git a/conf/example/masterbuf.json b/conf/example/masterbuf.json deleted file mode 100644 index fe51488..0000000 --- a/conf/example/masterbuf.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/conf/example/monitor.json b/conf/example/monitor.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/monitor.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/core/relay.py b/core/relay.py index a3bb8f9..7c7e020 100644 --- a/core/relay.py +++ b/core/relay.py @@ -5,7 +5,7 @@ from copy import deepcopy import main from utils.logging.log import * -validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"] +validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight"] class Relay(Protocol): def __init__(self, addr): diff --git a/main.py b/main.py index 681c755..e5ec77d 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,6 @@ filemap = { "config": ["config.json", "configuration", "json"], "help": ["help.json", "command help", "json"], "counters": ["counters.json", "counters file", "json"], - "monitor": ["monitor.json", "monitoring database", "json"], "tokens": ["tokens.json", "authentication tokens", "json"], "aliasdata": ["aliasdata.json", "data for alias generation", "json"], "alias": ["alias.json", "provisioned alias data", "json"], diff --git a/modules/monitor.py b/modules/monitor.py index 99c7e6f..76ff3ea 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -11,52 +11,6 @@ order = ["type", "net", "num", "channel", "msg", "nick", "ident", "host", "mtype", "user", "mode", "modearg", "realname", "server", "status", "time"] -def testNetTarget(name, target): - called = False - for i in main.monitor.keys(): - if "sources" in main.monitor[i].keys(): - if name in main.monitor[i]["sources"]: - if main.monitor[i]["sources"][name] == True: - yield i - called = True - elif target in main.monitor[i]["sources"][name]: - yield i - called = True - else: - yield i - called = True - if not called: - return False - -def contained_in(x, y): - if x is None or y is None: - return False - elif x == y: - return True - else: - return y in x - -def is_in(k, vs, A): - return any(contained_in(A.get(k), vp) for vp in vs) - -def matches(A, B): - return all(is_in(k, vs, A) for (k, vs) in B.items()) - -def magicFunction(A, B): - isInside = False - if "send" in B.keys(): - del B["send"] - if "sources" in B.keys(): - del B["sources"] - if "inside" in B.keys(): - if B["inside"] == True: - isInside = True - del B["inside"] - if isInside: - return matches(A, B) - else: - return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) - def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen if not "channel" in c.keys(): c["channel"] = None @@ -89,23 +43,3 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe del c["muser"] sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order - - # only monitors below - monitorGroups = testNetTarget(c["net"], c["channel"]) - if monitorGroups == False: - return - for monitorGroup in monitorGroups: - matcher = magicFunction(deepcopy(c), deepcopy(main.monitor[monitorGroup])) - if matcher == True: - c["monitor"] = True - if "send" in main.monitor[monitorGroup].keys(): - for i in main.monitor[monitorGroup]["send"].keys(): - if isinstance(main.monitor[monitorGroup]["send"][i], bool): - sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "errdeliv"}) - continue - if not i in main.pool.keys(): - sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noname"}) - if not i in main.IRCPool.keys(): - sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noinstance"}) - for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["net"], c)) From 3acf182171b0526b81298e81f8e83ed480b49157 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 7 Jun 2020 17:26:53 +0100 Subject: [PATCH 196/394] Fixes to auth detection and message parsing * don't check authentication if the network doesn't need to register * don't pass through muser for ZNC type messages * avoid duplicate message for queries containing highlights * make a copy of the cast for metadata analysis to avoid poisoning it * set up callback for when the instance is authenticated, so we can request a LIST immediately if so desired * separate out seeding functions to populate CHANLIMIT to ease future work involving other options, such as PREFIX --- core/bot.py | 155 +++++++++++++++++++++++++------------------- modules/chankeep.py | 2 +- modules/monitor.py | 14 ++-- modules/network.py | 12 ++-- modules/regproc.py | 18 +++-- 5 files changed, 116 insertions(+), 85 deletions(-) diff --git a/core/bot.py b/core/bot.py index 2fd7bfc..b2f1cb0 100644 --- a/core/bot.py +++ b/core/bot.py @@ -125,9 +125,9 @@ class IRCRelay(IRCClient): class IRCBot(IRCClient): def __init__(self, net, num): self.isconnected = False - self.authenticated = False self.channels = [] self.net = net + self.authenticated = not regproc.needToRegister(self.net) self.num = num self.buffer = "" self.name = net + str(num) @@ -230,6 +230,7 @@ class IRCBot(IRCClient): del cast["ident"] del cast["host"] del cast["channel"] + del cast["muser"] if "Disconnected from IRC" in cast["msg"]: log("ZNC disconnected on %s - %i" % (self.net, self.num)) self.isconnected = False @@ -244,16 +245,16 @@ class IRCBot(IRCClient): cast["num"] = self.num if "channel" in cast.keys(): if cast["type"] == "mode": - if self.nickname.lower() == cast["channel"].lower(): - castDup = deepcopy(cast) - castDup["mtype"] = cast["type"] - castDup["type"] = "self" - self.event(**castDup) - if cast["modearg"]: + if cast["channel"].lower() == self.nickname.lower(): + #castDup = deepcopy(cast) + cast["mtype"] = cast["type"] + cast["type"] = "self" + #self.event(**castDup) + if cast["modearg"]: # check if modearg is non-NoneType if self.nickname.lower() == cast["modearg"].lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] - castDup["type"] = "self" + castDup["type"] = "highlight" self.event(**castDup) else: if cast["channel"].lower() == self.nickname.lower(): @@ -263,34 +264,39 @@ class IRCBot(IRCClient): # Don't call self.event for this one because queries are not events on a # channel, but we still want to see them - # we have been kicked - if "user" in cast.keys(): - if cast["user"].lower() == self.nickname.lower(): - castDup = deepcopy(cast) - castDup["mtype"] = cast["type"] - castDup["type"] = "self" - self.event(**castDup) - - # we sent a message/left/joined/kick someone/quit - if "nick" in cast.keys(): - if cast["nick"].lower() == self.nickname.lower(): - castDup = deepcopy(cast) - castDup["mtype"] = cast["type"] - castDup["type"] = "self" - - # we have been mentioned in a msg/notice/action/part/quit/topic message - if "msg" in cast.keys(): # Don't highlight queries - if not cast["msg"] == None: - if self.nickname.lower() in cast["msg"].lower(): + # TODO: better way to do this + # as we changed the types above, check again + if not cast["type"] in {"query", "self", "highlight", "znc", "who"}: + # we have been kicked + if "user" in cast.keys(): + if cast["user"].lower() == self.nickname.lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] - castDup["type"] = "highlight" + castDup["type"] = "self" self.event(**castDup) + # we sent a message/left/joined/kick someone/quit + if "nick" in cast.keys(): + if cast["nick"].lower() == self.nickname.lower(): + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "self" + + # we have been mentioned in a msg/notice/action/part/quit/topic message + if "msg" in cast.keys(): # Don't highlight queries + if not cast["msg"] == None: + if self.nickname.lower() in cast["msg"].lower(): + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "highlight" + self.event(**castDup) + if not "net" in cast.keys(): cast["net"] = self.net if not "num" in cast.keys(): cast["num"] = self.num + if not self.authenticated: + regproc.registerTest(cast) counters.event(self.net, cast["type"]) monitor.event(self.net, cast) @@ -429,8 +435,8 @@ class IRCBot(IRCClient): return d def list(self, noargs=False, nocheck=False): - if not main.network[self.net].relays[self.num]["registered"]: - debug("Will not send LIST, unregistered: %s - %i" % (self.net, self.num)) + if not self.authenticated: + debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num)) return if self.listAttempted: debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num)) @@ -496,45 +502,15 @@ class IRCBot(IRCClient): return chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) - def isupport(self, options): - interested = ("CHANLIMIT", "MAXCHANNELS") - if not any((x for x in options if any(y in x for y in interested))): - return # check if any of interested is in any of options, some networks - chanlimit = None # call isupport() more than once, so discard swiftly anything - if not self.isconnected: # we don't care about - log("endpoint connected: %s - %i" % (self.net, self.num)) - self.isconnected = True - if not main.network[self.net].relays[self.num]["registered"]: - if main.config["AutoReg"]: - self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) - #regproc.registerAccount(self.net, self.num) - for i in options: - if i.startswith("CHANLIMIT"): - if ":" in i: - split = i.split(":") - if len(split) >= 2: - chanlimit = split[1] - break - elif i.startswith("MAXCHANNELS"): - if "=" in i: - split = i.split("=") - if len(split) == 2: - chanlimit = split[1] - break - try: - self.chanlimit = int(chanlimit) - except TypeError: - warn("Invalid chanlimit: %s" % i) - if self.chanlimit == 0: - self.chanlimit = 200 # don't take the piss if it's not limited + def recheckList(self): + print("list being rechecked") allRelays = chankeep.allRelaysActive(self.net) if allRelays: - for i in main.network.keys(): - for x in main.network[i].relays.keys(): - name = i+str(x) - if main.IRCPool[name].wantList == True: - main.IRCPool[name].list(nocheck=True) - debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) + print("allrelays now passed") + name = self.net+"1" + if main.IRCPool[name].wantList == True: + main.IRCPool[name].list(nocheck=True) + debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) if self.num == 1: # Only one instance should do a list if self.chanlimit: if allRelays: @@ -545,6 +521,51 @@ class IRCBot(IRCClient): debug("Aborting LIST due to bad chanlimit") self.checkChannels() + def seed_chanlimit(self, chanlimit): + if not main.network[self.net].relays[self.num]["registered"]: #TODO: add check for register request sent, only send it once + if main.config["AutoReg"]: + self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) + #regproc.registerAccount(self.net, self.num) + try: + self.chanlimit = int(chanlimit) + except TypeError: + warn("Invalid chanlimit: %s" % i) + if self.chanlimit == 0: + self.chanlimit = 200 # don't take the piss if it's not limited + if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation + self.recheckList() + + def seed_prefix(self, prefix): + print("PREFIX", prefix) + + def isupport(self, options): + interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"} + newOptions = {x for x in options if any(y in x for y in interested)} + if len(newOptions) == 0: + return + if not self.isconnected: + log("endpoint connected: %s - %i" % (self.net, self.num)) + self.isconnected = True + for i in newOptions: + if i.startswith("PREFIX"): + if "=" in i: + split = i.split("=") + if len(split) == 2: + prefix = split[1] + self.seed_prefix(prefix) + elif i.startswith("CHANLIMIT"): + if ":" in i: + split = i.split(":") + if len(split) >= 2: + chanlimit = split[1] + self.seed_chanlimit(chanlimit) + elif i.startswith("MAXCHANNELS"): + if "=" in i: + split = i.split("=") + if len(split) == 2: + chanlimit = split[1] + self.seed_chanlimit(chanlimit) + # We need to override these functions as Twisted discards # the hostname and other useful information in the functions # that these call by default diff --git a/modules/chankeep.py b/modules/chankeep.py index 05e7c61..dd2a19c 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -12,7 +12,7 @@ def allRelaysActive(net): for i in main.network[net].relays.keys(): name = net+str(i) if name in main.IRCPool.keys(): - if main.IRCPool[name].authenticated and main.network[net].relays[i]["registered"]: + if main.IRCPool[name].authenticated: existNum += 1 if existNum == relayNum: return True diff --git a/modules/monitor.py b/modules/monitor.py index 76ff3ea..5de655f 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -11,13 +11,9 @@ order = ["type", "net", "num", "channel", "msg", "nick", "ident", "host", "mtype", "user", "mode", "modearg", "realname", "server", "status", "time"] -def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen +def parsemeta(numName, c): if not "channel" in c.keys(): c["channel"] = None - - if dedup(numName, c): - return - regproc.registerTest(c) # metadata scraping # need to check if this was received from a relay # in which case, do not do this @@ -39,7 +35,13 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if c["mtype"] == "nick": userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) +def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen + if dedup(numName, c): + return + + # make a copy of the object with dict() to prevent sending notifications with channel of None + parsemeta(numName, dict(c)) + if "muser" in c.keys(): del c["muser"] - sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order diff --git a/modules/network.py b/modules/network.py index 5df2577..8cff3db 100644 --- a/modules/network.py +++ b/modules/network.py @@ -3,6 +3,7 @@ import json from modules import alias from modules.chankeep import nukeNetwork +from modules.regproc import needToRegister from twisted.internet import reactor from core.bot import IRCBot, IRCBotFactory import main @@ -28,13 +29,10 @@ class Network: elif num == self.last: self.last += 1 registered = False - if self.net in main.irc.keys(): - if "register" in main.irc[self.net].keys(): - if not main.irc[self.net]["register"]: - registered = True - # Don't need to register if it's been disabled in definitions, - # so we'll pretend we already did - + if not needToRegister(self.net): + registered = True + # Don't need to register if it's been disabled in definitions, + # so we'll pretend we already did self.relays[num] = { "enabled": main.config["ConnectOnCreate"], "net": self.net, diff --git a/modules/regproc.py b/modules/regproc.py index 1784d31..b9c3aa0 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -4,6 +4,14 @@ from utils.logging.log import * from utils.logging.debug import * from copy import deepcopy +def needToRegister(net): + inst = selectInst(net) + if "register" in inst.keys(): + if inst["register"]: + return True + else: + return False + def selectInst(net): if net in main.irc.keys(): inst = deepcopy(main.irc[net]) @@ -48,6 +56,7 @@ def confirmRegistration(net, num): if name in main.IRCPool.keys(): debug("Relay authenticated: %s - %i" %(net, num)) main.IRCPool[name].authenticated = True + main.IRCPool[name].recheckList() if obj.relays[num]["registered"]: return if name in main.IRCPool.keys(): @@ -78,7 +87,8 @@ def registerTest(c): confirmRegistration(c["net"], c["num"]) return elif inst["checktype"] == "mode": - if c["type"] == "mode": - if inst["checkmode"] in c["modes"] and c["status"] == True: - confirmRegistration(c["net"], c["num"]) - return + if c["type"] == "self": + if c["mtype"] == "mode": + if inst["checkmode"] in c["mode"] and c["status"] == True: + confirmRegistration(c["net"], c["num"]) + return From 82c5c2d16322581a68cfba21c253bfeb28618e1d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 9 Jul 2020 19:43:47 +0100 Subject: [PATCH 197/394] Start implementing prefixes --- core/bot.py | 25 ++++++++++++++----------- modules/monitor.py | 2 ++ modules/userinfo.py | 19 +++++++++++++++---- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/core/bot.py b/core/bot.py index b2f1cb0..53dac25 100644 --- a/core/bot.py +++ b/core/bot.py @@ -156,6 +156,7 @@ class IRCBot(IRCClient): # syntax from now on self.wantList = False # we want to send a LIST, but not all relays are active yet self.chanlimit = 0 + self.prefix = {} self.servername = None self._regAttempt = None @@ -376,12 +377,11 @@ class IRCBot(IRCClient): def sanit(self, data): if len(data) >= 1: - if data[0] in ["!", "~", "&", "@", "%", "+"]: - data = data[1:] - return data - return data + if data[0] in self.prefix.keys(): + return (self.prefix[data[0]], data[1:]) # would use a set but it's possible these are the same + return (None, data) else: - return False + return (None, False) def names(self, channel): d = Deferred() @@ -412,9 +412,9 @@ class IRCBot(IRCClient): newNicklist = [] for i in nicklist[1]: for x in i: - f = self.sanit(x) - if f: - newNicklist.append(f) + mode, nick = self.sanit(x) + if nick: + newNicklist.append((mode, nick)) userinfo.initialNames(self.net, nicklist[0], newNicklist) def myInfo(self, servername, version, umodes, cmodes): @@ -503,10 +503,8 @@ class IRCBot(IRCClient): chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) def recheckList(self): - print("list being rechecked") allRelays = chankeep.allRelaysActive(self.net) if allRelays: - print("allrelays now passed") name = self.net+"1" if main.IRCPool[name].wantList == True: main.IRCPool[name].list(nocheck=True) @@ -536,7 +534,12 @@ class IRCBot(IRCClient): self.recheckList() def seed_prefix(self, prefix): - print("PREFIX", prefix) + prefix = prefix.replace(")", "") + prefix = prefix.replace("(", "") + length = len(prefix) + half = int(length/2) + prefixToMode = dict(zip(prefix[half:], prefix[:half])) + self.prefix = prefixToMode def isupport(self, options): interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"} diff --git a/modules/monitor.py b/modules/monitor.py index 5de655f..e3a0590 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -19,6 +19,8 @@ def parsemeta(numName, c): # in which case, do not do this if c["type"] in ["msg", "notice", "action", "topic", "mode"]: userinfo.editUser(c["net"], c["muser"]) + if c["type"] == "mode": + userinfo.updateMode(c) elif c["type"] == "nick": userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) elif c["type"] == "kick": diff --git a/modules/userinfo.py b/modules/userinfo.py index 706089e..11234ad 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -91,9 +91,11 @@ def initialUsers(name, channel, users): def _initialNames(name, channel, names): namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() - for i in names: - p.sadd(namespace, i) - p.sadd("live.chan."+name+"."+i, channel) + for mode, nick in names: + p.sadd(namespace, nick) + p.sadd("live.chan."+name+"."+nick, channel) + if mode: + p.hset("live.prefix."+name+"."+channel, nick, mode) p.execute() def initialNames(name, channel, names): @@ -126,6 +128,7 @@ def delUser(name, channel, nick, user): p.srem(namespace, nick) if channels == {channel.encode()}: # can we only see them on this channel? p.delete(chanspace) # remove channel tracking entry + p.hdel("live.prefix."+name+"."+channel, nick) # remove prefix tracking entry p.hdel(mapspace, nick) # remove nick mapping entry if user: p.srem(gnamespace, user) # remove global userinfo entry @@ -177,6 +180,10 @@ def renameUser(name, oldnick, olduser, newnick, newuser): p.sadd("live.who."+name+"."+i, newnick) p.hdel(mapspace, oldnick) p.hset(mapspace, newnick, newuser) + if main.r.exists("live.prefix."+name+"."+i): # if there's a prefix entry for the channel + if main.r.hexists("live.prefix."+name+"."+i, oldnick): # if the old nick is in it + mode = main.r.hget("live.prefix."+name+"."+i, oldnick) # retrieve old modes + p.hset("live.prefix."+name+"."+i, newnick, mode) # set old modes to new nickname if main.r.exists(chanspace): p.rename(chanspace, newchanspace) else: @@ -197,6 +204,8 @@ def delUserByNetwork(name, nick, user): # quit p.srem(gnamespace, user) for i in main.r.smembers(chanspace): p.srem("live.who."+name+"."+i.decode(), nick) + p.hdel("live.prefix."+name+"."+i, nick) + p.delete(chanspace) p.hdel(mapspace, nick) p.execute() @@ -216,14 +225,16 @@ def _delChannels(net, channels): if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}: if user: p.srem(gnamespace, user) + p.delete("live.chan."+net+"."+nick) p.hdel(mapspace, nick) # remove map entry else: p.srem("live.chan."+net+"."+nick, channel) p.delete(namespace) + p.delete("live.prefix."+net+"."+channel) p.execute() -def delChannels(net, channels): +def delChannels(net, channels): # we have left a channel debug("Purging channel %s for %s" % (", ".join(channels), net)) d = deferToThread(_delChannels, net, channels) #d.addCallback(testCallback) From 913009ab717d94d23063b1c137adc960e5a0e529 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 28 Oct 2020 18:38:27 +0000 Subject: [PATCH 198/394] Fix circular import in ChanKeep/provisioning modules --- modules/chankeep.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index dd2a19c..e4f3f00 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -3,7 +3,7 @@ from utils.logging.log import * from utils.logging.debug import * from copy import deepcopy from math import ceil -from modules.provision import provisionMultipleRelays +import modules.provision from twisted.internet.threads import deferToThread def allRelaysActive(net): @@ -98,12 +98,12 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): return if coverAll: needed = relay-len(main.network[net].relays.keys()) - newNums = provisionMultipleRelays(net, needed) + newNums = modules.provision.provisionMultipleRelays(net, needed) flist = [i[0] for i in listinfo] populateChans(net, flist, relay, newNums) else: needed = sigrelay-len(main.network[net].relays.keys()) - newNums = provisionMultipleRelays(net, needed) + newNums = modules.provision.provisionMultipleRelays(net, needed) siglist = [i[0] for i in listinfo if int(i[1]) > mean] populateChans(net, siglist, sigrelay, newNums) notifyJoin(net) From 7d9a45ee91645e7f000330aa6d214f6b8d578fea Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 28 Oct 2020 22:26:41 +0000 Subject: [PATCH 199/394] Add the time field to some notifications --- core/bot.py | 20 ++++++++++++++------ modules/monitor.py | 4 ++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/core/bot.py b/core/bot.py index 53dac25..04299a2 100644 --- a/core/bot.py +++ b/core/bot.py @@ -135,7 +135,7 @@ class IRCBot(IRCClient): relay = main.network[self.net].relays[num] self.nickname = alias["nick"] self.realname = alias["realname"] - self.username = alias["nick"]+"/"+relay["net"] + self.username = alias["nick"].lower()+"/"+relay["net"] self.password = main.config["Relay"]["Password"] self.userinfo = None # self.fingerReply = None # @@ -335,8 +335,8 @@ class IRCBot(IRCClient): self.setNick(self._attemptedNick) def irc_ERR_PASSWDMISMATCH(self, prefix, params): - log("%s - %i: password mismatch" % (self.net, self.num)) - sendAll("%s - %i: password mismatch" % (self.net, self.num)) + log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username)) + sendAll("%s - %i: password mismatch as %s" % (self.net, self.num, self.username)) def _who(self, channel): d = Deferred() @@ -618,7 +618,12 @@ class IRCBot(IRCClient): def signedOn(self): log("signed on: %s - %i" % (self.net, self.num)) - sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"}) + ctime = str(datetime.now().isoformat()) + sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon", "time": ctime}) + if not self.authenticated: + inst = regproc.selectInst(self.net) + if inst["ping"] and inst["check"]: + self.msg(inst["entity"], inst["pingmsg"]) def joined(self, channel): if not channel in self.channels: @@ -661,6 +666,7 @@ class IRCBot(IRCClient): self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee) def chanlessEvent(self, cast): + cast["time"] = str(datetime.now().isoformat()) cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) if dedup(self.name, cast): # Needs to be kept self.name until the dedup # function is converted to the new net, num @@ -731,7 +737,8 @@ class IRCBotFactory(ReconnectingClientFactory): if not self.relay: log("%s - %i: connection lost: %s" % (self.net, self.num, error)) sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) - sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error}) + ctime = str(datetime.now().isoformat()) + sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error, "time": ctime}) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -744,7 +751,8 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s - %i: connection failed: %s" % (self.net, self.num, error)) if not self.relay: sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) - sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error}) + ctime = str(datetime.now().isoformat()) + sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error, "time": ctime}) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/modules/monitor.py b/modules/monitor.py index e3a0590..4b3d581 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -19,8 +19,8 @@ def parsemeta(numName, c): # in which case, do not do this if c["type"] in ["msg", "notice", "action", "topic", "mode"]: userinfo.editUser(c["net"], c["muser"]) - if c["type"] == "mode": - userinfo.updateMode(c) + #if c["type"] == "mode": + # userinfo.updateMode(c) elif c["type"] == "nick": userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) elif c["type"] == "kick": From 6acb1067610ba5972e23cad7db4971a68b1950e7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 28 Oct 2020 22:30:04 +0000 Subject: [PATCH 200/394] Provision users with lowercase names --- modules/provision.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/modules/provision.py b/modules/provision.py index 2e0fab5..b00b592 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -8,28 +8,30 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, stage2commands = {} stage2commands["status"] = [] commands["controlpanel"] = [] - commands["controlpanel"].append("AddUser %s %s" % (nick, main.config["Relay"]["Password"])) - commands["controlpanel"].append("AddNetwork %s %s" % (nick, network)) - commands["controlpanel"].append("Set Nick %s %s" % (nick, nick)) - commands["controlpanel"].append("Set Altnick %s %s" % (nick, altnick)) - commands["controlpanel"].append("Set Ident %s %s" % (nick, ident)) - commands["controlpanel"].append("Set RealName %s %s" % (nick, realname)) + user = nick.lower() + commands["controlpanel"].append("AddUser %s %s" % (user, main.config["Relay"]["Password"])) + commands["controlpanel"].append("AddNetwork %s %s" % (user, network)) + commands["controlpanel"].append("Set Nick %s %s" % (user, nick)) + commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick)) + commands["controlpanel"].append("Set Ident %s %s" % (user, ident)) + commands["controlpanel"].append("Set RealName %s %s" % (user, realname)) if security == "ssl": - commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (nick, network)) # Don't judge me - commands["controlpanel"].append("AddServer %s %s %s +%s" % (nick, network, host, port)) + commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me + commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port)) elif security == "plain": - commands["controlpanel"].append("AddServer %s %s %s %s" % (nick, network, host, port)) + commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port)) if not main.config["ConnectOnCreate"]: stage2commands["status"].append("Disconnect") if main.config["Toggles"]["CycleChans"]: stage2commands["status"].append("LoadMod disconkick") stage2commands["status"].append("LoadMod chansaver") deliverRelayCommands(num, commands, - stage2=[[nick+"/"+network, stage2commands]]) + stage2=[[user+"/"+network, stage2commands]]) def provisionAuthenticationData(num, nick, network, security, auth, password): commands = {} commands["status"] = [] + user = nick.lower() if auth == "sasl": commands["sasl"] = [] commands["status"].append("LoadMod sasl") @@ -39,7 +41,7 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): commands["nickserv"] = [] commands["status"].append("LoadMod nickserv") commands["nickserv"].append("Set %s" % password) - deliverRelayCommands(num, commands, user=nick+"/"+network) + deliverRelayCommands(num, commands, user=user+"/"+network) def provisionRelay(num, network): # provision user and network data From b16b5d690b2924fdcc7392d426c3046cb3bbee4f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 28 Oct 2020 22:30:49 +0000 Subject: [PATCH 201/394] Fix decoding issue with some Redis keys --- modules/userinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index 11234ad..fef0faa 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -204,7 +204,7 @@ def delUserByNetwork(name, nick, user): # quit p.srem(gnamespace, user) for i in main.r.smembers(chanspace): p.srem("live.who."+name+"."+i.decode(), nick) - p.hdel("live.prefix."+name+"."+i, nick) + p.hdel("live.prefix."+name+"."+i.decode(), nick) p.delete(chanspace) p.hdel(mapspace, nick) From 812db95995828e7cce97fece430802e50a306cba Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 28 Oct 2020 22:46:22 +0000 Subject: [PATCH 202/394] Add checks in dedup for time-less messages --- utils/dedup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/dedup.py b/utils/dedup.py index 3545e68..a5c3c45 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -7,7 +7,8 @@ from utils.logging.debug import debug def dedup(numName, b): c = deepcopy(b) - del c["time"] + if "time" in c.keys(): + del c["time"] c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] From 69fbe180f179b59f25636a828dfca5570f0ccfd9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 28 Oct 2020 22:48:27 +0000 Subject: [PATCH 203/394] Implement authentication checking on connection --- conf/example/irc.json | 7 +++++-- modules/regproc.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/conf/example/irc.json b/conf/example/irc.json index 680eb58..d2f6bb7 100644 --- a/conf/example/irc.json +++ b/conf/example/irc.json @@ -2,12 +2,15 @@ "_": { "register": true, "entity": "NickServ", - "email": "{nickname}.irc@domain.com", + "email": "{nickname}@domain.com", "register": "REGISTER {password} {email}", "confirm": "CONFIRM {token}", "check": true, + "ping": true, + "pingmsg": "STATUS {nickname}", "checktype": "mode", "checkmode": "r", - "checkmsg": "Password accepted - you are now recognized." + "checkmsg": "Password accepted - you are now recognized.", + "checkmsg2": "You are logged in as" } } diff --git a/modules/regproc.py b/modules/regproc.py index b9c3aa0..4ad63a4 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -81,9 +81,14 @@ def registerTest(c): inst = selectInst(c["net"]) if inst["check"] == False: return - if inst["checktype"] == "msg": - if c["type"] == "query": - if inst["checkmsg"] in c["msg"] and c["nick"] == inst["entity"]: + if "msg" in c.keys(): + if inst["checktype"] == "msg": + if c["type"] == "query": + if inst["checkmsg"] in c["msg"] and c["nick"] == inst["entity"]: + confirmRegistration(c["net"], c["num"]) + return + if inst["ping"]: + if inst["checkmsg2"] in c["msg"] and c["nick"] == inst["entity"]: confirmRegistration(c["net"], c["num"]) return elif inst["checktype"] == "mode": From 4d2550562532c568d1eaa980a72e9c3f42b81744 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 00:06:35 +0000 Subject: [PATCH 204/394] Note that arguments to list are optional --- conf/help.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/help.json b/conf/help.json index 3940fa4..d016a67 100644 --- a/conf/help.json +++ b/conf/help.json @@ -7,7 +7,6 @@ "part": "part ", "enable": "enable ", "disable": "disable ", - "list": "list", "stats": "stats []", "save": "save <(file)|list|all>", "load": "load <(file)|list|all>", @@ -27,7 +26,7 @@ "allc": "allc <(network)|(alias)> ", "admall": "admall ", "swho": "swho []", - "list": "list ", + "list": "list []", "exec": "exec ", "reg": "reg ", "confirm": "confirm ", From 8deac2ab173dd7c8a265fc1a0d534d002448f0bd Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 00:10:33 +0000 Subject: [PATCH 205/394] Implement another level of logging for tracing --- conf/example/config.json | 1 + threshold | 2 ++ utils/logging/debug.py | 3 +++ 3 files changed, 6 insertions(+) diff --git a/conf/example/config.json b/conf/example/config.json index a3dfecc..348af88 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -17,6 +17,7 @@ "ConnectOnCreate": false, "AutoReg": false, "Debug": false, + "Trace", false, "Relay": { "Host": "127.0.0.1", "Port": "201x", diff --git a/threshold b/threshold index 9425e60..59e9fab 100755 --- a/threshold +++ b/threshold @@ -17,6 +17,8 @@ from utils.cleanup import handler signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine if "--debug" in sys.argv: # yes really main.config["Debug"] = True +if "--trace" in sys.argv: + main.config["Trace"] = True from utils.logging.log import * from utils.loaders.command_loader import loadCommands from core.server import Server, ServerFactory diff --git a/utils/logging/debug.py b/utils/logging/debug.py index 4b05602..4b8c708 100644 --- a/utils/logging/debug.py +++ b/utils/logging/debug.py @@ -5,3 +5,6 @@ def debug(*data): if main.config["Debug"]: print("[DEBUG]", *data) +def trace(*data): + if main.config["Trace"]: + print("[TRACE]", *data) From c06e9227499e3ff9f8a5e29fc27a6b9abaecd39b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 00:11:28 +0000 Subject: [PATCH 206/394] Clarify error message to be more helpful --- modules/chankeep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index e4f3f00..0fc7a12 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -41,7 +41,7 @@ def emptyChanAllocate(net, flist, relay, new): allocated = {} toalloc = len(flist) if toalloc > sum(chanfree[0].values()): - error("Too many channels to allocate for %s - this is probably a bug" % net) + error("Too many channels to allocate for %s - channels joined since initial query, wait a while" % net) return False for i in chanfree[0].keys(): for x in range(chanfree[0][i]): From b986d6ac45962bb6932c4c7d8b8a1e3689ecfef0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 00:12:06 +0000 Subject: [PATCH 207/394] Deauth bot when disconnected and lowercase user --- core/bot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index 04299a2..3b0660d 100644 --- a/core/bot.py +++ b/core/bot.py @@ -72,7 +72,7 @@ class IRCRelay(IRCClient): if user == None: self.user = main.config["Relay"]["User"] else: - self.user = user + self.user = user.lower() password = main.config["Relay"]["Password"] self.nickname = "relay" self.realname = "relay" @@ -167,6 +167,7 @@ class IRCBot(IRCClient): line = line.decode("utf-8", "replace") line = lowDequote(line) + trace(self.net, self.num, line) try: prefix, command, params = parsemsg(line) if command in numeric_to_symbolic: @@ -235,6 +236,7 @@ class IRCBot(IRCClient): if "Disconnected from IRC" in cast["msg"]: log("ZNC disconnected on %s - %i" % (self.net, self.num)) self.isconnected = False + self.authenticated = False if "Connected!" in cast["msg"]: log("ZNC connected on %s - %i" % (self.net, self.num)) self.isconnected = True From 388cd1e4b981d9e8aee04bb883dda744ac48dfca Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 00:13:09 +0000 Subject: [PATCH 208/394] Error checking in testing for registration message --- modules/regproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/regproc.py b/modules/regproc.py index 4ad63a4..101c752 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -81,7 +81,7 @@ def registerTest(c): inst = selectInst(c["net"]) if inst["check"] == False: return - if "msg" in c.keys(): + if "msg" in c.keys() and not c["msg"] == None: if inst["checktype"] == "msg": if c["type"] == "query": if inst["checkmsg"] in c["msg"] and c["nick"] == inst["entity"]: From eaeb4b72c26dd148d7577e3b8a1f55efc8cd36ca Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 00:13:59 +0000 Subject: [PATCH 209/394] Use zero-padded numbers to maximise usuable ports --- utils/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/get.py b/utils/get.py index 46e9edf..6843479 100644 --- a/utils/get.py +++ b/utils/get.py @@ -2,7 +2,7 @@ import main def getRelay(num): host = main.config["Relay"]["Host"].replace("x", str(num)) - port = int(str(main.config["Relay"]["Port"]).replace("x", str(num))) + port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2))) user = main.config["Relay"]["User"] password = main.config["Relay"]["Password"] try: From d60d89dbf634f30d26c93164f79c5369cf5a6308 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 16:49:37 +0000 Subject: [PATCH 210/394] Improve authentication detection Add a negative check in the event we are authenticated and registered, but not confirmed, as this fools other checks. --- commands/authcheck.py | 4 +-- commands/recheckauth.py | 36 +++++++++++++++++++++++++ conf/example/irc.json | 13 ++++++--- conf/help.json | 3 ++- core/bot.py | 40 ++++++++++++++++++++++++--- modules/regproc.py | 60 ++++++++++++++++++++++++++--------------- 6 files changed, 125 insertions(+), 31 deletions(-) create mode 100644 commands/recheckauth.py diff --git a/commands/authcheck.py b/commands/authcheck.py index 8ed5575..120adce 100644 --- a/commands/authcheck.py +++ b/commands/authcheck.py @@ -16,8 +16,8 @@ class AuthcheckCommand: info("\n".join(results)) return elif length == 2: - if not spl[1] in main.IRCPool.keys(): - failure("No such instance: %s" % spl[1]) + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) return results = [] for i in main.IRCPool.keys(): diff --git a/commands/recheckauth.py b/commands/recheckauth.py new file mode 100644 index 0000000..0c5c11a --- /dev/null +++ b/commands/recheckauth.py @@ -0,0 +1,36 @@ +import main + +class RecheckauthCommand: + def __init__(self, *args): + self.recheckauth(*args) + + def recheckauth(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 1: + results = [] + for i in main.IRCPool.keys(): + num = main.IRCPool[i].num + net = main.IRCPool[i].net + main.IRCPool[i].authenticated = False + main.IRCPool[i].regPing() + success("Successfully reset authentication status") + return + elif length == 2: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + results = [] + for i in main.IRCPool.keys(): + num = main.IRCPool[i].num + net = main.IRCPool[i].net + if not net == spl[1]: + continue + main.IRCPool[i].authenticated = False + main.IRCPool[i].regPing() + success("Successfully reset authentication status on %s" % spl[1]) + return + else: + incUsage("recheckauth") + return + else: + incUsage(None) diff --git a/conf/example/irc.json b/conf/example/irc.json index d2f6bb7..8270e2f 100644 --- a/conf/example/irc.json +++ b/conf/example/irc.json @@ -3,14 +3,21 @@ "register": true, "entity": "NickServ", "email": "{nickname}@domain.com", - "register": "REGISTER {password} {email}", + "registermsg": "REGISTER {password} {email}", "confirm": "CONFIRM {token}", "check": true, "ping": true, - "pingmsg": "STATUS {nickname}", + "negative": true, + "pingmsg": "STATUS", + "negativemsg": "INFO {nickname}", "checktype": "mode", "checkmode": "r", "checkmsg": "Password accepted - you are now recognized.", - "checkmsg2": "You are logged in as" + "checkmsg2": "You are logged in as", + "checknegativemsg": "has \u0002NOT COMPLETED\u0002 registration verification", + "checkendnegative": "End of Info" + }, + "freenode": { + "confirm": "VERIFY REGISTER {nickname} {token}" } } diff --git a/conf/help.json b/conf/help.json index d016a67..b70dead 100644 --- a/conf/help.json +++ b/conf/help.json @@ -31,5 +31,6 @@ "reg": "reg ", "confirm": "confirm ", "pending": "pending []", - "authcheck": "authcheck []" + "authcheck": "authcheck []", + "recheckauth": "recheckauth []" } diff --git a/core/bot.py b/core/bot.py index 3b0660d..c1be085 100644 --- a/core/bot.py +++ b/core/bot.py @@ -160,6 +160,7 @@ class IRCBot(IRCClient): self.servername = None self._regAttempt = None + self._negativePass = None def lineReceived(self, line): if bytes != str and isinstance(line, bytes): @@ -167,7 +168,6 @@ class IRCBot(IRCClient): line = line.decode("utf-8", "replace") line = lowDequote(line) - trace(self.net, self.num, line) try: prefix, command, params = parsemsg(line) if command in numeric_to_symbolic: @@ -618,14 +618,46 @@ class IRCBot(IRCClient): self.topicUpdated(prefix, channel, newtopic) # End of Twisted hackery + def regPing(self, negativepass=None): + if self.authenticated: + return + sinst = regproc.substitute(self.net, self.num) + if not self._negativePass == True: + if negativepass == False: + self._negativePass = False + return + if negativepass == True: + if self._negativePass == None: + self._negativePass = True + debug("Positive registration check - %s - %i" % (self.net, self.num)) + if sinst["ping"]: + debug("Sending ping - %s - %i" % (self.net, self.num)) + self.msg(sinst["entity"], sinst["pingmsg"]) + return + else: + debug("Negative registration for %s - %i" % (self.net, self.num)) + return + + if sinst["check"]: + if sinst["negative"]: + self._negativePass = None + self.msg(sinst["entity"], sinst["negativemsg"]) + return + else: + self._negativePass = True + if sinst["ping"]: + self.msg(sinst["entity"], sinst["pingmsg"]) + return + else: + self.authenticated = True + warn("Won't send registration ping for %s - %i" % (self.net, self.num)) + def signedOn(self): log("signed on: %s - %i" % (self.net, self.num)) ctime = str(datetime.now().isoformat()) sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon", "time": ctime}) if not self.authenticated: - inst = regproc.selectInst(self.net) - if inst["ping"] and inst["check"]: - self.msg(inst["entity"], inst["pingmsg"]) + reactor.callLater(10, self.regPing) def joined(self, channel): if not channel in self.channels: diff --git a/modules/regproc.py b/modules/regproc.py index 101c752..0df5d33 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -22,38 +22,45 @@ def selectInst(net): inst = main.irc["_"] return inst -def registerAccount(net, num): - debug("Attempting to register: %s - %i" % (net, num)) +def substitute(net, num, token=None): + inst = selectInst(net) alias = main.alias[num] nickname = alias["nick"] username = nickname+"/"+net password = main.network[net].aliases[num]["password"] - inst = selectInst(net) + inst["email"] = inst["email"].replace("{nickname}", nickname) + for i in inst.keys(): + if not isinstance(inst[i], str): + continue + inst[i] = inst[i].replace("{nickname}", nickname) + inst[i] = inst[i].replace("{password}", password) + inst[i] = inst[i].replace("{email}", inst["email"]) + if token: + inst[i] = inst[i].replace("{token}", token) + return inst + +def registerAccount(net, num): + debug("Attempting to register: %s - %i" % (net, num)) + sinst = substitute(net, num) if not inst["register"]: error("Cannot register for %s: function disabled" % (net)) return False - entity = inst["entity"] - email = inst["email"] - cmd = inst["register"] - email = email.replace("{nickname}", nickname) - cmd = cmd.replace("{password}", password) - cmd = cmd.replace("{email}", email) name = net+str(num) - main.IRCPool[name].msg(entity, cmd) + main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"]) def confirmAccount(net, num, token): - inst = selectInst(net) - entity = inst["entity"] - cmd = inst["confirm"] - cmd = cmd.replace("{token}", token) + sinst = substitute(net, num, token=token) name = net+str(num) - main.IRCPool[name].msg(entity, cmd) + main.IRCPool[name].msg(sinst["entity"], sinst["confirm"]) enableAuthentication(net, num) -def confirmRegistration(net, num): +def confirmRegistration(net, num, negativepass=None): obj = main.network[net] name = net+str(num) if name in main.IRCPool.keys(): + if not negativepass == None: + main.IRCPool[name].regPing(negativepass=negativepass) + return debug("Relay authenticated: %s - %i" %(net, num)) main.IRCPool[name].authenticated = True main.IRCPool[name].recheckList() @@ -79,16 +86,27 @@ def enableAuthentication(net, num): def registerTest(c): inst = selectInst(c["net"]) + name = c["net"]+str(c["num"]) if inst["check"] == False: return if "msg" in c.keys() and not c["msg"] == None: - if inst["checktype"] == "msg": - if c["type"] == "query": - if inst["checkmsg"] in c["msg"] and c["nick"] == inst["entity"]: - confirmRegistration(c["net"], c["num"]) - return + if inst["negative"]: + if name in main.IRCPool.keys(): + if not main.IRCPool[name]._negativePass == True: + if c["type"] == "query" and c["nick"] == inst["entity"]: + if inst["checknegativemsg"] in c["msg"]: + confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back + return + if inst["checkendnegative"] in c["msg"]: + confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back + return if inst["ping"]: if inst["checkmsg2"] in c["msg"] and c["nick"] == inst["entity"]: + print("CHECKMSG2 SUCCEEDED", c["num"]) + confirmRegistration(c["net"], c["num"]) + return + if inst["checktype"] == "msg": + if inst["checkmsg"] in c["msg"]: confirmRegistration(c["net"], c["num"]) return elif inst["checktype"] == "mode": From 9e17223258e9a2696dc2168467baa69f8716a6e4 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 16:51:24 +0000 Subject: [PATCH 211/394] Don't deduplicate global messages (NICK/QUIT) --- utils/dedup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/dedup.py b/utils/dedup.py index a5c3c45..6746112 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -6,6 +6,8 @@ import main from utils.logging.debug import debug def dedup(numName, b): + if b["type"] in ["quit", "nick"]: + return False # QUITs and NICKs can't be duplicated, they are global and duplications are likely c = deepcopy(b) if "time" in c.keys(): del c["time"] From b0eaa7fd472d1c7020701a65a006a7e1824fa158 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 16:52:00 +0000 Subject: [PATCH 212/394] Move WHO and NAMES logging to trace --- modules/userinfo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/userinfo.py b/modules/userinfo.py index fef0faa..a9104da 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -3,7 +3,7 @@ from string import digits import main from utils.logging.log import * -from utils.logging.debug import debug +from utils.logging.debug import debug, trace from utils.parsing import parsen def getWhoSingle(name, query): @@ -84,7 +84,7 @@ def _initialUsers(name, channel, users): p.execute() def initialUsers(name, channel, users): - debug("Initialising WHO records for %s on %s" % (channel, name)) + trace("Initialising WHO records for %s on %s" % (channel, name)) d = deferToThread(_initialUsers, name, channel, users) #d.addCallback(testCallback) @@ -99,7 +99,7 @@ def _initialNames(name, channel, names): p.execute() def initialNames(name, channel, names): - debug("Initialising NAMES records for %s on %s" % (channel, name)) + trace("Initialising NAMES records for %s on %s" % (channel, name)) d = deferToThread(_initialNames, name, channel, names) #d.addCallback(testCallback) @@ -235,6 +235,6 @@ def _delChannels(net, channels): p.execute() def delChannels(net, channels): # we have left a channel - debug("Purging channel %s for %s" % (", ".join(channels), net)) + trace("Purging channel %s for %s" % (", ".join(channels), net)) d = deferToThread(_delChannels, net, channels) #d.addCallback(testCallback) From 49fd03304d8839c7ff55445fa8db917238c68ae2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 23:55:11 +0000 Subject: [PATCH 213/394] Fix various bugs and off by one with provisioning --- modules/chankeep.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 0fc7a12..3de11b0 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -41,7 +41,10 @@ def emptyChanAllocate(net, flist, relay, new): allocated = {} toalloc = len(flist) if toalloc > sum(chanfree[0].values()): - error("Too many channels to allocate for %s - channels joined since initial query, wait a while" % net) + correction = round(toalloc-sum(chanfree[0].values()) / chanfree[1]) + #print("correction", correction) + warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) + #newNums = modules.provision.provisionMultipleRelays(net, correction) return False for i in chanfree[0].keys(): for x in range(chanfree[0][i]): From b652b11335a97788ba91913202c14df9643452df Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 23:58:03 +0000 Subject: [PATCH 214/394] Fix registration cancellation bug in regproc --- conf/help.json | 2 +- modules/regproc.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/conf/help.json b/conf/help.json index b70dead..b8f9ad7 100644 --- a/conf/help.json +++ b/conf/help.json @@ -33,4 +33,4 @@ "pending": "pending []", "authcheck": "authcheck []", "recheckauth": "recheckauth []" -} +} \ No newline at end of file diff --git a/modules/regproc.py b/modules/regproc.py index 0df5d33..0cdb1c9 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -42,7 +42,7 @@ def substitute(net, num, token=None): def registerAccount(net, num): debug("Attempting to register: %s - %i" % (net, num)) sinst = substitute(net, num) - if not inst["register"]: + if not sinst["register"]: error("Cannot register for %s: function disabled" % (net)) return False name = net+str(num) @@ -68,7 +68,10 @@ def confirmRegistration(net, num, negativepass=None): return if name in main.IRCPool.keys(): if main.IRCPool[name]._regAttempt: - main.IRCPool[name]._regAttempt.cancel() + try: + main.IRCPool[name]._regAttempt.cancel() + except: + pass obj.relays[num]["registered"] = True main.saveConf("network") @@ -102,7 +105,6 @@ def registerTest(c): return if inst["ping"]: if inst["checkmsg2"] in c["msg"] and c["nick"] == inst["entity"]: - print("CHECKMSG2 SUCCEEDED", c["num"]) confirmRegistration(c["net"], c["num"]) return if inst["checktype"] == "msg": From e22349802b7cacaac8a52ab91429143dd1c195f8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 31 Oct 2020 23:58:51 +0000 Subject: [PATCH 215/394] Log error when ZNC says a channel can't be joined --- core/bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index c1be085..f831286 100644 --- a/core/bot.py +++ b/core/bot.py @@ -240,6 +240,8 @@ class IRCBot(IRCClient): if "Connected!" in cast["msg"]: log("ZNC connected on %s - %i" % (self.net, self.num)) self.isconnected = True + if "could not be joined, disabling it" in cast["msg"]: + error(cast["msg"]) # # don't reprocess the same message twice @@ -650,7 +652,6 @@ class IRCBot(IRCClient): return else: self.authenticated = True - warn("Won't send registration ping for %s - %i" % (self.net, self.num)) def signedOn(self): log("signed on: %s - %i" % (self.net, self.num)) From a78e05c0c3bcf2de2d172cd0d84064241ba5b064 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 03:36:23 +0000 Subject: [PATCH 216/394] Clarify message output on confirm command --- commands/confirm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/confirm.py b/commands/confirm.py index 93af4d0..c7b1552 100644 --- a/commands/confirm.py +++ b/commands/confirm.py @@ -15,7 +15,7 @@ class ConfirmCommand: failure("Must be a number, not %s" % spl[2]) return if not int(spl[2]) in main.network[spl[1]].relays.keys(): - failure("No such relay on %s: %s" % (spl[2], spl[1])) + failure("No such relay on %s: %s" % (spl[1], spl[2])) return regproc.confirmAccount(spl[1], int(spl[2]), spl[3]) success("Requested confirmation on %s - %s with token %s" % (spl[1], spl[2], spl[3])) From f7e1f2d221939ce44a467d4155208145de0de72a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 03:37:29 +0000 Subject: [PATCH 217/394] Implement registration at net-level --- commands/reg.py | 10 +++++++++- conf/help.json | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/commands/reg.py b/commands/reg.py index 406dda4..1e3c272 100644 --- a/commands/reg.py +++ b/commands/reg.py @@ -7,7 +7,15 @@ class RegCommand: def reg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 3: + if length == 2: + if not spl[1] in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + if for i in main.network[spl[1]].relays.keys(): + regproc.registerAccount(spl[1], int(spl[2])) + success("Requested registration for all relays on %s" % spl[1]) + return + elif length == 3: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return diff --git a/conf/help.json b/conf/help.json index b8f9ad7..704d979 100644 --- a/conf/help.json +++ b/conf/help.json @@ -28,9 +28,9 @@ "swho": "swho []", "list": "list []", "exec": "exec ", - "reg": "reg ", + "reg": "reg []", "confirm": "confirm ", "pending": "pending []", "authcheck": "authcheck []", "recheckauth": "recheckauth []" -} \ No newline at end of file +} From abdfc48b959ad93a1cffd437742acdfae80bcfcc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 03:38:47 +0000 Subject: [PATCH 218/394] Prepare command loader for reloading commands --- utils/loaders/command_loader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 031f2aa..bc9ecf4 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -6,7 +6,7 @@ import commands from main import CommandMap -def loadCommands(): +def loadCommands(allowDup=False): for filename in listdir("commands"): if filename.endswith(".py") and filename != "__init__.py": commandName = filename[0:-3] @@ -17,6 +17,10 @@ def loadCommands(): CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) else: + if allowDup: + CommandMap[commandName] = getattr(getattr(module, commandName), className) + debug("Registered command: %s" % commandName) + error("Duplicate command: %s" % (commandName)) except Exception as err: error("Exception while loading command %s:\n%s" % (commandName, err)) From 19e04dbf36ee963cca0f3c0afdb70a41587ab28f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 03:39:32 +0000 Subject: [PATCH 219/394] Implement setting modes in ZNC --- modules/provision.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/provision.py b/modules/provision.py index b00b592..69fe99e 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -2,6 +2,7 @@ import main from core.bot import deliverRelayCommands from utils.logging.log import * from twisted.internet import reactor +import modules.regproc def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security, auth, password): commands = {} @@ -25,6 +26,10 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, if main.config["Toggles"]["CycleChans"]: stage2commands["status"].append("LoadMod disconkick") stage2commands["status"].append("LoadMod chansaver") + inst = modules.regproc.selectInst(network) + if "setmode" in inst.keys(): + stage2commands["status"].append("LoadMod perform") + #stage2commands["perform"].append("add mode %nick% +"+inst["setmode"]) deliverRelayCommands(num, commands, stage2=[[user+"/"+network, stage2commands]]) @@ -41,6 +46,10 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): commands["nickserv"] = [] commands["status"].append("LoadMod nickserv") commands["nickserv"].append("Set %s" % password) + inst = modules.regproc.selectInst(network) + if "setmode" in inst.keys(): + #commands["status"].append("LoadMod perform") + commands["perform"] = ["add mode %nick% +"+inst["setmode"]] deliverRelayCommands(num, commands, user=user+"/"+network) From 5d09e1ade750e5a4fa825710257240677246d295 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 18:50:17 +0000 Subject: [PATCH 220/394] Fix syntax error in reg command --- commands/reg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/reg.py b/commands/reg.py index 1e3c272..46ddae0 100644 --- a/commands/reg.py +++ b/commands/reg.py @@ -11,7 +11,7 @@ class RegCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - if for i in main.network[spl[1]].relays.keys(): + for i in main.network[spl[1]].relays.keys(): regproc.registerAccount(spl[1], int(spl[2])) success("Requested registration for all relays on %s" % spl[1]) return From a0bea0b18ae146f10d9ba12e8bea69db8cfa30ea Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 19:03:56 +0000 Subject: [PATCH 221/394] Fix bug with using muser attribute when absent --- core/bot.py | 3 ++- modules/monitor.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index f831286..0bb64b2 100644 --- a/core/bot.py +++ b/core/bot.py @@ -221,7 +221,8 @@ class IRCBot(IRCClient): # expand out the hostmask if not {"nick", "ident", "host"}.issubset(set(cast.keys())): - cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) + if "muser" in cast.keys(): + cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) # handle ZNC stuff if {"nick", "ident", "host", "msg"}.issubset(set(cast)): diff --git a/modules/monitor.py b/modules/monitor.py index 4b3d581..e9dea5a 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -18,7 +18,8 @@ def parsemeta(numName, c): # need to check if this was received from a relay # in which case, do not do this if c["type"] in ["msg", "notice", "action", "topic", "mode"]: - userinfo.editUser(c["net"], c["muser"]) + if "muser" in c.keys(): + userinfo.editUser(c["net"], c["muser"]) #if c["type"] == "mode": # userinfo.updateMode(c) elif c["type"] == "nick": From 399075afd1598879622a0f69dcb4b60f1f03b688 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 19:54:24 +0000 Subject: [PATCH 222/394] Implement channel blacklisting --- .gitignore | 1 + commands/blacklist.py | 41 +++++++++++++++++++++++++++++++++++++++++ conf/help.json | 3 ++- core/bot.py | 4 ++++ main.py | 1 + 5 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 commands/blacklist.py diff --git a/.gitignore b/.gitignore index 9d56c45..34d919c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ conf/network.dat conf/alias.json conf/irc.json conf/dist.sh +conf/blacklist.json env/ diff --git a/commands/blacklist.py b/commands/blacklist.py new file mode 100644 index 0000000..89d25c6 --- /dev/null +++ b/commands/blacklist.py @@ -0,0 +1,41 @@ +import main +from yaml import dump + +class BlacklistCommand: + def __init__(self, *args): + self.blacklist(*args) + + def blacklist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 1: + info(dump(main.blacklist)) + return + elif length == 4: + if spl[1] == "add": + if spl[2] in main.blacklist.keys(): + main.blacklist[spl[2]].append(spl[3]) + else: + main.blacklist[spl[2]] = [spl[3]] + success("Blacklisted %s on %s" % (spl[3], spl[2])) + main.saveConf("blacklist") + return + elif spl[1] == "del": + if spl[2] in main.blacklist.keys(): + if spl[3] in main.blacklist[spl[2]]: + main.blacklist[spl[2]].remove(spl[3]) + if len(main.blacklist[spl[2]]) == 0: + del main.blacklist[spl[2]] + else: + failure("Not in list: %s" % spl[3]) + return + else: + failure("No such entry: %s" % spl[2]) + return + success("Removed blacklist %s on %s" % (spl[3], spl[2])) + main.saveConf("blacklist") + return + else: + incUsage("blacklist") + return + else: + incUsage(None) diff --git a/conf/help.json b/conf/help.json index 704d979..e36b6ec 100644 --- a/conf/help.json +++ b/conf/help.json @@ -32,5 +32,6 @@ "confirm": "confirm ", "pending": "pending []", "authcheck": "authcheck []", - "recheckauth": "recheckauth []" + "recheckauth": "recheckauth []", + "blacklist": "blacklist " } diff --git a/core/bot.py b/core/bot.py index 0bb64b2..75cee7c 100644 --- a/core/bot.py +++ b/core/bot.py @@ -181,6 +181,10 @@ class IRCBot(IRCClient): increment = 0.8 for i in channels: if not i in self.channels: + if self.net in main.blacklist.keys(): + if i in main.blacklist[self.net]: + debug("Not joining blacklisted channel %s on %s - %i" % (i, self.net, self.num)) + continue debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds") reactor.callLater(sleeptime, self.join, i) sleeptime += increment diff --git a/main.py b/main.py index e5ec77d..caebef7 100644 --- a/main.py +++ b/main.py @@ -24,6 +24,7 @@ filemap = { "aliasdata": ["aliasdata.json", "data for alias generation", "json"], "alias": ["alias.json", "provisioned alias data", "json"], "irc": ["irc.json", "IRC network definitions", "json"], + "blacklist": ["blacklist.json", "IRC channel blacklist", "json"], # Binary (pickle) configs "network": ["network.dat", "network list", "pickle"] From d405a4cd10a6b0588c1eee498fe94168a8a5465c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 19:55:32 +0000 Subject: [PATCH 223/394] Add example file for blacklist --- conf/example/blacklist.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 conf/example/blacklist.json diff --git a/conf/example/blacklist.json b/conf/example/blacklist.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/conf/example/blacklist.json @@ -0,0 +1 @@ +{} From 735fee9286b14399255c90f82541b24bf6d23088 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 20:43:51 +0000 Subject: [PATCH 224/394] Fix bug with reg command --- commands/reg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/reg.py b/commands/reg.py index 46ddae0..27c0150 100644 --- a/commands/reg.py +++ b/commands/reg.py @@ -12,7 +12,7 @@ class RegCommand: failure("No such network: %s" % spl[1]) return for i in main.network[spl[1]].relays.keys(): - regproc.registerAccount(spl[1], int(spl[2])) + regproc.registerAccount(spl[1], i) success("Requested registration for all relays on %s" % spl[1]) return elif length == 3: From 0473c57291edfc21e3a3b3f4549a26669ed12f48 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 22:18:48 +0000 Subject: [PATCH 225/394] Additional error handling for command parsing --- core/bot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index 75cee7c..fce82c8 100644 --- a/core/bot.py +++ b/core/bot.py @@ -172,7 +172,10 @@ class IRCBot(IRCClient): prefix, command, params = parsemsg(line) if command in numeric_to_symbolic: command = numeric_to_symbolic[command] - self.handleCommand(command, prefix, params) + try: + self.handleCommand(command, prefix, params) + except Exception as err: + error(err) except IRCBadMessage: self.badMessage(line, *sys.exc_info()) From 45fa21fea316e6994a855b401baec4a2c23cb892 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Nov 2020 22:19:03 +0000 Subject: [PATCH 226/394] Use substitutions in registration tests --- modules/regproc.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/regproc.py b/modules/regproc.py index 0cdb1c9..97c330a 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -88,32 +88,32 @@ def enableAuthentication(net, num): confirmRegistration(net, num) def registerTest(c): - inst = selectInst(c["net"]) + sinst = substitute(c["net"], c["num"]) name = c["net"]+str(c["num"]) - if inst["check"] == False: + if sinst["check"] == False: return if "msg" in c.keys() and not c["msg"] == None: - if inst["negative"]: + if sinst["negative"]: if name in main.IRCPool.keys(): if not main.IRCPool[name]._negativePass == True: - if c["type"] == "query" and c["nick"] == inst["entity"]: - if inst["checknegativemsg"] in c["msg"]: + if c["type"] == "query" and c["nick"] == sinst["entity"]: + if sinst["checknegativemsg"] in c["msg"]: confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back return - if inst["checkendnegative"] in c["msg"]: + if sinst["checkendnegative"] in c["msg"]: confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back return - if inst["ping"]: - if inst["checkmsg2"] in c["msg"] and c["nick"] == inst["entity"]: + if sinst["ping"]: + if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: confirmRegistration(c["net"], c["num"]) return - if inst["checktype"] == "msg": - if inst["checkmsg"] in c["msg"]: + if sinst["checktype"] == "msg": + if sinst["checkmsg"] in c["msg"]: confirmRegistration(c["net"], c["num"]) return - elif inst["checktype"] == "mode": + elif sinst["checktype"] == "mode": if c["type"] == "self": if c["mtype"] == "mode": - if inst["checkmode"] in c["mode"] and c["status"] == True: + if sinst["checkmode"] in c["mode"] and c["status"] == True: confirmRegistration(c["net"], c["num"]) return From 14daa9dfef7997881437d6d043e5048b77b03280 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 2 Nov 2020 20:13:36 +0000 Subject: [PATCH 227/394] Don't discard server messages --- core/bot.py | 8 ++++---- utils/dedup.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/bot.py b/core/bot.py index fce82c8..bd3743f 100644 --- a/core/bot.py +++ b/core/bot.py @@ -220,10 +220,10 @@ class IRCBot(IRCClient): # remove server stuff if "muser" in cast.keys(): if cast["muser"] == self.servername: - return + cast["type"] = "conn" if "channel" in cast.keys(): if cast["channel"] == "*": - return + cast["type"] = "conn" ## # expand out the hostmask @@ -254,7 +254,7 @@ class IRCBot(IRCClient): # don't reprocess the same message twice # if the type is in that list, it's already been here, don't run it again - if not cast["type"] in {"query", "self", "highlight", "znc", "who"}: + if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}: cast["num"] = self.num if "channel" in cast.keys(): if cast["type"] == "mode": @@ -279,7 +279,7 @@ class IRCBot(IRCClient): # TODO: better way to do this # as we changed the types above, check again - if not cast["type"] in {"query", "self", "highlight", "znc", "who"}: + if not cast["type"] in {"query", "self", "highlight", "znc", "who", "conn"}: # we have been kicked if "user" in cast.keys(): if cast["user"].lower() == self.nickname.lower(): diff --git a/utils/dedup.py b/utils/dedup.py index 6746112..a5c3c45 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -6,8 +6,6 @@ import main from utils.logging.debug import debug def dedup(numName, b): - if b["type"] in ["quit", "nick"]: - return False # QUITs and NICKs can't be duplicated, they are global and duplications are likely c = deepcopy(b) if "time" in c.keys(): del c["time"] From 9c95fa8eafaf99f39bec824d56cd50c792180a07 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 2 Nov 2020 20:14:02 +0000 Subject: [PATCH 228/394] Implement relay-independent join --- commands/join.py | 10 +++++++++- modules/chankeep.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/commands/join.py b/commands/join.py index ff912bc..02067c7 100644 --- a/commands/join.py +++ b/commands/join.py @@ -1,4 +1,5 @@ import main +import modules.chankeep class JoinCommand: def __init__(self, *args): @@ -6,7 +7,14 @@ class JoinCommand: def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: - if length == 4: + if length == 3: + if not spl[1] in main.network.keys(): + failure("Network does not exist: %s" % spl[1]) + return + modules.chankeep.joinSingle(spl[1], spl[2]) + success("Joined %s" % spl[2]) + + elif length == 4: if not spl[1] in main.network.keys(): failure("Network does not exist: %s" % spl[1]) return diff --git a/modules/chankeep.py b/modules/chankeep.py index 3de11b0..1cc70c0 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -111,6 +111,17 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): populateChans(net, siglist, sigrelay, newNums) notifyJoin(net) +def joinSingle(net, channel): + if allRelaysActive(net): + chanfree = getChanFree(net, []) + print("chanfree", chanfree) + for i in chanfree[0]: + if chanfree[0][i] < 0: + print("JOIN CHAN") + else: + error("All relays for %s are not active" % net) + return False + def nukeNetwork(net): #purgeRecords(net) #p = main.g.pipeline() From 5d63d7a1e9767b06a996ab68c53deb953386ff9d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 6 Jun 2021 10:13:43 +0000 Subject: [PATCH 229/394] Update requirements without versions --- requirements.txt | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/requirements.txt b/requirements.txt index 11a3e06..5ff3e80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,7 @@ -asn1crypto==0.24.0 -attrs==19.1.0 -Automat==0.7.0 -cffi==1.12.3 -constantly==15.1.0 -cryptography==2.7 -csiphash==0.0.5 -hyperlink==19.0.0 -idna==2.8 -incremental==17.5.0 -numpy==1.17.2 -pyasn1==0.4.6 -pyasn1-modules==0.2.6 -pycparser==2.19 -PyHamcrest==1.9.0 -pyOpenSSL==19.0.0 -PyYAML==5.1.2 -redis==3.3.8 -service-identity==18.1.0 -six==1.12.0 -Twisted==19.7.0 -zope.interface==4.6.0 +twisted +pyOpenSSL +redis +pyYaML +python-logstash +service_identity +csiphash From c38909436539c6982e5336d3af81fea206b0b841 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 6 Jun 2021 10:16:04 +0000 Subject: [PATCH 230/394] Finish Logstash implementation --- conf/example/config.json | 4 ++++ modules/monitor.py | 6 +++++- threshold | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/example/config.json b/conf/example/config.json index 348af88..9471a47 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -24,6 +24,10 @@ "User": "sir", "Password": "sir" }, + "Logstash": { + "Host": "10.0.0.2", + "Port": "4000" + }, "ChanKeep": { "MaxRelay": 30, "SigSwitch": 20 diff --git a/modules/monitor.py b/modules/monitor.py index e9dea5a..2dbfc2e 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -3,6 +3,7 @@ from json import dumps import main from core.relay import sendRelayNotification +from core.logstash import sendLogstashNotification from modules import userinfo from modules import regproc from utils.dedup import dedup @@ -47,4 +48,7 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if "muser" in c.keys(): del c["muser"] - sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order + sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order + sortedKeys["src"] = "irc" + sendLogstashNotification(sortedKeys) + sendRelayNotification(sortedKeys) diff --git a/threshold b/threshold index 59e9fab..46c0770 100755 --- a/threshold +++ b/threshold @@ -25,6 +25,8 @@ from core.server import Server, ServerFactory from core.relay import Relay, RelayFactory import modules.counters loadCommands() +import core.logstash +core.logstash.init_logstash() if __name__ == "__main__": listener = ServerFactory() From edc5f85ba616ba32dc33a43a3004e7e071ea7b3e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 6 Jun 2021 10:31:13 +0000 Subject: [PATCH 231/394] Implement modifying emails for aliases --- commands/email.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++ modules/alias.py | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 commands/email.py diff --git a/commands/email.py b/commands/email.py new file mode 100644 index 0000000..ab0b5a2 --- /dev/null +++ b/commands/email.py @@ -0,0 +1,55 @@ +import main +from yaml import dump + +class EmailCommand: + def __init__(self, *args): + self.email(*args) + + def email(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 4: + if spl[1] == "add": + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return + num = int(spl[2]) + if not num in main.alias.keys(): + failure("No such alias: %i" % num) + return + if not spl[3] in main.alias[num]["emails"]: + main.alias[num]["emails"].append(spl[3]) + main.saveConf("alias") + success("Successfully added email %s to alias %i" % (spl[3], num)) + return + else: + failure("Email already exists in alias %i: %s" % (num, spl[3])) + return + elif spl[1] == "del": + if not spl[2].isdigit(): + failure("Must be a number, not %s" % spl[2]) + return + num = int(spl[2]) + if not num in main.alias.keys(): + failure("No such alias: %i" % num) + return + if spl[3] in main.alias[num]["emails"]: + main.alias[num]["emails"].remove(spl[3]) + main.saveConf("alias") + success("Successfully removed email %s from alias %i" % (spl[3], num)) + return + else: + failure("Email does not exist in alias %i: %s" % (spl[3], num)) + return + elif length == 2: + if spl[1] == "list": + info(dump(main.alias)) + return + + else: + incUsage("save") + return + else: + incUsage("save") + return + else: + incUsage(None) diff --git a/modules/alias.py b/modules/alias.py index 4a3949f..59ef99a 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -63,4 +63,4 @@ def generate_alias(): if rand == 3 or rand == 4: realname = realname.capitalize() - return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname} + return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "emails": []} From 152bc0897028dea47092866fa1f3f95880720118 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 24 Aug 2021 20:08:18 +0000 Subject: [PATCH 232/394] Add Logstash file --- core/logstash.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 core/logstash.py diff --git a/core/logstash.py b/core/logstash.py new file mode 100644 index 0000000..aa5924b --- /dev/null +++ b/core/logstash.py @@ -0,0 +1,19 @@ +import logstash +import logging + +from json import dumps, loads +import main +from utils.logging.log import * + +logger = None +def init_logstash(): + global logger + logger = logging.getLogger('ingest') + logger.setLevel(logging.INFO) + logger.addHandler(logstash.TCPLogstashHandler(main.config["Logstash"]["Host"], int(main.config["Logstash"]["Port"]), version=1)) + +def sendLogstashNotification(text): + if not logger == None: + logger.info(dumps(text)) + return True + return False From 0777a55264ddc9415051b743d9789edbdea475a7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 25 Aug 2021 07:47:54 +0000 Subject: [PATCH 233/394] Start implementing email command --- commands/email.py | 70 ++++++++++++++++++++++++++--------------------- conf/help.json | 3 +- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/commands/email.py b/commands/email.py index ab0b5a2..56388ec 100644 --- a/commands/email.py +++ b/commands/email.py @@ -8,38 +8,46 @@ class EmailCommand: def email(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 4: - if spl[1] == "add": - if not spl[2].isdigit(): - failure("Must be a number, not %s" % spl[2]) - return + if not spl[2].isdigit(): + # failure("Must be a number, not %s" % spl[2]) + if spl[2] == "domain": + domain = spl[3] + if "@" in domain: + failure("Not a domain: %s" % domain) + return + if not domain in main.irc["_"]["domains"]: + main.irc["_"]["domains"].append(domain) + success("Successfully added domain %s to default config" % domain) + else: + failure("Domain already exists in default config: %s" % domain) + return + + else: num = int(spl[2]) - if not num in main.alias.keys(): - failure("No such alias: %i" % num) - return - if not spl[3] in main.alias[num]["emails"]: - main.alias[num]["emails"].append(spl[3]) - main.saveConf("alias") - success("Successfully added email %s to alias %i" % (spl[3], num)) - return - else: - failure("Email already exists in alias %i: %s" % (num, spl[3])) - return - elif spl[1] == "del": - if not spl[2].isdigit(): - failure("Must be a number, not %s" % spl[2]) - return - num = int(spl[2]) - if not num in main.alias.keys(): - failure("No such alias: %i" % num) - return - if spl[3] in main.alias[num]["emails"]: - main.alias[num]["emails"].remove(spl[3]) - main.saveConf("alias") - success("Successfully removed email %s from alias %i" % (spl[3], num)) - return - else: - failure("Email does not exist in alias %i: %s" % (spl[3], num)) - return + if spl[1] == "add": + if not num in main.alias.keys(): + failure("No such alias: %i" % num) + return + if not spl[3] in main.alias[num]["emails"]: + main.alias[num]["emails"].append(spl[3]) + main.saveConf("alias") + success("Successfully added email %s to alias %i" % (spl[3], num)) + return + else: + failure("Email already exists in alias %i: %s" % (num, spl[3])) + return + elif spl[1] == "del": + if not num in main.alias.keys(): + failure("No such alias: %i" % num) + return + if spl[3] in main.alias[num]["emails"]: + main.alias[num]["emails"].remove(spl[3]) + main.saveConf("alias") + success("Successfully removed email %s from alias %i" % (spl[3], num)) + return + else: + failure("Email does not exist in alias %i: %s" % (spl[3], num)) + return elif length == 2: if spl[1] == "list": info(dump(main.alias)) diff --git a/conf/help.json b/conf/help.json index e36b6ec..7ea570d 100644 --- a/conf/help.json +++ b/conf/help.json @@ -33,5 +33,6 @@ "pending": "pending []", "authcheck": "authcheck []", "recheckauth": "recheckauth []", - "blacklist": "blacklist " + "blacklist": "blacklist ", + "email": "email [(domain)|] []" } From ff1ee6390056775aac0afb46bc115d7c1c27ba03 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:41 +0100 Subject: [PATCH 234/394] Reformat code with pre-commit --- commands/admall.py | 1 + commands/alias.py | 3 +- commands/all.py | 5 +- commands/allc.py | 13 +- commands/authcheck.py | 1 + commands/auto.py | 1 + commands/blacklist.py | 1 + commands/chans.py | 3 +- commands/cmd.py | 1 + commands/confirm.py | 1 + commands/disable.py | 5 +- commands/dist.py | 1 + commands/email.py | 3 +- commands/enable.py | 5 +- commands/exec.py | 1 + commands/help.py | 1 + commands/join.py | 9 +- commands/list.py | 9 +- commands/load.py | 1 + commands/loadmod.py | 1 + commands/logout.py | 1 + commands/mod.py | 3 +- commands/msg.py | 7 +- commands/network.py | 3 +- commands/part.py | 7 +- commands/pass.py | 1 + commands/pending.py | 1 + commands/recheckauth.py | 1 + commands/reg.py | 1 + commands/relay.py | 1 + commands/save.py | 1 + commands/stats.py | 1 + commands/swho.py | 1 + commands/token.py | 12 +- commands/users.py | 3 +- commands/who.py | 3 +- core/bot.py | 220 ++++++++++++++++++++------------ core/logstash.py | 13 +- core/parser.py | 3 +- core/relay.py | 24 +++- core/server.py | 4 +- main.py | 28 ++-- modules/alias.py | 14 +- modules/chankeep.py | 68 +++++----- modules/counters.py | 4 + modules/monitor.py | 45 +++++-- modules/network.py | 15 ++- modules/provision.py | 35 ++--- modules/regproc.py | 36 ++++-- modules/userinfo.py | 102 +++++++++------ requirements.txt | 1 + threshold | 59 +++++++-- utils/cleanup.py | 6 +- utils/dedup.py | 7 +- utils/get.py | 1 + utils/loaders/command_loader.py | 5 +- utils/loaders/single_loader.py | 11 +- utils/logging/debug.py | 2 + utils/logging/log.py | 2 + utils/logging/send.py | 7 + 60 files changed, 547 insertions(+), 278 deletions(-) diff --git a/commands/admall.py b/commands/admall.py index cbbf854..8d74545 100644 --- a/commands/admall.py +++ b/commands/admall.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands + class AdmallCommand: def __init__(self, *args): self.admall(*args) diff --git a/commands/alias.py b/commands/alias.py index baccad5..d04cfdf 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -2,6 +2,7 @@ import main from yaml import dump from modules import alias + class AliasCommand: def __init__(self, *args): self.alias(*args) @@ -21,7 +22,7 @@ class AliasCommand: if len(main.alias.keys()) == 0: nextNum = 1 else: - nextNum = max(main.alias.keys())+1 + nextNum = max(main.alias.keys()) + 1 main.alias[nextNum] = alias.generate_alias() success("Generated new alias: %i" % nextNum) main.saveConf("alias") diff --git a/commands/all.py b/commands/all.py index e59a9fd..a2994e9 100644 --- a/commands/all.py +++ b/commands/all.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands + class AllCommand: def __init__(self, *args): self.all(*args) @@ -14,8 +15,8 @@ class AllCommand: net = main.network[i].relays[x]["net"] alias = main.alias[x]["nick"] commands = {spl[1]: [" ".join(spl[2:])]} - success("Sending commands to relay %s as user %s" % (num, alias+"/"+net)) - deliverRelayCommands(num, commands, user=alias+"/"+net) + success("Sending commands to relay %s as user %s" % (num, alias + "/" + net)) + deliverRelayCommands(num, commands, user=alias + "/" + net) return else: incUsage("all") diff --git a/commands/allc.py b/commands/allc.py index 5d1d7cb..7a7b5fd 100644 --- a/commands/allc.py +++ b/commands/allc.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands + class AllcCommand: def __init__(self, *args): self.allc(*args) @@ -16,9 +17,11 @@ class AllcCommand: targets.append((i, x)) elif spl[1] == "alias": for i in main.network.keys(): - [targets.append((i, x)) for x in main.alias.keys() if - main.alias[x]["nick"] == spl[2] and - x in main.network[i].aliases.keys()] + [ + targets.append((i, x)) + for x in main.alias.keys() + if main.alias[x]["nick"] == spl[2] and x in main.network[i].aliases.keys() + ] else: incUsage("allc") return @@ -30,8 +33,8 @@ class AllcCommand: num = i[1] alias = main.alias[num]["nick"] commands = {spl[3]: [" ".join(spl[4:])]} - success("Sending commands to relay %i as user %s" % (num, alias+"/"+net)) - deliverRelayCommands(num, commands, user=alias+"/"+net) + success("Sending commands to relay %i as user %s" % (num, alias + "/" + net)) + deliverRelayCommands(num, commands, user=alias + "/" + net) return else: incUsage("allc") diff --git a/commands/authcheck.py b/commands/authcheck.py index 120adce..26cc14e 100644 --- a/commands/authcheck.py +++ b/commands/authcheck.py @@ -1,5 +1,6 @@ import main + class AuthcheckCommand: def __init__(self, *args): self.authcheck(*args) diff --git a/commands/auto.py b/commands/auto.py index 29bd3a8..ffc6df5 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -1,6 +1,7 @@ import main from modules import provision + class AutoCommand: def __init__(self, *args): self.auto(*args) diff --git a/commands/blacklist.py b/commands/blacklist.py index 89d25c6..42684a6 100644 --- a/commands/blacklist.py +++ b/commands/blacklist.py @@ -1,6 +1,7 @@ import main from yaml import dump + class BlacklistCommand: def __init__(self, *args): self.blacklist(*args) diff --git a/commands/chans.py b/commands/chans.py index f92482b..1ccf710 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -1,6 +1,7 @@ import main import modules.userinfo as userinfo + class ChansCommand: def __init__(self, *args): self.chans(*args) @@ -16,7 +17,7 @@ class ChansCommand: rtrn += "Matches from: %s" % i rtrn += "\n" for x in result[i]: - rtrn += (x) + rtrn += x rtrn += "\n" rtrn += "\n" info(rtrn) diff --git a/commands/cmd.py b/commands/cmd.py index 466b197..babdb2d 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands + class CmdCommand: def __init__(self, *args): self.cmd(*args) diff --git a/commands/confirm.py b/commands/confirm.py index c7b1552..86d657c 100644 --- a/commands/confirm.py +++ b/commands/confirm.py @@ -1,6 +1,7 @@ import main from modules import regproc + class ConfirmCommand: def __init__(self, *args): self.confirm(*args) diff --git a/commands/disable.py b/commands/disable.py index 62b42df..9c52234 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands + class DisableCommand: def __init__(self, *args): self.disable(*args) @@ -15,7 +16,7 @@ class DisableCommand: failure("Must be a number, not %s" % spl[2]) return relayNum = int(spl[2]) - name = spl[1]+spl[2] + name = spl[1] + spl[2] if not spl[1] in main.IRCPool.keys(): info("Note - instance not running, proceeding anyway") if not relayNum in main.network[spl[1]].relays.keys(): @@ -26,7 +27,7 @@ class DisableCommand: network = spl[1] relay = main.network[spl[1]].relays[relayNum] commands = {"status": ["Disconnect"]} - deliverRelayCommands(relayNum, commands, user=user+"/"+network) + deliverRelayCommands(relayNum, commands, user=user + "/" + network) main.saveConf("network") if name in main.ReactorPool.keys(): if name in main.FactoryPool.keys(): diff --git a/commands/dist.py b/commands/dist.py index d5f6f53..2348e29 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -1,6 +1,7 @@ import main from subprocess import run, PIPE + class DistCommand: def __init__(self, *args): self.dist(*args) diff --git a/commands/email.py b/commands/email.py index 56388ec..1b0ec5e 100644 --- a/commands/email.py +++ b/commands/email.py @@ -1,6 +1,7 @@ import main from yaml import dump + class EmailCommand: def __init__(self, *args): self.email(*args) @@ -21,7 +22,7 @@ class EmailCommand: else: failure("Domain already exists in default config: %s" % domain) return - + else: num = int(spl[2]) if spl[1] == "add": diff --git a/commands/enable.py b/commands/enable.py index af645cd..439b26c 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -1,6 +1,7 @@ import main from core.bot import deliverRelayCommands + class EnableCommand: def __init__(self, *args): self.enable(*args) @@ -22,9 +23,9 @@ class EnableCommand: user = main.alias[int(spl[2])]["nick"] network = spl[1] commands = {"status": ["Connect"]} - deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network) + deliverRelayCommands(int(spl[2]), commands, user=user + "/" + network) main.saveConf("network") - if not spl[1]+spl[2] in main.IRCPool.keys(): + if not spl[1] + spl[2] in main.IRCPool.keys(): main.network[spl[1]].start_bot(int(spl[2])) else: pass diff --git a/commands/exec.py b/commands/exec.py index c199aa8..c2a5598 100644 --- a/commands/exec.py +++ b/commands/exec.py @@ -1,5 +1,6 @@ import main + class ExecCommand: def __init__(self, *args): self.exec(*args) diff --git a/commands/help.py b/commands/help.py index bb12aff..6dcf937 100644 --- a/commands/help.py +++ b/commands/help.py @@ -1,5 +1,6 @@ import main + class HelpCommand: def __init__(self, *args): self.help(*args) diff --git a/commands/join.py b/commands/join.py index 02067c7..e6cbc20 100644 --- a/commands/join.py +++ b/commands/join.py @@ -1,6 +1,7 @@ import main import modules.chankeep + class JoinCommand: def __init__(self, *args): self.join(*args) @@ -21,10 +22,10 @@ class JoinCommand: if not int(spl[2]) in main.network[spl[1]].relays.keys(): failure("Relay %s does not exist on network %s" % (spl[2], spl[1])) return - if not spl[1]+spl[2] in main.IRCPool.keys(): + if not spl[1] + spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - main.IRCPool[spl[1]+spl[2]].join(spl[3]) + main.IRCPool[spl[1] + spl[2]].join(spl[3]) success("Joined %s" % spl[3]) return elif length == 5: @@ -34,10 +35,10 @@ class JoinCommand: if not int(spl[2]) in main.network[spl[1]].relays.keys(): failure("Relay % does not exist on network %", (spl[2], spl[1])) return - if not spl[1]+spl[2] in main.IRCPool.keys(): + if not spl[1] + spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - main.IRCPool[spl[1]+spl[2]].join(spl[3], spl[4]) + main.IRCPool[spl[1] + spl[2]].join(spl[3], spl[4]) success("Joined %s with key %s" % (spl[3], spl[4])) return else: diff --git a/commands/list.py b/commands/list.py index 3839914..217b051 100644 --- a/commands/list.py +++ b/commands/list.py @@ -1,5 +1,6 @@ import main + class ListCommand: def __init__(self, *args): self.list(*args) @@ -11,10 +12,10 @@ class ListCommand: if not 1 in main.network[i].relays.keys(): info("Network has no first instance: %s" % i) continue - if not i+"1" in main.IRCPool.keys(): + if not i + "1" in main.IRCPool.keys(): info("No IRC instance: %s - 1" % i) continue - main.IRCPool[i+"1"].list() + main.IRCPool[i + "1"].list() success("Requested list with first instance of %s" % i) return elif length == 2: @@ -24,10 +25,10 @@ class ListCommand: if not 1 in main.network[spl[1]].relays.keys(): failure("Network has no first instance") return - if not spl[1]+"1" in main.IRCPool.keys(): + if not spl[1] + "1" in main.IRCPool.keys(): failure("No IRC instance: %s - 1" % spl[1]) return - main.IRCPool[spl[1]+"1"].list() + main.IRCPool[spl[1] + "1"].list() success("Requested list with first instance of %s" % spl[1]) return else: diff --git a/commands/load.py b/commands/load.py index aa6b37d..31e66f4 100644 --- a/commands/load.py +++ b/commands/load.py @@ -1,5 +1,6 @@ import main + class LoadCommand: def __init__(self, *args): self.load(*args) diff --git a/commands/loadmod.py b/commands/loadmod.py index fb542b9..2923fd7 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -1,6 +1,7 @@ import main from utils.loaders.single_loader import loadSingle + class LoadmodCommand: def __init__(self, *args): self.loadmod(*args) diff --git a/commands/logout.py b/commands/logout.py index d1dea67..61cb834 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -1,5 +1,6 @@ import main + class LogoutCommand: def __init__(self, *args): self.logout(*args) diff --git a/commands/mod.py b/commands/mod.py index 0df1ada..10a19e1 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,6 +1,7 @@ import main from yaml import dump + class ModCommand: # This could be greatly improved, but not really important right now def __init__(self, *args): @@ -23,7 +24,7 @@ class ModCommand: success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return # Find a better way to do this - #elif length == 6: + # elif length == 6: # if not spl[1] in main.network.keys(): # failure("Network does not exist: %s" % spl[1]) # return diff --git a/commands/msg.py b/commands/msg.py index 4ea2b2f..d92494a 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -1,5 +1,6 @@ import main + class MsgCommand: def __init__(self, *args): self.msg(*args) @@ -13,12 +14,12 @@ class MsgCommand: if not int(spl[2]) in main.network[spl[1]].relays.keys(): failure("Relay % does not exist on network %" % (spl[2], spl[1])) return - if not spl[1]+spl[2] in main.IRCPool.keys(): + if not spl[1] + spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) return - if not spl[3] in main.IRCPool[spl[1]+spl[2]].channels: + if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels: info("Bot not on channel: %s" % spl[3]) - main.IRCPool[spl[1]+spl[2]].msg(spl[3], " ".join(spl[4:])) + main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:])) success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1])) return else: diff --git a/commands/network.py b/commands/network.py index e88a905..4a6bdf2 100644 --- a/commands/network.py +++ b/commands/network.py @@ -3,6 +3,7 @@ from yaml import dump from modules.network import Network from string import digits + class NetworkCommand: def __init__(self, *args): self.network(*args) @@ -38,7 +39,7 @@ class NetworkCommand: elif length == 3: if spl[1] == "del": if spl[2] in main.network.keys(): - main.network[spl[2]].seppuku() # ;( + main.network[spl[2]].seppuku() # ;( del main.network[spl[2]] success("Successfully removed network: %s" % spl[2]) main.saveConf("network") diff --git a/commands/part.py b/commands/part.py index e34d7ff..249c32a 100644 --- a/commands/part.py +++ b/commands/part.py @@ -1,5 +1,6 @@ import main + class PartCommand: def __init__(self, *args): self.part(*args) @@ -13,10 +14,10 @@ class PartCommand: if not int(spl[2]) in main.network[spl[1]].relays.keys(): failure("Relay % does not exist on network %", (spl[2], spl[1])) return - if not spl[1]+spl[2] in main.IRCPool.keys(): - failure("Name has no instance: %s" % spl[1]+spl[2]) + if not spl[1] + spl[2] in main.IRCPool.keys(): + failure("Name has no instance: %s" % spl[1] + spl[2]) return - main.IRCPool[spl[1]+spl[2]].part(spl[3]) + main.IRCPool[spl[1] + spl[2]].part(spl[3]) success("Left %s" % spl[3]) return else: diff --git a/commands/pass.py b/commands/pass.py index fe3f240..e37a51e 100644 --- a/commands/pass.py +++ b/commands/pass.py @@ -1,5 +1,6 @@ import main + class PassCommand: def __init__(self, *args): self.password(*args) diff --git a/commands/pending.py b/commands/pending.py index a245a52..b65f2ad 100644 --- a/commands/pending.py +++ b/commands/pending.py @@ -1,5 +1,6 @@ import main + class PendingCommand: def __init__(self, *args): self.pending(*args) diff --git a/commands/recheckauth.py b/commands/recheckauth.py index 0c5c11a..ab4b5b0 100644 --- a/commands/recheckauth.py +++ b/commands/recheckauth.py @@ -1,5 +1,6 @@ import main + class RecheckauthCommand: def __init__(self, *args): self.recheckauth(*args) diff --git a/commands/reg.py b/commands/reg.py index 27c0150..41bb5f0 100644 --- a/commands/reg.py +++ b/commands/reg.py @@ -1,6 +1,7 @@ import main from modules import regproc + class RegCommand: def __init__(self, *args): self.reg(*args) diff --git a/commands/relay.py b/commands/relay.py index 49dca97..34bbd70 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -1,6 +1,7 @@ import main from yaml import dump + class RelayCommand: def __init__(self, *args): self.relay(*args) diff --git a/commands/save.py b/commands/save.py index d61d1c1..12624cd 100644 --- a/commands/save.py +++ b/commands/save.py @@ -1,5 +1,6 @@ import main + class SaveCommand: def __init__(self, *args): self.save(*args) diff --git a/commands/stats.py b/commands/stats.py index 0db414a..a396f12 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -3,6 +3,7 @@ import modules.counters as count import modules.userinfo as userinfo from string import digits + class StatsCommand: def __init__(self, *args): self.stats(*args) diff --git a/commands/swho.py b/commands/swho.py index 49c71d7..18dbc12 100644 --- a/commands/swho.py +++ b/commands/swho.py @@ -1,5 +1,6 @@ import main + class SwhoCommand: def __init__(self, *args): self.swho(*args) diff --git a/commands/token.py b/commands/token.py index 7e20e5b..7d38eac 100644 --- a/commands/token.py +++ b/commands/token.py @@ -2,6 +2,7 @@ import main from yaml import dump from uuid import uuid4 + class TokenCommand: def __init__(self, *args): self.token(*args) @@ -31,11 +32,12 @@ class TokenCommand: elif length == 4: if spl[1] == "add": if not spl[2] in main.tokens.keys(): - if spl[3] in ["relay"]: # more to come! - main.tokens[spl[2]] = {"hello": str(uuid4()), - "usage": spl[3], - "counter": str(uuid4()), - } + if spl[3] in ["relay"]: # more to come! + main.tokens[spl[2]] = { + "hello": str(uuid4()), + "usage": spl[3], + "counter": str(uuid4()), + } main.saveConf("tokens") success("Successfully created token %s:" % spl[2]) info(dump(main.tokens[spl[2]])) diff --git a/commands/users.py b/commands/users.py index 30da28e..7d80e84 100644 --- a/commands/users.py +++ b/commands/users.py @@ -1,6 +1,7 @@ import main import modules.userinfo as userinfo + class UsersCommand: def __init__(self, *args): self.users(*args) @@ -16,7 +17,7 @@ class UsersCommand: rtrn += "Matches from: %s" % i rtrn += "\n" for x in result[i]: - rtrn += (x) + rtrn += x rtrn += "\n" rtrn += "\n" info(rtrn) diff --git a/commands/who.py b/commands/who.py index f19e462..e80f488 100644 --- a/commands/who.py +++ b/commands/who.py @@ -1,6 +1,7 @@ import main import modules.userinfo as userinfo + class WhoCommand: def __init__(self, *args): self.who(*args) @@ -14,7 +15,7 @@ class WhoCommand: rtrn += "Matches from: %s" % i rtrn += "\n" for x in result[i]: - rtrn += (x) + rtrn += x rtrn += "\n" rtrn += "\n" info(rtrn) diff --git a/core/bot.py b/core/bot.py index bd3743f..b735e96 100644 --- a/core/bot.py +++ b/core/bot.py @@ -3,7 +3,12 @@ from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall from twisted.internet import reactor, task -from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage +from twisted.words.protocols.irc import ( + symbolic_to_numeric, + numeric_to_symbolic, + lowDequote, + IRCBadMessage, +) import sys from string import digits @@ -29,16 +34,15 @@ from utils.parsing import parsen from twisted.internet.ssl import DefaultOpenSSLContextFactory + def deliverRelayCommands(num, relayCommands, user=None, stage2=None): - keyFN = main.certPath+main.config["Key"] - certFN = main.certPath+main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), - certFN.encode("utf-8", "replace")) + keyFN = main.certPath + main.config["Key"] + certFN = main.certPath + main.config["Certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) host, port = getRelay(num) - rct = reactor.connectSSL(host, - port, - bot, contextFactory) + rct = reactor.connectSSL(host, port, bot, contextFactory) + # Copied from the Twisted source so we can fix a bug def parsemsg(s): @@ -50,21 +54,22 @@ def parsemsg(s): @return: A tuple of (prefix, command, args). @rtype: L{tuple} """ - prefix = '' + prefix = "" trailing = [] if not s: raise IRCBadMessage("Empty line.") - if s[0:1] == ':': - prefix, s = s[1:].split(' ', 1) - if s.find(' :') != -1: - s, trailing = s.split(' :', 1) - args = s.split(' ') # Twisted bug fixed by adding an argument to split() + if s[0:1] == ":": + prefix, s = s[1:].split(" ", 1) + if s.find(" :") != -1: + s, trailing = s.split(" :", 1) + args = s.split(" ") # Twisted bug fixed by adding an argument to split() args.append(trailing) else: - args = s.split(' ') # And again + args = s.split(" ") # And again command = args.pop(0) return prefix, command, args + class IRCRelay(IRCClient): def __init__(self, num, relayCommands, user, stage2): self.isconnected = False @@ -77,7 +82,7 @@ class IRCRelay(IRCClient): self.nickname = "relay" self.realname = "relay" self.username = self.user - self.password = self.user+":"+password + self.password = self.user + ":" + password self.relayCommands = relayCommands self.num = num @@ -109,19 +114,25 @@ class IRCRelay(IRCClient): def signedOn(self): if not self.isconnected: self.isconnected = True - #log("signed on as a relay: %s" % self.num) + # log("signed on as a relay: %s" % self.num) sleeptime = 0 increment = 0.8 for i in self.relayCommands.keys(): for x in self.relayCommands[i]: - reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) + reactor.callLater( + sleeptime, + self.msg, + main.config["Tweaks"]["ZNC"]["Prefix"] + i, + x, + ) sleeptime += increment increment += 0.8 if not self.stage2 == None: reactor.callLater(sleeptime, self.sendStage2) - reactor.callLater(sleeptime+5, self.transport.loseConnection) + reactor.callLater(sleeptime + 5, self.transport.loseConnection) return + class IRCBot(IRCClient): def __init__(self, net, num): self.isconnected = False @@ -135,26 +146,26 @@ class IRCBot(IRCClient): relay = main.network[self.net].relays[num] self.nickname = alias["nick"] self.realname = alias["realname"] - self.username = alias["nick"].lower()+"/"+relay["net"] + self.username = alias["nick"].lower() + "/" + relay["net"] self.password = main.config["Relay"]["Password"] - self.userinfo = None # - self.fingerReply = None # - self.versionName = None # Don't give out information - self.versionNum = None # - self.versionEnv = None # - self.sourceURL = None # + self.userinfo = None # + self.fingerReply = None # + self.versionName = None # Don't give out information + self.versionNum = None # + self.versionEnv = None # + self.sourceURL = None # - self._getWho = {} # LoopingCall objects -- needed to be able to stop them + self._getWho = {} # LoopingCall objects -- needed to be able to stop them - self._tempWho = {} # temporary storage for gathering WHO info - self._tempNames = {} # temporary storage for gathering NAMES info - self._tempList = ([], []) # temporary storage for gathering LIST info - self.listOngoing = False # we are currently receiving a LIST - self.listRetried = False # we asked and got nothing so asked again - self.listAttempted = False # we asked for a list - self.listSimple = False # after asking again we got the list, so use the simple - # syntax from now on - self.wantList = False # we want to send a LIST, but not all relays are active yet + self._tempWho = {} # temporary storage for gathering WHO info + self._tempNames = {} # temporary storage for gathering NAMES info + self._tempList = ([], []) # temporary storage for gathering LIST info + self.listOngoing = False # we are currently receiving a LIST + self.listRetried = False # we asked and got nothing so asked again + self.listAttempted = False # we asked for a list + self.listSimple = False # after asking again we got the list, so use the simple + # syntax from now on + self.wantList = False # we want to send a LIST, but not all relays are active yet self.chanlimit = 0 self.prefix = {} self.servername = None @@ -214,8 +225,8 @@ class IRCBot(IRCClient): cast["time"] = str(datetime.now().isoformat()) # remove odd stuff - for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over - if cast[i] == "": # a dictionary that changes length with each iteration + for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over + if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] # remove server stuff if "muser" in cast.keys(): @@ -259,11 +270,11 @@ class IRCBot(IRCClient): if "channel" in cast.keys(): if cast["type"] == "mode": if cast["channel"].lower() == self.nickname.lower(): - #castDup = deepcopy(cast) + # castDup = deepcopy(cast) cast["mtype"] = cast["type"] cast["type"] = "self" - #self.event(**castDup) - if cast["modearg"]: # check if modearg is non-NoneType + # self.event(**castDup) + if cast["modearg"]: # check if modearg is non-NoneType if self.nickname.lower() == cast["modearg"].lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] @@ -273,7 +284,7 @@ class IRCBot(IRCClient): if cast["channel"].lower() == self.nickname.lower(): cast["mtype"] = cast["type"] cast["type"] = "query" - #self.event(**castDup) + # self.event(**castDup) # Don't call self.event for this one because queries are not events on a # channel, but we still want to see them @@ -296,7 +307,7 @@ class IRCBot(IRCClient): castDup["type"] = "self" # we have been mentioned in a msg/notice/action/part/quit/topic message - if "msg" in cast.keys(): # Don't highlight queries + if "msg" in cast.keys(): # Don't highlight queries if not cast["msg"] == None: if self.nickname.lower() in cast["msg"].lower(): castDup = deepcopy(cast) @@ -373,7 +384,16 @@ class IRCBot(IRCClient): return n = self._tempWho[channel][1] n.append([nick, nick, host, server, status, realname]) - self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status) + self.event( + type="who", + nick=nick, + ident=ident, + host=host, + realname=realname, + channel=channel, + server=server, + status=status, + ) def irc_RPL_ENDOFWHO(self, prefix, params): channel = params[1] @@ -390,7 +410,10 @@ class IRCBot(IRCClient): def sanit(self, data): if len(data) >= 1: if data[0] in self.prefix.keys(): - return (self.prefix[data[0]], data[1:]) # would use a set but it's possible these are the same + return ( + self.prefix[data[0]], + data[1:], + ) # would use a set but it's possible these are the same return (None, data) else: return (None, False) @@ -405,7 +428,7 @@ class IRCBot(IRCClient): def irc_RPL_NAMREPLY(self, prefix, params): channel = params[2] - nicklist = params[3].split(' ') + nicklist = params[3].split(" ") if channel not in self._tempNames: return n = self._tempNames[channel][1] @@ -438,7 +461,7 @@ class IRCBot(IRCClient): self._tempList[0].append(d) if self.listSimple: self.sendLine("LIST") - return d # return early if we know what to do + return d # return early if we know what to do if noargs: self.sendLine("LIST") @@ -460,8 +483,8 @@ class IRCBot(IRCClient): return else: if nocheck: - allRelays = True # override the system - if this is - else: # specified, we already did this + allRelays = True # override the system - if this is + else: # specified, we already did this allRelays = chankeep.allRelaysActive(self.net) if not allRelays: self.wantList = True @@ -481,7 +504,7 @@ class IRCBot(IRCClient): self._tempList[1].append([channel, users, topic]) def irc_RPL_LISTEND(self, prefix, params): - if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there + if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there debug("We didn't ask for this LIST, discarding") self._tempList[0].clear() self._tempList[1].clear() @@ -510,18 +533,18 @@ class IRCBot(IRCClient): self.listSimple = True def got_list(self, listinfo): - if len(listinfo) == 0: # probably ngircd not supporting LIST >0 + if len(listinfo) == 0: # probably ngircd not supporting LIST >0 return chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) def recheckList(self): allRelays = chankeep.allRelaysActive(self.net) if allRelays: - name = self.net+"1" + name = self.net + "1" if main.IRCPool[name].wantList == True: main.IRCPool[name].list(nocheck=True) debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) - if self.num == 1: # Only one instance should do a list + if self.num == 1: # Only one instance should do a list if self.chanlimit: if allRelays: self.list() @@ -532,24 +555,26 @@ class IRCBot(IRCClient): self.checkChannels() def seed_chanlimit(self, chanlimit): - if not main.network[self.net].relays[self.num]["registered"]: #TODO: add check for register request sent, only send it once + if not main.network[self.net].relays[self.num][ + "registered" + ]: # TODO: add check for register request sent, only send it once if main.config["AutoReg"]: self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) - #regproc.registerAccount(self.net, self.num) + # regproc.registerAccount(self.net, self.num) try: self.chanlimit = int(chanlimit) except TypeError: warn("Invalid chanlimit: %s" % i) if self.chanlimit == 0: - self.chanlimit = 200 # don't take the piss if it's not limited - if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation + self.chanlimit = 200 # don't take the piss if it's not limited + if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation self.recheckList() def seed_prefix(self, prefix): prefix = prefix.replace(")", "") prefix = prefix.replace("(", "") length = len(prefix) - half = int(length/2) + half = int(length / 2) prefixToMode = dict(zip(prefix[half:], prefix[:half])) self.prefix = prefixToMode @@ -585,7 +610,7 @@ class IRCBot(IRCClient): # the hostname and other useful information in the functions # that these call by default def irc_JOIN(self, prefix, params): - nick = prefix.split('!')[0] + nick = prefix.split("!")[0] channel = params[-1] if nick == self.nickname: self.joined(channel) @@ -593,7 +618,7 @@ class IRCBot(IRCClient): self.userJoined(prefix, channel) def irc_PART(self, prefix, params): - nick = prefix.split('!')[0] + nick = prefix.split("!")[0] channel = params[0] if len(params) >= 2: message = params[1] @@ -605,11 +630,11 @@ class IRCBot(IRCClient): self.userLeft(prefix, channel, message) def irc_QUIT(self, prefix, params): - nick = prefix.split('!')[0] + nick = prefix.split("!")[0] self.userQuit(prefix, params[0]) def irc_NICK(self, prefix, params): - nick = prefix.split('!', 1)[0] + nick = prefix.split("!", 1)[0] if nick == self.nickname: self.nickChanged(prefix, params[0]) else: @@ -626,6 +651,7 @@ class IRCBot(IRCClient): channel = params[0] newtopic = params[1] self.topicUpdated(prefix, channel, newtopic) + # End of Twisted hackery def regPing(self, negativepass=None): @@ -664,7 +690,15 @@ class IRCBot(IRCClient): def signedOn(self): log("signed on: %s - %i" % (self.net, self.num)) ctime = str(datetime.now().isoformat()) - sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon", "time": ctime}) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "signedon", + "time": ctime, + } + ) if not self.authenticated: reactor.callLater(10, self.regPing) @@ -677,7 +711,7 @@ class IRCBot(IRCClient): self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] minint = main.config["Tweaks"]["Delays"]["WhoLoop"] - interval = randint(minint, minint+intrange) + interval = randint(minint, minint + intrange) lc.start(interval) def botLeft(self, channel): @@ -687,10 +721,11 @@ class IRCBot(IRCClient): lc = self._getWho[channel] lc.stop() del self._getWho[channel] - userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this - #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- - # other bots have different nicknames so - def left(self, user, channel, message): # even if they saw it, they wouldn't react + userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this + # log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- + # other bots have different nicknames so + + def left(self, user, channel, message): # even if they saw it, they wouldn't react self.event(type="part", muser=user, channel=channel, msg=message) self.botLeft(channel) @@ -711,14 +746,14 @@ class IRCBot(IRCClient): def chanlessEvent(self, cast): cast["time"] = str(datetime.now().isoformat()) cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) - if dedup(self.name, cast): # Needs to be kept self.name until the dedup - # function is converted to the new net, num - # format - return # stop right there sir! + if dedup(self.name, cast): # Needs to be kept self.name until the dedup + # function is converted to the new net, num + # format + return # stop right there sir! chans = userinfo.getChanList(self.net, cast["nick"]) if chans == None: error("No channels returned for chanless event: %s" % cast) - # self.event(**cast) -- no, should NEVER happen + # self.event(**cast) -- no, should NEVER happen return # getChansSingle returns all channels of the user, we only want to use # ones we have common with them @@ -737,7 +772,15 @@ class IRCBot(IRCClient): argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", muser=user, channel=channel, mode=m, status=toset, modearg=a) + self.event( + type="mode", + muser=user, + channel=channel, + mode=m, + status=toset, + modearg=a, + ) + class IRCBotFactory(ReconnectingClientFactory): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): @@ -747,7 +790,7 @@ class IRCBotFactory(ReconnectingClientFactory): self.name = "relay - %i" % num self.relay = True else: - self.name = net+str(num) + self.name = net + str(num) self.num = num self.net = net self.relay = False @@ -781,9 +824,18 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s - %i: connection lost: %s" % (self.net, self.num, error)) sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) ctime = str(datetime.now().isoformat()) - sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error, "time": ctime}) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "lost", + "message": error, + "time": ctime, + } + ) self.retry(connector) - #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) + # ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): if not self.client == None: @@ -795,7 +847,15 @@ class IRCBotFactory(ReconnectingClientFactory): if not self.relay: sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) ctime = str(datetime.now().isoformat()) - sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error, "time": ctime}) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "failed", + "message": error, + "time": ctime, + } + ) self.retry(connector) - #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) - + # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/logstash.py b/core/logstash.py index aa5924b..17e8dfd 100644 --- a/core/logstash.py +++ b/core/logstash.py @@ -6,11 +6,20 @@ import main from utils.logging.log import * logger = None + + def init_logstash(): global logger - logger = logging.getLogger('ingest') + logger = logging.getLogger("ingest") logger.setLevel(logging.INFO) - logger.addHandler(logstash.TCPLogstashHandler(main.config["Logstash"]["Host"], int(main.config["Logstash"]["Port"]), version=1)) + logger.addHandler( + logstash.TCPLogstashHandler( + main.config["Logstash"]["Host"], + int(main.config["Logstash"]["Port"]), + version=1, + ) + ) + def sendLogstashNotification(text): if not logger == None: diff --git a/core/parser.py b/core/parser.py index 64270f8..1eb34c5 100644 --- a/core/parser.py +++ b/core/parser.py @@ -2,8 +2,9 @@ import main from utils.logging.log import * from utils.logging.send import * + def parseCommand(addr, authed, data): - #call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length) + # call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length) spl = data.split() if addr in main.connections.keys(): obj = main.connections[addr] diff --git a/core/relay.py b/core/relay.py index 7c7e020..66a9830 100644 --- a/core/relay.py +++ b/core/relay.py @@ -5,7 +5,25 @@ from copy import deepcopy import main from utils.logging.log import * -validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight"] +validTypes = [ + "msg", + "notice", + "action", + "who", + "part", + "join", + "kick", + "quit", + "nick", + "topic", + "mode", + "conn", + "znc", + "query", + "self", + "highlight", +] + class Relay(Protocol): def __init__(self, addr): @@ -108,7 +126,7 @@ class Relay(Protocol): def connectionMade(self): log("Relay connection from %s:%s" % (self.addr.host, self.addr.port)) - #self.send("Greetings.") + # self.send("Greetings.") def connectionLost(self, reason): self.authed = False @@ -118,6 +136,7 @@ class Relay(Protocol): else: warn("Tried to remove a non-existant relay connection.") + class RelayFactory(Factory): def buildProtocol(self, addr): entry = Relay(addr) @@ -131,6 +150,7 @@ class RelayFactory(Factory): else: return + def sendRelayNotification(cast): for i in main.relayConnections.keys(): if main.relayConnections[i].authed: diff --git a/core/server.py b/core/server.py index e3f125a..94af7a0 100644 --- a/core/server.py +++ b/core/server.py @@ -4,6 +4,7 @@ from utils.logging.log import * from core.parser import parseCommand + class Server(Protocol): def __init__(self, addr): self.addr = addr @@ -18,7 +19,7 @@ class Server(Protocol): def dataReceived(self, data): data = data.decode("utf-8", "replace") - #log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) + # log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) if "\n" in data: splitData = [x for x in data.split("\n") if x] if "\n" in data: @@ -39,6 +40,7 @@ class Server(Protocol): else: warn("Tried to remove a non-existant connection.") + class ServerFactory(Factory): def buildProtocol(self, addr): entry = Server(addr) diff --git a/main.py b/main.py index caebef7..a29e1d7 100644 --- a/main.py +++ b/main.py @@ -7,10 +7,7 @@ from os import urandom from utils.logging.log import * # List of errors ZNC can give us -ZNCErrors = ["Error:", - "Unable to load", - "does not exist", - "doesn't exist"] +ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] configPath = "conf/" certPath = "cert/" @@ -25,9 +22,8 @@ filemap = { "alias": ["alias.json", "provisioned alias data", "json"], "irc": ["irc.json", "IRC network definitions", "json"], "blacklist": ["blacklist.json", "IRC channel blacklist", "json"], - # Binary (pickle) configs - "network": ["network.dat", "network list", "pickle"] + "network": ["network.dat", "network list", "pickle"], } # Connections to the plain-text interface @@ -70,43 +66,45 @@ def liveNets(): networks.add("".join([x for x in i if not x in digits])) return networks + def saveConf(var): if filemap[var][2] == "json": - with open(configPath+filemap[var][0], "w") as f: + with open(configPath + filemap[var][0], "w") as f: json.dump(globals()[var], f, indent=4) elif filemap[var][2] == "pickle": - with open(configPath+filemap[var][0], "wb") as f: + with open(configPath + filemap[var][0], "wb") as f: pickle.dump(globals()[var], f) else: raise Exception("invalid format") + def loadConf(var): if filemap[var][2] == "json": - with open(configPath+filemap[var][0], "r") as f: + with open(configPath + filemap[var][0], "r") as f: globals()[var] = json.load(f) if var == "alias": # This is a workaround to convert all the keys into integers since JSON # turns them into strings... # Dammit Jason! global alias - alias = {int(x):y for x, y in alias.items()} + alias = {int(x): y for x, y in alias.items()} elif filemap[var][2] == "pickle": try: - with open(configPath+filemap[var][0], "rb") as f: + with open(configPath + filemap[var][0], "rb") as f: globals()[var] = pickle.load(f) except FileNotFoundError: globals()[var] = {} else: raise Exception("invalid format") + def initConf(): for i in filemap.keys(): loadConf(i) + def initMain(): global r, g initConf() - r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit - g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent - - + r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit + g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent diff --git a/modules/alias.py b/modules/alias.py index 59ef99a..1961ec6 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -2,9 +2,11 @@ import main import random import re + def generate_password(): return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) + def generate_alias(): nick = random.choice(main.aliasdata["stubs"]) rand = random.randint(1, 2) @@ -12,8 +14,8 @@ def generate_alias(): nick = nick.capitalize() rand = random.randint(1, 4) while rand == 1: - split = random.randint(0, len(nick)-1) - nick = nick[:split] + nick[split+1:] + split = random.randint(0, len(nick) - 1) + nick = nick[:split] + nick[split + 1 :] rand = random.randint(1, 4) rand = random.randint(1, 3) if rand == 1 or rand == 4: @@ -63,4 +65,10 @@ def generate_alias(): if rand == 3 or rand == 4: realname = realname.capitalize() - return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "emails": []} + return { + "nick": nick, + "altnick": altnick, + "ident": ident, + "realname": realname, + "emails": [], + } diff --git a/modules/chankeep.py b/modules/chankeep.py index 1cc70c0..ec43b0f 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -6,11 +6,12 @@ from math import ceil import modules.provision from twisted.internet.threads import deferToThread + def allRelaysActive(net): relayNum = len(main.network[net].relays.keys()) existNum = 0 for i in main.network[net].relays.keys(): - name = net+str(i) + name = net + str(i) if name in main.IRCPool.keys(): if main.IRCPool[name].authenticated: existNum += 1 @@ -18,20 +19,22 @@ def allRelaysActive(net): return True return False + def getChanFree(net, new): chanfree = {} chanlimits = set() for i in main.network[net].relays.keys(): if i in new: continue - name = net+str(i) - chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels) + name = net + str(i) + chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels) chanlimits.add(main.IRCPool[name].chanlimit) if not len(chanlimits) == 1: error("Network %s has servers with different CHANLIMIT values" % net) return False return (chanfree, chanlimits.pop()) + def emptyChanAllocate(net, flist, relay, new): chanfree = getChanFree(net, new) if not chanfree: @@ -41,10 +44,10 @@ def emptyChanAllocate(net, flist, relay, new): allocated = {} toalloc = len(flist) if toalloc > sum(chanfree[0].values()): - correction = round(toalloc-sum(chanfree[0].values()) / chanfree[1]) - #print("correction", correction) + correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) + # print("correction", correction) warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) - #newNums = modules.provision.provisionMultipleRelays(net, correction) + # newNums = modules.provision.provisionMultipleRelays(net, correction) return False for i in chanfree[0].keys(): for x in range(chanfree[0][i]): @@ -56,8 +59,9 @@ def emptyChanAllocate(net, flist, relay, new): allocated[i] = [flist.pop()] return allocated + def populateChans(net, clist, relay, new): - #divided = array_split(clist, relay) + # divided = array_split(clist, relay) allocated = emptyChanAllocate(net, clist, relay, new) if not allocated: return @@ -67,18 +71,20 @@ def populateChans(net, clist, relay, new): else: main.TempChan[net] = {i: allocated[i]} + def notifyJoin(net): for i in main.network[net].relays.keys(): - name = net+str(i) + name = net + str(i) if name in main.IRCPool.keys(): main.IRCPool[name].checkChannels() + def minifyChans(net, listinfo): if not allRelaysActive(net): error("All relays for %s are not active, cannot minify list" % net) return False for i in main.network[net].relays.keys(): - name = net+str(i) + name = net + str(i) for x in main.IRCPool[name].channels: for y in listinfo: if y[0] == x: @@ -88,29 +94,31 @@ def minifyChans(net, listinfo): return False return listinfo + def keepChannels(net, listinfo, mean, sigrelay, relay): listinfo = minifyChans(net, listinfo) if not listinfo: return - if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels + if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels coverAll = True - elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels + elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels coverAll = False if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) return if coverAll: - needed = relay-len(main.network[net].relays.keys()) + needed = relay - len(main.network[net].relays.keys()) newNums = modules.provision.provisionMultipleRelays(net, needed) flist = [i[0] for i in listinfo] populateChans(net, flist, relay, newNums) else: - needed = sigrelay-len(main.network[net].relays.keys()) + needed = sigrelay - len(main.network[net].relays.keys()) newNums = modules.provision.provisionMultipleRelays(net, needed) siglist = [i[0] for i in listinfo if int(i[1]) > mean] populateChans(net, siglist, sigrelay, newNums) notifyJoin(net) + def joinSingle(net, channel): if allRelaysActive(net): chanfree = getChanFree(net, []) @@ -122,14 +130,16 @@ def joinSingle(net, channel): error("All relays for %s are not active" % net) return False -def nukeNetwork(net): - #purgeRecords(net) - #p = main.g.pipeline() - main.g.delete("analytics.list."+net) - #p.delete("list."+net) - #p.execute() -#def nukeNetwork(net): +def nukeNetwork(net): + # purgeRecords(net) + # p = main.g.pipeline() + main.g.delete("analytics.list." + net) + # p.delete("list."+net) + # p.execute() + + +# def nukeNetwork(net): # deferToThread(_nukeNetwork, net) @@ -141,7 +151,7 @@ def _initialList(net, num, listinfo, chanlimit): except TypeError: warn("Bad LIST data received from %s - %i" % (net, num)) return - mean = round(cumul/listlength, 2) + mean = round(cumul / listlength, 2) siglength = 0 insiglength = 0 sigcumul = 0 @@ -154,8 +164,8 @@ def _initialList(net, num, listinfo, chanlimit): insiglength += 1 insigcumul += int(i[1]) - sigrelay = ceil(siglength/chanlimit) - relay = ceil(listlength/chanlimit) + sigrelay = ceil(siglength / chanlimit) + relay = ceil(listlength / chanlimit) netbase = "list.%s" % net abase = "analytics.list.%s" % net p = main.g.pipeline() @@ -163,18 +173,18 @@ def _initialList(net, num, listinfo, chanlimit): p.hset(abase, "total", listlength) p.hset(abase, "sigtotal", siglength) p.hset(abase, "insigtotal", insiglength) - p.hset(abase, "sigperc", round(siglength/listlength*100, 2)) - p.hset(abase, "insigperc", round(insiglength/listlength*100, 2)) + p.hset(abase, "sigperc", round(siglength / listlength * 100, 2)) + p.hset(abase, "insigperc", round(insiglength / listlength * 100, 2)) p.hset(abase, "cumul", cumul) p.hset(abase, "sigcumul", sigcumul) p.hset(abase, "insigcumul", insigcumul) p.hset(abase, "relay", relay) p.hset(abase, "sigrelay", sigrelay) - p.hset(abase, "insigrelay", ceil(insiglength/chanlimit)) + p.hset(abase, "insigrelay", ceil(insiglength / chanlimit)) # Purge existing records before writing - #purgeRecords(net) - #for i in listinfo: + # purgeRecords(net) + # for i in listinfo: # p.rpush(netbase+"."+i[0], i[1]) # p.rpush(netbase+"."+i[0], i[2]) # p.sadd(netbase, i[0]) @@ -183,6 +193,6 @@ def _initialList(net, num, listinfo, chanlimit): debug("List parsing completed on %s" % net) keepChannels(net, listinfo, mean, sigrelay, relay) + def initialList(net, num, listinfo, chanlimit): deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) - diff --git a/modules/counters.py b/modules/counters.py index bb2b518..a088cc3 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -1,6 +1,7 @@ import main from twisted.internet.task import LoopingCall + def event(name, eventType): if not "local" in main.counters.keys(): main.counters["local"] = {} @@ -18,6 +19,7 @@ def event(name, eventType): main.counters["global"][eventType] += 1 main.runningSample += 1 + def getEvents(name=None): if name == None: if "global" in main.counters.keys(): @@ -30,10 +32,12 @@ def getEvents(name=None): else: return None + def takeSample(): main.lastMinuteSample = main.runningSample main.runningSample = 0 + def setupCounterLoop(): lc = LoopingCall(takeSample) lc.start(60) diff --git a/modules/monitor.py b/modules/monitor.py index 2dbfc2e..8c4681d 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -8,9 +8,25 @@ from modules import userinfo from modules import regproc from utils.dedup import dedup -order = ["type", "net", "num", "channel", "msg", "nick", - "ident", "host", "mtype", "user", "mode", "modearg", - "realname", "server", "status", "time"] +order = [ + "type", + "net", + "num", + "channel", + "msg", + "nick", + "ident", + "host", + "mtype", + "user", + "mode", + "modearg", + "realname", + "server", + "status", + "time", +] + def parsemeta(numName, c): if not "channel" in c.keys(): @@ -21,10 +37,16 @@ def parsemeta(numName, c): if c["type"] in ["msg", "notice", "action", "topic", "mode"]: if "muser" in c.keys(): userinfo.editUser(c["net"], c["muser"]) - #if c["type"] == "mode": + # if c["type"] == "mode": # userinfo.updateMode(c) elif c["type"] == "nick": - userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + userinfo.renameUser( + c["net"], + c["nick"], + c["muser"], + c["user"], + c["user"] + "!" + c["ident"] + "@" + c["host"], + ) elif c["type"] == "kick": userinfo.editUser(c["net"], c["muser"]) userinfo.delUserByNick(c["net"], c["channel"], c["user"]) @@ -37,9 +59,16 @@ def parsemeta(numName, c): if "mtype" in c.keys(): if c["mtype"] == "nick": - userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + userinfo.renameUser( + c["net"], + c["nick"], + c["muser"], + c["user"], + c["user"] + "!" + c["ident"] + "@" + c["host"], + ) -def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen + +def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen if dedup(numName, c): return @@ -48,7 +77,7 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe if "muser" in c.keys(): del c["muser"] - sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order + sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order sortedKeys["src"] = "irc" sendLogstashNotification(sortedKeys) sendRelayNotification(sortedKeys) diff --git a/modules/network.py b/modules/network.py index 8cff3db..21f1753 100644 --- a/modules/network.py +++ b/modules/network.py @@ -10,6 +10,7 @@ import main from utils.logging.log import * from utils.get import getRelay + class Network: def __init__(self, net, host, port, security, auth): self.net = net @@ -37,20 +38,20 @@ class Network: "enabled": main.config["ConnectOnCreate"], "net": self.net, "id": num, - "registered": registered + "registered": registered, } password = alias.generate_password() if not num in main.alias.keys(): main.alias[num] = alias.generate_alias() main.saveConf("alias") self.aliases[num] = {"password": password} - #if main.config["ConnectOnCreate"]: -- Done in provision + # if main.config["ConnectOnCreate"]: -- Done in provision # self.start_bot(num) return num, main.alias[num]["nick"] def killAliases(self, aliasList): for i in aliasList: - name = self.net+str(i) + name = self.net + str(i) if name in main.ReactorPool.keys(): if name in main.FactoryPool.keys(): main.FactoryPool[name].stopTrying() @@ -63,7 +64,7 @@ class Network: def delete_relay(self, id): del self.relays[id] del self.aliases[id] - #del main.alias[id] - Aliases are global per num, so don't delete them! + # del main.alias[id] - Aliases are global per num, so don't delete them! self.killAliases([id]) def seppuku(self): @@ -74,11 +75,11 @@ class Network: def start_bot(self, num): # a single name is given to relays in the backend # e.g. freenode1 for the first relay on freenode network - keyFN = main.certPath+main.config["Key"] - certFN = main.certPath+main.config["Certificate"] + keyFN = main.certPath + main.config["Key"] + certFN = main.certPath + main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) bot = IRCBotFactory(self.net, num) - #host, port = self.relays[num]["host"], self.relays[num]["port"] + # host, port = self.relays[num]["host"], self.relays[num]["port"] host, port = getRelay(num) rct = reactor.connectSSL(host, port, bot, contextFactory) name = self.net + str(num) diff --git a/modules/provision.py b/modules/provision.py index 69fe99e..990db63 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -4,6 +4,7 @@ from utils.logging.log import * from twisted.internet import reactor import modules.regproc + def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security, auth, password): commands = {} stage2commands = {} @@ -17,7 +18,7 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, commands["controlpanel"].append("Set Ident %s %s" % (user, ident)) commands["controlpanel"].append("Set RealName %s %s" % (user, realname)) if security == "ssl": - commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me + commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port)) elif security == "plain": commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port)) @@ -29,9 +30,9 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, inst = modules.regproc.selectInst(network) if "setmode" in inst.keys(): stage2commands["status"].append("LoadMod perform") - #stage2commands["perform"].append("add mode %nick% +"+inst["setmode"]) - deliverRelayCommands(num, commands, - stage2=[[user+"/"+network, stage2commands]]) + # stage2commands["perform"].append("add mode %nick% +"+inst["setmode"]) + deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]]) + def provisionAuthenticationData(num, nick, network, security, auth, password): commands = {} @@ -48,23 +49,28 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): commands["nickserv"].append("Set %s" % password) inst = modules.regproc.selectInst(network) if "setmode" in inst.keys(): - #commands["status"].append("LoadMod perform") - commands["perform"] = ["add mode %nick% +"+inst["setmode"]] - deliverRelayCommands(num, commands, user=user+"/"+network) + # commands["status"].append("LoadMod perform") + commands["perform"] = ["add mode %nick% +" + inst["setmode"]] + deliverRelayCommands(num, commands, user=user + "/" + network) -def provisionRelay(num, network): # provision user and network data +def provisionRelay(num, network): # provision user and network data aliasObj = main.alias[num] alias = aliasObj["nick"] - provisionUserNetworkData(num, *aliasObj.values(), network, - main.network[network].host, - main.network[network].port, - main.network[network].security, - main.network[network].auth, - main.network[network].aliases[num]["password"]) + provisionUserNetworkData( + num, + *aliasObj.values(), + network, + main.network[network].host, + main.network[network].port, + main.network[network].security, + main.network[network].auth, + main.network[network].aliases[num]["password"] + ) if main.config["ConnectOnCreate"]: reactor.callLater(10, main.network[network].start_bot, num) + def provisionMultipleRelays(net, relaysNeeded): numsProvisioned = [] for i in range(relaysNeeded): @@ -73,4 +79,3 @@ def provisionMultipleRelays(net, relaysNeeded): provisionRelay(num, net) main.saveConf("network") return numsProvisioned - diff --git a/modules/regproc.py b/modules/regproc.py index 97c330a..26cdae2 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -4,6 +4,7 @@ from utils.logging.log import * from utils.logging.debug import * from copy import deepcopy + def needToRegister(net): inst = selectInst(net) if "register" in inst.keys(): @@ -12,6 +13,7 @@ def needToRegister(net): else: return False + def selectInst(net): if net in main.irc.keys(): inst = deepcopy(main.irc[net]) @@ -22,11 +24,12 @@ def selectInst(net): inst = main.irc["_"] return inst + def substitute(net, num, token=None): inst = selectInst(net) alias = main.alias[num] nickname = alias["nick"] - username = nickname+"/"+net + username = nickname + "/" + net password = main.network[net].aliases[num]["password"] inst["email"] = inst["email"].replace("{nickname}", nickname) for i in inst.keys(): @@ -39,29 +42,32 @@ def substitute(net, num, token=None): inst[i] = inst[i].replace("{token}", token) return inst + def registerAccount(net, num): debug("Attempting to register: %s - %i" % (net, num)) sinst = substitute(net, num) if not sinst["register"]: error("Cannot register for %s: function disabled" % (net)) return False - name = net+str(num) + name = net + str(num) main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"]) + def confirmAccount(net, num, token): sinst = substitute(net, num, token=token) - name = net+str(num) + name = net + str(num) main.IRCPool[name].msg(sinst["entity"], sinst["confirm"]) enableAuthentication(net, num) + def confirmRegistration(net, num, negativepass=None): obj = main.network[net] - name = net+str(num) + name = net + str(num) if name in main.IRCPool.keys(): if not negativepass == None: main.IRCPool[name].regPing(negativepass=negativepass) return - debug("Relay authenticated: %s - %i" %(net, num)) + debug("Relay authenticated: %s - %i" % (net, num)) main.IRCPool[name].authenticated = True main.IRCPool[name].recheckList() if obj.relays[num]["registered"]: @@ -75,21 +81,23 @@ def confirmRegistration(net, num, negativepass=None): obj.relays[num]["registered"] = True main.saveConf("network") + def enableAuthentication(net, num): obj = main.network[net] nick = main.alias[num]["nick"] security = obj.security auth = obj.auth password = obj.aliases[num]["password"] - uname = main.alias[num]["nick"]+"/"+net - provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth - main.IRCPool[net+str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"]+"status", "Jump") + uname = main.alias[num]["nick"] + "/" + net + provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth + main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") if selectInst(net)["check"] == False: confirmRegistration(net, num) + def registerTest(c): sinst = substitute(c["net"], c["num"]) - name = c["net"]+str(c["num"]) + name = c["net"] + str(c["num"]) if sinst["check"] == False: return if "msg" in c.keys() and not c["msg"] == None: @@ -98,19 +106,19 @@ def registerTest(c): if not main.IRCPool[name]._negativePass == True: if c["type"] == "query" and c["nick"] == sinst["entity"]: if sinst["checknegativemsg"] in c["msg"]: - confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back + confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back return if sinst["checkendnegative"] in c["msg"]: - confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back + confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: confirmRegistration(c["net"], c["num"]) return if sinst["checktype"] == "msg": - if sinst["checkmsg"] in c["msg"]: - confirmRegistration(c["net"], c["num"]) - return + if sinst["checkmsg"] in c["msg"]: + confirmRegistration(c["net"], c["num"]) + return elif sinst["checktype"] == "mode": if c["type"] == "self": if c["mtype"] == "mode": diff --git a/modules/userinfo.py b/modules/userinfo.py index a9104da..1722cb3 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -6,12 +6,14 @@ from utils.logging.log import * from utils.logging.debug import debug, trace from utils.parsing import parsen + def getWhoSingle(name, query): - result = main.r.sscan("live.who."+name, 0, query, count=999999) + result = main.r.sscan("live.who." + name, 0, query, count=999999) if result[1] == []: return None return (i.decode() for i in result[1]) + def getWho(query): result = {} for i in main.network.keys(): @@ -20,20 +22,23 @@ def getWho(query): result[i] = f return result + def getChansSingle(name, nick): - nick = ["live.chan."+name+"."+i for i in nick] + nick = ["live.chan." + name + "." + i for i in nick] result = main.r.sinter(*nick) if len(result) == 0: return None return (i.decode() for i in result) + def getChanList(name, nick): - chanspace = "live.chan."+name+"."+nick + chanspace = "live.chan." + name + "." + nick result = main.r.smembers(chanspace) if len(result) == 0: return None return (i.decode() for i in result) + def getChans(nick): result = {} for i in main.network.keys(): @@ -42,13 +47,15 @@ def getChans(nick): result[i] = f return result + def getUsersSingle(name, nick): - nick = ("live.who."+name+"."+i for i in nick) + nick = ("live.who." + name + "." + i for i in nick) result = main.r.sinter(*nick) if len(result) == 0: return None return (i.decode() for i in result) + def getUsers(nick): result = {} for i in main.network.keys(): @@ -57,8 +64,10 @@ def getUsers(nick): result[i] = f return result + def getNumWhoEntries(name): - return main.r.scard("live.who."+name) + return main.r.scard("live.who." + name) + def getNumTotalWhoEntries(): total = 0 @@ -66,6 +75,7 @@ def getNumTotalWhoEntries(): total += getNumWhoEntries(i) return total + def getNamespace(name, channel, nick): gnamespace = "live.who.%s" % name namespace = "live.who.%s.%s" % (name, channel) @@ -73,35 +83,40 @@ def getNamespace(name, channel, nick): mapspace = "live.map.%s" % name return (gnamespace, namespace, chanspace, mapspace) + def _initialUsers(name, channel, users): gnamespace = "live.who.%s" % name mapspace = "live.map.%s" % name p = main.r.pipeline() for i in users: - user = i[0]+"!"+i[1]+"@"+i[2] + user = i[0] + "!" + i[1] + "@" + i[2] p.hset(mapspace, i[0], user) p.sadd(gnamespace, user) p.execute() + def initialUsers(name, channel, users): trace("Initialising WHO records for %s on %s" % (channel, name)) d = deferToThread(_initialUsers, name, channel, users) - #d.addCallback(testCallback) + # d.addCallback(testCallback) + def _initialNames(name, channel, names): namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() for mode, nick in names: p.sadd(namespace, nick) - p.sadd("live.chan."+name+"."+nick, channel) + p.sadd("live.chan." + name + "." + nick, channel) if mode: - p.hset("live.prefix."+name+"."+channel, nick, mode) + p.hset("live.prefix." + name + "." + channel, nick, mode) p.execute() + def initialNames(name, channel, names): trace("Initialising NAMES records for %s on %s" % (channel, name)) d = deferToThread(_initialNames, name, channel, names) - #d.addCallback(testCallback) + # d.addCallback(testCallback) + def editUser(name, user): gnamespace = "live.who.%s" % name @@ -109,9 +124,10 @@ def editUser(name, user): parsed = parsen(user) p = main.r.pipeline() p.sadd(gnamespace, user) - p.hset(mapspace, parsed[0], user) # add nick -> user mapping + p.hset(mapspace, parsed[0], user) # add nick -> user mapping p.execute() + def addUser(name, channel, nick, user): gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick) p = main.r.pipeline() @@ -121,40 +137,43 @@ def addUser(name, channel, nick, user): p.hset(mapspace, nick, user) p.execute() + def delUser(name, channel, nick, user): gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick) p = main.r.pipeline() channels = main.r.smembers(chanspace) p.srem(namespace, nick) - if channels == {channel.encode()}: # can we only see them on this channel? - p.delete(chanspace) # remove channel tracking entry - p.hdel("live.prefix."+name+"."+channel, nick) # remove prefix tracking entry - p.hdel(mapspace, nick) # remove nick mapping entry + if channels == {channel.encode()}: # can we only see them on this channel? + p.delete(chanspace) # remove channel tracking entry + p.hdel("live.prefix." + name + "." + channel, nick) # remove prefix tracking entry + p.hdel(mapspace, nick) # remove nick mapping entry if user: - p.srem(gnamespace, user) # remove global userinfo entry + p.srem(gnamespace, user) # remove global userinfo entry else: warn("Attempt to delete nonexistent user: %s" % user) else: - p.srem(chanspace, channel) # keep up - remove the channel from their list + p.srem(chanspace, channel) # keep up - remove the channel from their list p.execute() + def escape(text): chars = ["[", "]", "^", "-", "*", "?"] text = text.replace("\\", "\\\\") for i in chars: - text = text.replace(i, "\\"+i) + text = text.replace(i, "\\" + i) return text + def getUserByNick(name, nick): - gnamespace = "live.who.%s" % name # "nick": "nick!ident@host" + gnamespace = "live.who.%s" % name # "nick": "nick!ident@host" mapspace = "live.map.%s" % name if main.r.hexists(mapspace, nick): return main.r.hget(mapspace, nick) else: warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) - #return False + # return False # legacy code below - remove when map is reliable - usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=999999999) + usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999) if usermatch[1] == []: warn("No matches found for user query: %s on %s" % (nick, name)) return False @@ -166,6 +185,7 @@ def getUserByNick(name, nick): warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace)) return False + def renameUser(name, oldnick, olduser, newnick, newuser): gnamespace = "live.who.%s" % name chanspace = "live.chan.%s.%s" % (name, oldnick) @@ -176,40 +196,43 @@ def renameUser(name, oldnick, olduser, newnick, newuser): p.sadd(gnamespace, newuser) for i in main.r.smembers(chanspace): i = i.decode() - p.srem("live.who."+name+"."+i, oldnick) - p.sadd("live.who."+name+"."+i, newnick) + p.srem("live.who." + name + "." + i, oldnick) + p.sadd("live.who." + name + "." + i, newnick) p.hdel(mapspace, oldnick) p.hset(mapspace, newnick, newuser) - if main.r.exists("live.prefix."+name+"."+i): # if there's a prefix entry for the channel - if main.r.hexists("live.prefix."+name+"."+i, oldnick): # if the old nick is in it - mode = main.r.hget("live.prefix."+name+"."+i, oldnick) # retrieve old modes - p.hset("live.prefix."+name+"."+i, newnick, mode) # set old modes to new nickname + if main.r.exists("live.prefix." + name + "." + i): # if there's a prefix entry for the channel + if main.r.hexists("live.prefix." + name + "." + i, oldnick): # if the old nick is in it + mode = main.r.hget("live.prefix." + name + "." + i, oldnick) # retrieve old modes + p.hset("live.prefix." + name + "." + i, newnick, mode) # set old modes to new nickname if main.r.exists(chanspace): p.rename(chanspace, newchanspace) else: warn("Key doesn't exist: %s" % chanspace) p.execute() -def delUserByNick(name, channel, nick): # kick + +def delUserByNick(name, channel, nick): # kick user = getUserByNick(name, nick) if not user: return delUser(name, channel, nick, user) -def delUserByNetwork(name, nick, user): # quit + +def delUserByNetwork(name, nick, user): # quit gnamespace = "live.who.%s" % name chanspace = "live.chan.%s.%s" % (name, nick) mapspace = "live.chan.%s" % name p = main.r.pipeline() p.srem(gnamespace, user) for i in main.r.smembers(chanspace): - p.srem("live.who."+name+"."+i.decode(), nick) - p.hdel("live.prefix."+name+"."+i.decode(), nick) + p.srem("live.who." + name + "." + i.decode(), nick) + p.hdel("live.prefix." + name + "." + i.decode(), nick) p.delete(chanspace) p.hdel(mapspace, nick) p.execute() + def _delChannels(net, channels): gnamespace = "live.who.%s" % net mapspace = "live.map.%s" % net @@ -218,23 +241,24 @@ def _delChannels(net, channels): namespace = "live.who.%s.%s" % (net, channel) for i in main.r.smembers(namespace): nick = i.decode() - #user = getUserByNick(net, nick) -- far too many function calls + # user = getUserByNick(net, nick) -- far too many function calls user = main.r.hget(mapspace, nick) if not user: warn("User lookup failed: %s on %s" % (nick, net)) - if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}: + if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}: if user: p.srem(gnamespace, user) - p.delete("live.chan."+net+"."+nick) - p.hdel(mapspace, nick) # remove map entry + p.delete("live.chan." + net + "." + nick) + p.hdel(mapspace, nick) # remove map entry else: - p.srem("live.chan."+net+"."+nick, channel) + p.srem("live.chan." + net + "." + nick, channel) p.delete(namespace) - p.delete("live.prefix."+net+"."+channel) + p.delete("live.prefix." + net + "." + channel) p.execute() -def delChannels(net, channels): # we have left a channel + +def delChannels(net, channels): # we have left a channel trace("Purging channel %s for %s" % (", ".join(channels), net)) d = deferToThread(_delChannels, net, channels) - #d.addCallback(testCallback) + # d.addCallback(testCallback) diff --git a/requirements.txt b/requirements.txt index 5ff3e80..8d6b25a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +pre-commit twisted pyOpenSSL redis diff --git a/threshold b/threshold index 46c0770..dc186b4 100755 --- a/threshold +++ b/threshold @@ -3,19 +3,23 @@ from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory import sys from signal import signal, SIGINT -#from twisted.python import log -#from sys import stdout -#log.startLogging(stdout) -from sys import stdout, stderr # Import again because we want to override -from codecs import getwriter # fix printing odd shit to the terminal + +# from twisted.python import log +# from sys import stdout +# log.startLogging(stdout) +from sys import stdout, stderr # Import again because we want to override +from codecs import getwriter # fix printing odd shit to the terminal + stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using - # unicode quit messages for no reason +# unicode quit messages for no reason import main + main.initMain() from utils.cleanup import handler -signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine -if "--debug" in sys.argv: # yes really + +signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine +if "--debug" in sys.argv: # yes really main.config["Debug"] = True if "--trace" in sys.argv: main.config["Trace"] = True @@ -24,25 +28,54 @@ from utils.loaders.command_loader import loadCommands from core.server import Server, ServerFactory from core.relay import Relay, RelayFactory import modules.counters + loadCommands() import core.logstash + core.logstash.init_logstash() if __name__ == "__main__": listener = ServerFactory() if main.config["Listener"]["UseSSL"] == True: - reactor.listenSSL(main.config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["Listener"]["Address"]) + reactor.listenSSL( + main.config["Listener"]["Port"], + listener, + DefaultOpenSSLContextFactory( + main.certPath + main.config["Key"], + main.certPath + main.config["Certificate"], + ), + interface=main.config["Listener"]["Address"], + ) log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) else: - reactor.listenTCP(main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"]) + reactor.listenTCP( + main.config["Listener"]["Port"], + listener, + interface=main.config["Listener"]["Address"], + ) log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) if main.config["RelayAPI"]["Enabled"]: relay = RelayFactory() if main.config["RelayAPI"]["UseSSL"] == True: - reactor.listenSSL(main.config["RelayAPI"]["Port"], relay, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["RelayAPI"]["Address"]) - log("Threshold relay running with SSL on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) + reactor.listenSSL( + main.config["RelayAPI"]["Port"], + relay, + DefaultOpenSSLContextFactory( + main.certPath + main.config["Key"], + main.certPath + main.config["Certificate"], + ), + interface=main.config["RelayAPI"]["Address"], + ) + log( + "Threshold relay running with SSL on %s:%s" + % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]) + ) else: - reactor.listenTCP(main.config["RelayAPI"]["Port"], relay, interface=main.config["RelayAPI"]["Address"]) + reactor.listenTCP( + main.config["RelayAPI"]["Port"], + relay, + interface=main.config["RelayAPI"]["Address"], + ) log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) for net in main.network.keys(): main.network[net].start_bots() diff --git a/utils/cleanup.py b/utils/cleanup.py index 3b818f3..83ed661 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -1,15 +1,17 @@ import main from twisted.internet import reactor from utils.logging.debug import debug -from utils.logging.log import * +from utils.logging.log import * import sys + def handler(sig, frame): log("Received SIGINT, cleaning up") cleanup() + def cleanup(): debug("Flushing Redis database") main.r.flushdb() reactor.stop() - #sys.exit(1) + # sys.exit(1) diff --git a/utils/dedup.py b/utils/dedup.py index a5c3c45..f64f26c 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -5,20 +5,21 @@ from json import dumps import main from utils.logging.debug import debug + def dedup(numName, b): c = deepcopy(b) if "time" in c.keys(): del c["time"] - c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]] + c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] - isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) + isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) if isDuplicate: debug("Duplicate: %s" % (c)) return True if numName in main.lastEvents.keys(): main.lastEvents[numName].insert(0, castHash) - main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]] + main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] else: main.lastEvents[numName] = [castHash] return False diff --git a/utils/get.py b/utils/get.py index 6843479..f455b6c 100644 --- a/utils/get.py +++ b/utils/get.py @@ -1,5 +1,6 @@ import main + def getRelay(num): host = main.config["Relay"]["Host"].replace("x", str(num)) port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2))) diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index bc9ecf4..9c4095b 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -6,13 +6,14 @@ import commands from main import CommandMap + def loadCommands(allowDup=False): for filename in listdir("commands"): if filename.endswith(".py") and filename != "__init__.py": commandName = filename[0:-3] - className = commandName.capitalize()+"Command" + className = commandName.capitalize() + "Command" try: - module = __import__('commands.%s' % commandName) + module = __import__("commands.%s" % commandName) if not commandName in CommandMap: CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index 81e5c76..f5219ac 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -8,16 +8,17 @@ import commands from main import CommandMap + def loadSingle(commandName): - if commandName+".py" in listdir("commands"): - className = commandName.capitalize()+"Command" + if commandName + ".py" in listdir("commands"): + className = commandName.capitalize() + "Command" try: if commandName in CommandMap.keys(): - reload(sys.modules["commands."+commandName]) - CommandMap[commandName] = getattr(sys.modules["commands."+commandName], className) + reload(sys.modules["commands." + commandName]) + CommandMap[commandName] = getattr(sys.modules["commands." + commandName], className) debug("Reloaded command: %s" % commandName) return "RELOAD" - module = __import__('commands.%s' % commandName) + module = __import__("commands.%s" % commandName) CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) return True diff --git a/utils/logging/debug.py b/utils/logging/debug.py index 4b8c708..80651f7 100644 --- a/utils/logging/debug.py +++ b/utils/logging/debug.py @@ -1,10 +1,12 @@ import main + # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main # to read the setting def debug(*data): if main.config["Debug"]: print("[DEBUG]", *data) + def trace(*data): if main.config["Trace"]: print("[TRACE]", *data) diff --git a/utils/logging/log.py b/utils/logging/log.py index 67b2148..1fd2a0a 100644 --- a/utils/logging/log.py +++ b/utils/logging/log.py @@ -1,8 +1,10 @@ def log(*data): print("[LOG]", *data) + def warn(*data): print("[WARNING]", *data) + def error(*data): print("[ERROR]", *data) diff --git a/utils/logging/send.py b/utils/logging/send.py index 0f4eeee..0623243 100644 --- a/utils/logging/send.py +++ b/utils/logging/send.py @@ -1,29 +1,36 @@ import main + def sendData(addr, data): main.connections[addr].send(data) + def sendWithPrefix(addr, data, prefix): toSend = "" for i in data.split("\n"): toSend += prefix + " " + i + "\n" sendData(addr, toSend) + def sendSuccess(addr, data): sendWithPrefix(addr, data, "[y]") + def sendFailure(addr, data): sendWithPrefix(addr, data, "[n]") + def sendInfo(addr, data): sendWithPrefix(addr, data, "[i]") + def sendAll(data): for i in main.connections: if main.connections[i].authed: main.connections[i].send(data) return + def incorrectUsage(addr, mode): if mode == None: sendFailure(addr, "Incorrect usage") From e5685286ae6db8af52c4cb67dfccb0cab9cc8fae Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:43 +0100 Subject: [PATCH 235/394] Improve email command --- commands/email.py | 58 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/commands/email.py b/commands/email.py index 1b0ec5e..4817988 100644 --- a/commands/email.py +++ b/commands/email.py @@ -9,19 +9,34 @@ class EmailCommand: def email(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 4: - if not spl[2].isdigit(): - # failure("Must be a number, not %s" % spl[2]) - if spl[2] == "domain": - domain = spl[3] - if "@" in domain: - failure("Not a domain: %s" % domain) - return - if not domain in main.irc["_"]["domains"]: - main.irc["_"]["domains"].append(domain) - success("Successfully added domain %s to default config" % domain) - else: - failure("Domain already exists in default config: %s" % domain) - return + if spl[1] == "add": + if not spl[2].isdigit(): + # failure("Must be a number, not %s" % spl[2]) + if spl[2] == "domain": + domain = spl[3] + if "@" in domain: + failure("Not a domain: %s" % domain) + return + if not domain in main.irc["_"]["domains"]: + main.irc["_"]["domains"].append(domain) + success("Successfully added domain %s to default config" % domain) + main.saveConf("irc") + else: + failure("Domain already exists in default config: %s" % domain) + return + elif spl[1] == "del": + if not spl[2].isdigit(): + # failure("Must be a number, not %s" % spl[2]) + if spl[2] == "domain": + domain = spl[3] + + if domain in main.irc["_"]["domains"]: + main.irc["_"]["domains"].remove(domain) + success("Successfully removed domain %s to default config" % domain) + main.saveConf("irc") + else: + failure("Domain does not exist in default config: %s" % domain) + return else: num = int(spl[2]) @@ -53,12 +68,23 @@ class EmailCommand: if spl[1] == "list": info(dump(main.alias)) return - else: - incUsage("save") + incUsage("email") + return + elif length == 3: + if spl[1] == "list": + if spl[2] == "domain": + filtered = {f"{k}:{k2}":v2 for k,v in main.irc.items() for k2,v2 in v.items() if k2 == "domains"} + info(dump(filtered)) + return + else: + incUsage("email") + return + else: + incUsage("email") return else: - incUsage("save") + incUsage("email") return else: incUsage(None) From 745c7caa120af171061410ad8ff11ef876a57c39 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:44 +0100 Subject: [PATCH 236/394] Fix some issues with the default config --- conf/example/config.json | 4 ++-- conf/example/irc.json | 1 + conf/help.json | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 9471a47..f238f8b 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -12,12 +12,12 @@ }, "Key": "key.pem", "Certificate": "cert.pem", - "RedisSocket": "/tmp/redis.sock", + "RedisSocket": "/var/run/redis/redis.sock", "UsePassword": true, "ConnectOnCreate": false, "AutoReg": false, "Debug": false, - "Trace", false, + "Trace": false, "Relay": { "Host": "127.0.0.1", "Port": "201x", diff --git a/conf/example/irc.json b/conf/example/irc.json index 8270e2f..a62dd51 100644 --- a/conf/example/irc.json +++ b/conf/example/irc.json @@ -3,6 +3,7 @@ "register": true, "entity": "NickServ", "email": "{nickname}@domain.com", + "domains": [], "registermsg": "REGISTER {password} {email}", "confirm": "CONFIRM {token}", "check": true, diff --git a/conf/help.json b/conf/help.json index 7ea570d..b86ef14 100644 --- a/conf/help.json +++ b/conf/help.json @@ -13,7 +13,6 @@ "dist": "dist", "loadmod": "loadmod ", "msg": "msg ", - "mon": "mon -h", "chans": "chans [ ...]", "users": "users [ ...]", "relay": "relay [] []", From 5c7d71dc9983e2fff0e040905af9a5a59a907210 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:46 +0100 Subject: [PATCH 237/394] Fix provisioning with emails --- modules/provision.py | 2 +- modules/regproc.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/provision.py b/modules/provision.py index 990db63..8e1dab2 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -5,7 +5,7 @@ from twisted.internet import reactor import modules.regproc -def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security, auth, password): +def provisionUserNetworkData(num, nick, altnick, emails, ident, realname, network, host, port, security, auth, password): commands = {} stage2commands = {} stage2commands["status"] = [] diff --git a/modules/regproc.py b/modules/regproc.py index 26cdae2..fe19cc7 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -3,7 +3,7 @@ from modules import provision from utils.logging.log import * from utils.logging.debug import * from copy import deepcopy - +from random import choice def needToRegister(net): inst = selectInst(net) @@ -28,16 +28,23 @@ def selectInst(net): def substitute(net, num, token=None): inst = selectInst(net) alias = main.alias[num] + if "emails" in inst: + # First priority is explicit email lists + email = choice(alias["emails"]) + elif "domains" in inst: + domain = choice(inst["domains"]) + email = f"{alias['nickname']}@{domain}" + print("Constructed email: {email}") nickname = alias["nick"] username = nickname + "/" + net password = main.network[net].aliases[num]["password"] - inst["email"] = inst["email"].replace("{nickname}", nickname) + #inst["email"] = inst["email"].replace("{nickname}", nickname) for i in inst.keys(): if not isinstance(inst[i], str): continue inst[i] = inst[i].replace("{nickname}", nickname) inst[i] = inst[i].replace("{password}", password) - inst[i] = inst[i].replace("{email}", inst["email"]) + inst[i] = inst[i].replace("{email}", email) if token: inst[i] = inst[i].replace("{token}", token) return inst From 4b2a1f273502c813f2bfa8a8ed592a4d7b54aa09 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:48 +0100 Subject: [PATCH 238/394] Add Redis DB numbers to configuration --- conf/example/config.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/example/config.json b/conf/example/config.json index f238f8b..3846c71 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -13,6 +13,8 @@ "Key": "key.pem", "Certificate": "cert.pem", "RedisSocket": "/var/run/redis/redis.sock", + "RedisDBEphemeral": 2, + "RedisDBPersistent": 3, "UsePassword": true, "ConnectOnCreate": false, "AutoReg": false, From 1532cf482ce6b9c59c334cc694565b86c80d47c6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:50 +0100 Subject: [PATCH 239/394] Make Redis DBs configurable --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index a29e1d7..473fd3e 100644 --- a/main.py +++ b/main.py @@ -106,5 +106,5 @@ def initConf(): def initMain(): global r, g initConf() - r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit - g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent + r = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]) # Ephemeral - flushed on quit + g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent From 757b22c4a13109f6abec07e01f4a81aadab95ed6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:52 +0100 Subject: [PATCH 240/394] Extra error handling around emails --- core/bot.py | 3 +++ modules/provision.py | 8 +++++++- modules/regproc.py | 23 +++++++++++++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/core/bot.py b/core/bot.py index b735e96..04e32dd 100644 --- a/core/bot.py +++ b/core/bot.py @@ -658,6 +658,9 @@ class IRCBot(IRCClient): if self.authenticated: return sinst = regproc.substitute(self.net, self.num) + if not sinst: + error(f"Registration ping failed for {self.net} - {self.num}") + return if not self._negativePass == True: if negativepass == False: self._negativePass = False diff --git a/modules/provision.py b/modules/provision.py index 8e1dab2..1f17a20 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -5,7 +5,12 @@ from twisted.internet import reactor import modules.regproc -def provisionUserNetworkData(num, nick, altnick, emails, ident, realname, network, host, port, security, auth, password): +def provisionUserNetworkData(num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password): + print("nick", nick) + print("altnick", altnick) + print("emails", emails) + print("ident", ident) + print("realname", realname) commands = {} stage2commands = {} stage2commands["status"] = [] @@ -56,6 +61,7 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): def provisionRelay(num, network): # provision user and network data aliasObj = main.alias[num] + print("ALIASOBJ FALUES", aliasObj.values()) alias = aliasObj["nick"] provisionUserNetworkData( num, diff --git a/modules/regproc.py b/modules/regproc.py index fe19cc7..5cf52f6 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -28,13 +28,21 @@ def selectInst(net): def substitute(net, num, token=None): inst = selectInst(net) alias = main.alias[num] - if "emails" in inst: + gotemail = False + if "emails" in alias: # First priority is explicit email lists - email = choice(alias["emails"]) - elif "domains" in inst: - domain = choice(inst["domains"]) - email = f"{alias['nickname']}@{domain}" - print("Constructed email: {email}") + if alias["emails"]: + email = choice(alias["emails"]) + gotemail = True + if "domains" in inst: + if inst["domains"]: + if not gotemail: + domain = choice(inst["domains"]) + email = f"{alias['nickname']}@{domain}" + gotemail = True + if not gotemail: + error(f"Could not get email for {net} - {num}") + return False nickname = alias["nick"] username = nickname + "/" + net password = main.network[net].aliases[num]["password"] @@ -53,6 +61,9 @@ def substitute(net, num, token=None): def registerAccount(net, num): debug("Attempting to register: %s - %i" % (net, num)) sinst = substitute(net, num) + if not sinst: + error(f"Register account failed for {net} - {num}") + return if not sinst["register"]: error("Cannot register for %s: function disabled" % (net)) return False From 7ffdc63eebb17fd9ab2d87916146c2db20caae0d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:54 +0100 Subject: [PATCH 241/394] Rename time to ts --- core/bot.py | 12 ++++++------ modules/monitor.py | 2 +- utils/dedup.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/bot.py b/core/bot.py index 04e32dd..ff0bde7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -221,8 +221,8 @@ class IRCBot(IRCClient): del main.TempChan[self.net] def event(self, **cast): - if not "time" in cast.keys(): - cast["time"] = str(datetime.now().isoformat()) + if not "ts" in cast.keys(): + cast["ts"] = str(datetime.now().isoformat()) # remove odd stuff for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over @@ -699,7 +699,7 @@ class IRCBot(IRCClient): "net": self.net, "num": self.num, "status": "signedon", - "time": ctime, + "ts": ctime, } ) if not self.authenticated: @@ -747,7 +747,7 @@ class IRCBot(IRCClient): self.event(type="kick", muser=kicker, channel=channel, msg=message, user=kickee) def chanlessEvent(self, cast): - cast["time"] = str(datetime.now().isoformat()) + cast["ts"] = str(datetime.now().isoformat()) cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) if dedup(self.name, cast): # Needs to be kept self.name until the dedup # function is converted to the new net, num @@ -834,7 +834,7 @@ class IRCBotFactory(ReconnectingClientFactory): "num": self.num, "status": "lost", "message": error, - "time": ctime, + "ts": ctime, } ) self.retry(connector) @@ -857,7 +857,7 @@ class IRCBotFactory(ReconnectingClientFactory): "num": self.num, "status": "failed", "message": error, - "time": ctime, + "ts": ctime, } ) self.retry(connector) diff --git a/modules/monitor.py b/modules/monitor.py index 8c4681d..f174a80 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -24,7 +24,7 @@ order = [ "realname", "server", "status", - "time", + "ts", ] diff --git a/utils/dedup.py b/utils/dedup.py index f64f26c..dc8d4c3 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -8,8 +8,8 @@ from utils.logging.debug import debug def dedup(numName, b): c = deepcopy(b) - if "time" in c.keys(): - del c["time"] + if "ts" in c.keys(): + del c["ts"] c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] From e4a6e0d3c2f306aaf1eb8e4871356e76cec431e9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:56 +0100 Subject: [PATCH 242/394] Don't attempt to register if it is disabled --- .pre-commit-config.yaml | 13 +++++++++++++ modules/regproc.py | 5 +++++ 2 files changed, 18 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..98fa1cb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: +- repo: https://github.com/ambv/black + rev: 22.3.0 + hooks: + - id: black + args: + - --line-length=120 +- repo: https://gitlab.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: + - "--max-line-length=120" diff --git a/modules/regproc.py b/modules/regproc.py index 5cf52f6..ce0e5a2 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -6,6 +6,11 @@ from copy import deepcopy from random import choice def needToRegister(net): + # Check if the network does not support authentication + networkObj = main.network[net] + if networkObj.auth == "none": + return False + # Check if the IRC network definition has registration disabled inst = selectInst(net) if "register" in inst.keys(): if inst["register"]: From 9d4d31fdc2ba170d4eddea8816fd01edb7014aa5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:57 +0100 Subject: [PATCH 243/394] Don't attempt secondary registration if it is disabled --- core/bot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index ff0bde7..375703d 100644 --- a/core/bot.py +++ b/core/bot.py @@ -559,8 +559,9 @@ class IRCBot(IRCClient): "registered" ]: # TODO: add check for register request sent, only send it once if main.config["AutoReg"]: - self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) - # regproc.registerAccount(self.net, self.num) + if not self.authenticated: + self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) + # regproc.registerAccount(self.net, self.num) try: self.chanlimit = int(chanlimit) except TypeError: From 760e43b59ab614431372f7e2291d1700d3a248e3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:39:59 +0100 Subject: [PATCH 244/394] Reformat project --- .pre-commit-config.yaml | 24 ++++--- commands/admall.py | 4 +- commands/alias.py | 7 +- commands/all.py | 9 ++- commands/allc.py | 12 +++- commands/authcheck.py | 12 +++- commands/auto.py | 24 +++++-- commands/blacklist.py | 7 +- commands/chans.py | 4 +- commands/cmd.py | 4 +- commands/confirm.py | 9 ++- commands/disable.py | 4 +- commands/dist.py | 7 +- commands/email.py | 52 +++++++++++--- commands/enable.py | 4 +- commands/exec.py | 4 +- commands/help.py | 4 +- commands/join.py | 4 +- commands/list.py | 4 +- commands/load.py | 4 +- commands/loadmod.py | 4 +- commands/logout.py | 4 +- commands/mod.py | 11 ++- commands/msg.py | 9 ++- commands/network.py | 16 +++-- commands/part.py | 4 +- commands/pass.py | 4 +- commands/pending.py | 14 +++- commands/recheckauth.py | 4 +- commands/reg.py | 4 +- commands/relay.py | 21 ++++-- commands/save.py | 4 +- commands/stats.py | 7 +- commands/swho.py | 4 +- commands/token.py | 10 ++- commands/users.py | 4 +- commands/who.py | 4 +- core/bot.py | 119 ++++++++++++++++++++------------ core/logstash.py | 5 +- core/parser.py | 4 +- core/relay.py | 19 +++-- core/server.py | 11 +-- main.py | 13 ++-- modules/alias.py | 3 +- modules/chankeep.py | 27 +++++--- modules/counters.py | 3 +- modules/monitor.py | 9 +-- modules/network.py | 15 ++-- modules/provision.py | 36 ++++++++-- modules/regproc.py | 28 +++++--- modules/userinfo.py | 30 +++++--- threshold | 35 ++++++---- utils/cleanup.py | 6 +- utils/dedup.py | 20 ++++-- utils/loaders/command_loader.py | 13 ++-- utils/loaders/single_loader.py | 13 ++-- utils/logging/debug.py | 1 + 57 files changed, 523 insertions(+), 218 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98fa1cb..11c38e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,21 @@ repos: -- repo: https://github.com/ambv/black - rev: 22.3.0 + - repo: https://github.com/psf/black + rev: 22.6.0 hooks: - id: black - args: - - --line-length=120 -- repo: https://gitlab.com/pycqa/flake8 + exclude: ^core/migrations + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: - - id: flake8 - args: - - "--max-line-length=120" + - id: flake8 + args: [--max-line-length=88] + exclude: ^core/migrations + - repo: https://github.com/thibaudcolas/curlylint + rev: v0.13.1 + hooks: + - id: curlylint + files: \.(html|sls)$ diff --git a/commands/admall.py b/commands/admall.py index 8d74545..cb0c009 100644 --- a/commands/admall.py +++ b/commands/admall.py @@ -6,7 +6,9 @@ class AdmallCommand: def __init__(self, *args): self.admall(*args) - def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def admall( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length > 2: for i in main.network.keys(): diff --git a/commands/alias.py b/commands/alias.py index d04cfdf..5bc12bf 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -1,5 +1,6 @@ -import main from yaml import dump + +import main from modules import alias @@ -7,7 +8,9 @@ class AliasCommand: def __init__(self, *args): self.alias(*args) - def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def alias( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: info(dump(main.alias)) diff --git a/commands/all.py b/commands/all.py index a2994e9..16a6721 100644 --- a/commands/all.py +++ b/commands/all.py @@ -6,7 +6,9 @@ class AllCommand: def __init__(self, *args): self.all(*args) - def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def all( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length > 2: for i in main.network.keys(): @@ -15,7 +17,10 @@ class AllCommand: net = main.network[i].relays[x]["net"] alias = main.alias[x]["nick"] commands = {spl[1]: [" ".join(spl[2:])]} - success("Sending commands to relay %s as user %s" % (num, alias + "/" + net)) + success( + "Sending commands to relay %s as user %s" + % (num, alias + "/" + net) + ) deliverRelayCommands(num, commands, user=alias + "/" + net) return else: diff --git a/commands/allc.py b/commands/allc.py index 7a7b5fd..01698e5 100644 --- a/commands/allc.py +++ b/commands/allc.py @@ -6,7 +6,9 @@ class AllcCommand: def __init__(self, *args): self.allc(*args) - def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def allc( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length > 4: targets = [] @@ -20,7 +22,8 @@ class AllcCommand: [ targets.append((i, x)) for x in main.alias.keys() - if main.alias[x]["nick"] == spl[2] and x in main.network[i].aliases.keys() + if main.alias[x]["nick"] == spl[2] + and x in main.network[i].aliases.keys() ] else: incUsage("allc") @@ -33,7 +36,10 @@ class AllcCommand: num = i[1] alias = main.alias[num]["nick"] commands = {spl[3]: [" ".join(spl[4:])]} - success("Sending commands to relay %i as user %s" % (num, alias + "/" + net)) + success( + "Sending commands to relay %i as user %s" + % (num, alias + "/" + net) + ) deliverRelayCommands(num, commands, user=alias + "/" + net) return else: diff --git a/commands/authcheck.py b/commands/authcheck.py index 26cc14e..7c41dd1 100644 --- a/commands/authcheck.py +++ b/commands/authcheck.py @@ -5,7 +5,9 @@ class AuthcheckCommand: def __init__(self, *args): self.authcheck(*args) - def authcheck(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def authcheck( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: results = [] @@ -13,7 +15,9 @@ class AuthcheckCommand: num = main.IRCPool[i].num net = main.IRCPool[i].net if not main.IRCPool[i].authenticated: - results.append("%s - %s: %s" % (net, num, main.alias[num]["nick"])) + results.append( + "%s - %s: %s" % (net, num, main.alias[num]["nick"]) + ) info("\n".join(results)) return elif length == 2: @@ -27,7 +31,9 @@ class AuthcheckCommand: if not net == spl[1]: continue if not main.IRCPool[i].authenticated: - results.append("%s - %s: %s" % (net, num, main.alias[num]["nick"])) + results.append( + "%s - %s: %s" % (net, num, main.alias[num]["nick"]) + ) info("\n".join(results)) return else: diff --git a/commands/auto.py b/commands/auto.py index ffc6df5..378a102 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -6,7 +6,9 @@ class AutoCommand: def __init__(self, *args): self.auto(*args) - def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def auto( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: for i in main.network.keys(): @@ -14,9 +16,15 @@ class AutoCommand: info("Skipping %s - first relay exists" % i) else: num, alias = main.network[i].add_relay(1) - success("Successfully created first relay on network %s with alias %s" % (i, alias)) + success( + "Successfully created first relay on network %s with alias %s" + % (i, alias) + ) provision.provisionRelay(num, i) - success("Started provisioning network %s on first relay for alias %s" % (i, alias)) + success( + "Started provisioning network %s on first relay for alias %s" + % (i, alias) + ) main.saveConf("network") return elif length == 2: @@ -27,9 +35,15 @@ class AutoCommand: failure("First relay exists on %s" % spl[1]) return num, alias = main.network[spl[1]].add_relay(1) - success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) + success( + "Successfully created relay %i on network %s with alias %s" + % (num, spl[1], alias) + ) provision.provisionRelay(num, spl[1]) - success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias)) + success( + "Started provisioning network %s on relay %s for alias %s" + % (spl[1], num, alias) + ) main.saveConf("network") return else: diff --git a/commands/blacklist.py b/commands/blacklist.py index 42684a6..858da28 100644 --- a/commands/blacklist.py +++ b/commands/blacklist.py @@ -1,12 +1,15 @@ -import main from yaml import dump +import main + class BlacklistCommand: def __init__(self, *args): self.blacklist(*args) - def blacklist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def blacklist( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: info(dump(main.blacklist)) diff --git a/commands/chans.py b/commands/chans.py index 1ccf710..8b820ff 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -6,7 +6,9 @@ class ChansCommand: def __init__(self, *args): self.chans(*args) - def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def chans( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if len(spl) < 2: incUsage("chans") diff --git a/commands/cmd.py b/commands/cmd.py index babdb2d..f2706db 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -6,7 +6,9 @@ class CmdCommand: def __init__(self, *args): self.cmd(*args) - def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def cmd( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length > 4: if not spl[1].isdigit(): diff --git a/commands/confirm.py b/commands/confirm.py index 86d657c..53a70a2 100644 --- a/commands/confirm.py +++ b/commands/confirm.py @@ -6,7 +6,9 @@ class ConfirmCommand: def __init__(self, *args): self.confirm(*args) - def confirm(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def confirm( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 4: if not spl[1] in main.network.keys(): @@ -19,7 +21,10 @@ class ConfirmCommand: failure("No such relay on %s: %s" % (spl[1], spl[2])) return regproc.confirmAccount(spl[1], int(spl[2]), spl[3]) - success("Requested confirmation on %s - %s with token %s" % (spl[1], spl[2], spl[3])) + success( + "Requested confirmation on %s - %s with token %s" + % (spl[1], spl[2], spl[3]) + ) return else: incUsage("confirm") diff --git a/commands/disable.py b/commands/disable.py index 9c52234..9a56fb4 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -6,7 +6,9 @@ class DisableCommand: def __init__(self, *args): self.disable(*args) - def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def disable( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 3: if not spl[1] in main.network.keys(): diff --git a/commands/dist.py b/commands/dist.py index 2348e29..7e94c4e 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -1,12 +1,15 @@ +from subprocess import PIPE, run + import main -from subprocess import run, PIPE class DistCommand: def __init__(self, *args): self.dist(*args) - def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def dist( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if main.config["Dist"]["Enabled"]: rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE) diff --git a/commands/email.py b/commands/email.py index 4817988..3b78931 100644 --- a/commands/email.py +++ b/commands/email.py @@ -1,12 +1,15 @@ -import main from yaml import dump +import main + class EmailCommand: def __init__(self, *args): self.email(*args) - def email(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def email( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 4: if spl[1] == "add": @@ -19,10 +22,16 @@ class EmailCommand: return if not domain in main.irc["_"]["domains"]: main.irc["_"]["domains"].append(domain) - success("Successfully added domain %s to default config" % domain) + success( + "Successfully added domain %s to default config" + % domain + ) main.saveConf("irc") else: - failure("Domain already exists in default config: %s" % domain) + failure( + "Domain already exists in default config: %s" + % domain + ) return elif spl[1] == "del": if not spl[2].isdigit(): @@ -32,10 +41,16 @@ class EmailCommand: if domain in main.irc["_"]["domains"]: main.irc["_"]["domains"].remove(domain) - success("Successfully removed domain %s to default config" % domain) + success( + "Successfully removed domain %s to default config" + % domain + ) main.saveConf("irc") else: - failure("Domain does not exist in default config: %s" % domain) + failure( + "Domain does not exist in default config: %s" + % domain + ) return else: @@ -47,10 +62,15 @@ class EmailCommand: if not spl[3] in main.alias[num]["emails"]: main.alias[num]["emails"].append(spl[3]) main.saveConf("alias") - success("Successfully added email %s to alias %i" % (spl[3], num)) + success( + "Successfully added email %s to alias %i" + % (spl[3], num) + ) return else: - failure("Email already exists in alias %i: %s" % (num, spl[3])) + failure( + "Email already exists in alias %i: %s" % (num, spl[3]) + ) return elif spl[1] == "del": if not num in main.alias.keys(): @@ -59,10 +79,15 @@ class EmailCommand: if spl[3] in main.alias[num]["emails"]: main.alias[num]["emails"].remove(spl[3]) main.saveConf("alias") - success("Successfully removed email %s from alias %i" % (spl[3], num)) + success( + "Successfully removed email %s from alias %i" + % (spl[3], num) + ) return else: - failure("Email does not exist in alias %i: %s" % (spl[3], num)) + failure( + "Email does not exist in alias %i: %s" % (spl[3], num) + ) return elif length == 2: if spl[1] == "list": @@ -74,7 +99,12 @@ class EmailCommand: elif length == 3: if spl[1] == "list": if spl[2] == "domain": - filtered = {f"{k}:{k2}":v2 for k,v in main.irc.items() for k2,v2 in v.items() if k2 == "domains"} + filtered = { + f"{k}:{k2}": v2 + for k, v in main.irc.items() + for k2, v2 in v.items() + if k2 == "domains" + } info(dump(filtered)) return else: diff --git a/commands/enable.py b/commands/enable.py index 439b26c..1cfb0e4 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -6,7 +6,9 @@ class EnableCommand: def __init__(self, *args): self.enable(*args) - def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def enable( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 3: if not spl[1] in main.network.keys(): diff --git a/commands/exec.py b/commands/exec.py index c2a5598..6f7a924 100644 --- a/commands/exec.py +++ b/commands/exec.py @@ -5,7 +5,9 @@ class ExecCommand: def __init__(self, *args): self.exec(*args) - def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def exec( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length > 1: try: diff --git a/commands/help.py b/commands/help.py index 6dcf937..0ca3ff7 100644 --- a/commands/help.py +++ b/commands/help.py @@ -5,7 +5,9 @@ class HelpCommand: def __init__(self, *args): self.help(*args) - def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def help( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: helpMap = [] for i in main.help.keys(): diff --git a/commands/join.py b/commands/join.py index e6cbc20..d6147a0 100644 --- a/commands/join.py +++ b/commands/join.py @@ -6,7 +6,9 @@ class JoinCommand: def __init__(self, *args): self.join(*args) - def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def join( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 3: if not spl[1] in main.network.keys(): diff --git a/commands/list.py b/commands/list.py index 217b051..7fbc616 100644 --- a/commands/list.py +++ b/commands/list.py @@ -5,7 +5,9 @@ class ListCommand: def __init__(self, *args): self.list(*args) - def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def list( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: for i in main.network.keys(): diff --git a/commands/load.py b/commands/load.py index 31e66f4..b62da89 100644 --- a/commands/load.py +++ b/commands/load.py @@ -5,7 +5,9 @@ class LoadCommand: def __init__(self, *args): self.load(*args) - def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def load( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: if spl[1] in main.filemap.keys(): diff --git a/commands/loadmod.py b/commands/loadmod.py index 2923fd7..e440b58 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -6,7 +6,9 @@ class LoadmodCommand: def __init__(self, *args): self.loadmod(*args) - def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def loadmod( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: rtrn = loadSingle(spl[1]) diff --git a/commands/logout.py b/commands/logout.py index 61cb834..06bd437 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -5,7 +5,9 @@ class LogoutCommand: def __init__(self, *args): self.logout(*args) - def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def logout( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: obj.authed = False success("Logged out") diff --git a/commands/mod.py b/commands/mod.py index 10a19e1..94d3a7a 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,13 +1,16 @@ -import main from yaml import dump +import main + class ModCommand: # This could be greatly improved, but not really important right now def __init__(self, *args): self.mod(*args) - def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def mod( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 4: if not spl[1] in main.network.keys(): @@ -21,7 +24,9 @@ class ModCommand: return main.saveConf("network") - success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) + success( + "Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]) + ) return # Find a better way to do this # elif length == 6: diff --git a/commands/msg.py b/commands/msg.py index d92494a..754176c 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -5,7 +5,9 @@ class MsgCommand: def __init__(self, *args): self.msg(*args) - def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def msg( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length >= 5: if not spl[1] in main.network.keys(): @@ -20,7 +22,10 @@ class MsgCommand: if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels: info("Bot not on channel: %s" % spl[3]) main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:])) - success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1])) + success( + "Sent %s to %s on relay %s on network %s" + % (" ".join(spl[4:]), spl[3], spl[2], spl[1]) + ) return else: incUsage("msg") diff --git a/commands/network.py b/commands/network.py index 4a6bdf2..13b23bd 100644 --- a/commands/network.py +++ b/commands/network.py @@ -1,14 +1,18 @@ -import main -from yaml import dump -from modules.network import Network from string import digits +from yaml import dump + +import main +from modules.network import Network + class NetworkCommand: def __init__(self, *args): self.network(*args) - def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def network( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 7: if spl[1] == "add": @@ -28,7 +32,9 @@ class NetworkCommand: failure("Auth must be sasl, ns or none, not %s" % spl[5]) return else: - main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()) + main.network[spl[2]] = Network( + spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower() + ) success("Successfully created network: %s" % spl[2]) main.saveConf("network") return diff --git a/commands/part.py b/commands/part.py index 249c32a..5a2de7e 100644 --- a/commands/part.py +++ b/commands/part.py @@ -5,7 +5,9 @@ class PartCommand: def __init__(self, *args): self.part(*args) - def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def part( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 4: if not spl[1] in main.network.keys(): diff --git a/commands/pass.py b/commands/pass.py index e37a51e..4b9e601 100644 --- a/commands/pass.py +++ b/commands/pass.py @@ -5,7 +5,9 @@ class PassCommand: def __init__(self, *args): self.password(*args) - def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def password( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: info("You are already authenticated") return diff --git a/commands/pending.py b/commands/pending.py index b65f2ad..5f962dd 100644 --- a/commands/pending.py +++ b/commands/pending.py @@ -5,14 +5,19 @@ class PendingCommand: def __init__(self, *args): self.pending(*args) - def pending(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def pending( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: results = [] for i in main.network.keys(): for x in main.network[i].relays.keys(): if not main.network[i].relays[x]["registered"]: - results.append("%s: confirm %s %s [code]" % (main.alias[x]["nick"], i, x)) + results.append( + "%s: confirm %s %s [code]" + % (main.alias[x]["nick"], i, x) + ) info("\n".join(results)) return elif length == 2: @@ -22,7 +27,10 @@ class PendingCommand: results = [] for x in main.network[spl[1]].relays.keys(): if not main.network[spl[1]].relays[x]["registered"]: - results.append("%s: confirm %s %s [code]" % (main.alias[x]["nick"], spl[1], x)) + results.append( + "%s: confirm %s %s [code]" + % (main.alias[x]["nick"], spl[1], x) + ) info("\n".join(results)) return else: diff --git a/commands/recheckauth.py b/commands/recheckauth.py index ab4b5b0..6cea741 100644 --- a/commands/recheckauth.py +++ b/commands/recheckauth.py @@ -5,7 +5,9 @@ class RecheckauthCommand: def __init__(self, *args): self.recheckauth(*args) - def recheckauth(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def recheckauth( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: results = [] diff --git a/commands/reg.py b/commands/reg.py index 41bb5f0..fb3794a 100644 --- a/commands/reg.py +++ b/commands/reg.py @@ -6,7 +6,9 @@ class RegCommand: def __init__(self, *args): self.reg(*args) - def reg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def reg( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: if not spl[1] in main.network.keys(): diff --git a/commands/relay.py b/commands/relay.py index 34bbd70..7bfc3d8 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -1,18 +1,24 @@ -import main from yaml import dump +import main + class RelayCommand: def __init__(self, *args): self.relay(*args) - def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def relay( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 3: if spl[1] == "add": if spl[2] in main.network.keys(): id, alias = main.network[spl[2]].add_relay() - success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) + success( + "Successfully created relay %s on network %s with alias %s" + % (str(id), spl[2], alias) + ) main.saveConf("network") return else: @@ -35,7 +41,10 @@ class RelayCommand: failure("Must be a number, not %s" % spl[3]) return id, alias = main.network[spl[2]].add_relay(int(spl[3])) - success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) + success( + "Successfully created relay %s on network %s with alias %s" + % (str(id), spl[2], alias) + ) main.saveConf("network") return else: @@ -52,7 +61,9 @@ class RelayCommand: failure("No such relay: %s on network %s" % (spl[3], spl[2])) return main.network[spl[2]].delete_relay(int(spl[3])) - success("Successfully deleted relay %s on network %s" % (spl[3], spl[2])) + success( + "Successfully deleted relay %s on network %s" % (spl[3], spl[2]) + ) main.saveConf("network") return else: diff --git a/commands/save.py b/commands/save.py index 12624cd..0dd22b6 100644 --- a/commands/save.py +++ b/commands/save.py @@ -5,7 +5,9 @@ class SaveCommand: def __init__(self, *args): self.save(*args) - def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def save( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: if spl[1] in main.filemap.keys(): diff --git a/commands/stats.py b/commands/stats.py index a396f12..ab6e686 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,14 +1,17 @@ +from string import digits + import main import modules.counters as count import modules.userinfo as userinfo -from string import digits class StatsCommand: def __init__(self, *args): self.stats(*args) - def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def stats( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 1: stats = [] diff --git a/commands/swho.py b/commands/swho.py index 18dbc12..aa3a78c 100644 --- a/commands/swho.py +++ b/commands/swho.py @@ -5,7 +5,9 @@ class SwhoCommand: def __init__(self, *args): self.swho(*args) - def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def swho( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: if not spl[1] in main.network.keys(): diff --git a/commands/token.py b/commands/token.py index 7d38eac..8d95394 100644 --- a/commands/token.py +++ b/commands/token.py @@ -1,13 +1,17 @@ -import main -from yaml import dump from uuid import uuid4 +from yaml import dump + +import main + class TokenCommand: def __init__(self, *args): self.token(*args) - def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def token( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: if spl[1] == "list": diff --git a/commands/users.py b/commands/users.py index 7d80e84..6ba7563 100644 --- a/commands/users.py +++ b/commands/users.py @@ -6,7 +6,9 @@ class UsersCommand: def __init__(self, *args): self.users(*args) - def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def users( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if len(spl) < 2: incUsage("users") diff --git a/commands/who.py b/commands/who.py index e80f488..8273ef4 100644 --- a/commands/who.py +++ b/commands/who.py @@ -6,7 +6,9 @@ class WhoCommand: def __init__(self, *args): self.who(*args) - def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + def who( + self, addr, authed, data, obj, spl, success, failure, info, incUsage, length + ): if authed: if length == 2: result = userinfo.getWho(spl[1]) diff --git a/core/bot.py b/core/bot.py index 375703d..31ff5d5 100644 --- a/core/bot.py +++ b/core/bot.py @@ -1,45 +1,38 @@ -from twisted.internet.protocol import ReconnectingClientFactory -from twisted.words.protocols.irc import IRCClient -from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall -from twisted.internet import reactor, task -from twisted.words.protocols.irc import ( - symbolic_to_numeric, - numeric_to_symbolic, - lowDequote, - IRCBadMessage, -) - import sys -from string import digits -from random import randint from copy import deepcopy from datetime import datetime +from random import randint +from string import digits -from modules import userinfo -from modules import counters -from modules import monitor -from modules import chankeep -from modules import regproc - -from core.relay import sendRelayNotification -from utils.dedup import dedup -from utils.get import getRelay +from twisted.internet import reactor, task +from twisted.internet.defer import Deferred +from twisted.internet.protocol import ReconnectingClientFactory +from twisted.internet.ssl import DefaultOpenSSLContextFactory +from twisted.internet.task import LoopingCall +from twisted.words.protocols.irc import (IRCBadMessage, IRCClient, lowDequote, + numeric_to_symbolic, + symbolic_to_numeric) import main -from utils.logging.log import * +from core.relay import sendRelayNotification +from modules import chankeep, counters, monitor, regproc, userinfo +from utils.dedup import dedup +from utils.get import getRelay from utils.logging.debug import * +from utils.logging.log import * from utils.logging.send import * from utils.parsing import parsen -from twisted.internet.ssl import DefaultOpenSSLContextFactory - def deliverRelayCommands(num, relayCommands, user=None, stage2=None): keyFN = main.certPath + main.config["Key"] certFN = main.certPath + main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) + contextFactory = DefaultOpenSSLContextFactory( + keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace") + ) + bot = IRCBotFactory( + net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2 + ) host, port = getRelay(num) rct = reactor.connectSSL(host, port, bot, contextFactory) @@ -165,7 +158,9 @@ class IRCBot(IRCClient): self.listAttempted = False # we asked for a list self.listSimple = False # after asking again we got the list, so use the simple # syntax from now on - self.wantList = False # we want to send a LIST, but not all relays are active yet + self.wantList = ( + False # we want to send a LIST, but not all relays are active yet + ) self.chanlimit = 0 self.prefix = {} self.servername = None @@ -197,9 +192,14 @@ class IRCBot(IRCClient): if not i in self.channels: if self.net in main.blacklist.keys(): if i in main.blacklist[self.net]: - debug("Not joining blacklisted channel %s on %s - %i" % (i, self.net, self.num)) + debug( + "Not joining blacklisted channel %s on %s - %i" + % (i, self.net, self.num) + ) continue - debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds") + debug( + self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds" + ) reactor.callLater(sleeptime, self.join, i) sleeptime += increment if sleeptime == 10: @@ -207,11 +207,17 @@ class IRCBot(IRCClient): increment = 0.7 increment += 0.1 else: - error("%s - Cannot join channel we are already on: %s - %i" % (i, self.net, self.num)) + error( + "%s - Cannot join channel we are already on: %s - %i" + % (i, self.net, self.num) + ) def checkChannels(self): if not chankeep.allRelaysActive(self.net): - debug("Skipping channel check as we have inactive relays: %s - %i" % (self.net, self.num)) + debug( + "Skipping channel check as we have inactive relays: %s - %i" + % (self.net, self.num) + ) return if self.net in main.TempChan.keys(): if self.num in main.TempChan[self.net].keys(): @@ -225,7 +231,9 @@ class IRCBot(IRCClient): cast["ts"] = str(datetime.now().isoformat()) # remove odd stuff - for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over + for i in list( + cast.keys() + ): # Make a copy of the .keys() as Python 3 cannot handle iterating over if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] # remove server stuff @@ -359,7 +367,9 @@ class IRCBot(IRCClient): def irc_ERR_PASSWDMISMATCH(self, prefix, params): log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username)) - sendAll("%s - %i: password mismatch as %s" % (self.net, self.num, self.username)) + sendAll( + "%s - %i: password mismatch as %s" % (self.net, self.num, self.username) + ) def _who(self, channel): d = Deferred() @@ -474,12 +484,17 @@ class IRCBot(IRCClient): debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num)) return if self.listAttempted: - debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num)) + debug( + "List request dropped, already asked for LIST - %s - %i" + % (self.net, self.num) + ) return else: self.listAttempted = True if self.listOngoing: - debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)) + debug( + "LIST request dropped, already ongoing - %s - %i" % (self.net, self.num) + ) return else: if nocheck: @@ -504,7 +519,9 @@ class IRCBot(IRCClient): self._tempList[1].append([channel, users, topic]) def irc_RPL_LISTEND(self, prefix, params): - if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there + if ( + not len(self._tempList[0]) > 0 + ): # there are no callbacks, can't do anything there debug("We didn't ask for this LIST, discarding") self._tempList[0].clear() self._tempList[1].clear() @@ -529,7 +546,10 @@ class IRCBot(IRCClient): else: if self.listRetried: self.listRetried = False - debug("List received after retry - defaulting to simple list syntax: %s - %i" % (self.net, self.num)) + debug( + "List received after retry - defaulting to simple list syntax: %s - %i" + % (self.net, self.num) + ) self.listSimple = True def got_list(self, listinfo): @@ -543,7 +563,10 @@ class IRCBot(IRCClient): name = self.net + "1" if main.IRCPool[name].wantList == True: main.IRCPool[name].list(nocheck=True) - debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) + debug( + "Asking for a list for %s after final relay %i connected" + % (self.net, self.num) + ) if self.num == 1: # Only one instance should do a list if self.chanlimit: if allRelays: @@ -560,7 +583,9 @@ class IRCBot(IRCClient): ]: # TODO: add check for register request sent, only send it once if main.config["AutoReg"]: if not self.authenticated: - self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) + self._regAttempt = reactor.callLater( + 5, regproc.registerAccount, self.net, self.num + ) # regproc.registerAccount(self.net, self.num) try: self.chanlimit = int(chanlimit) @@ -568,7 +593,9 @@ class IRCBot(IRCClient): warn("Invalid chanlimit: %s" % i) if self.chanlimit == 0: self.chanlimit = 200 # don't take the piss if it's not limited - if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation + if not regproc.needToRegister( + self.net + ): # if we need to register, only recheck on auth confirmation self.recheckList() def seed_prefix(self, prefix): @@ -669,7 +696,9 @@ class IRCBot(IRCClient): if negativepass == True: if self._negativePass == None: self._negativePass = True - debug("Positive registration check - %s - %i" % (self.net, self.num)) + debug( + "Positive registration check - %s - %i" % (self.net, self.num) + ) if sinst["ping"]: debug("Sending ping - %s - %i" % (self.net, self.num)) self.msg(sinst["entity"], sinst["pingmsg"]) @@ -725,7 +754,9 @@ class IRCBot(IRCClient): lc = self._getWho[channel] lc.stop() del self._getWho[channel] - userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this + userinfo.delChannels( + self.net, [channel] + ) # < we do not need to deduplicate this # log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so diff --git a/core/logstash.py b/core/logstash.py index 17e8dfd..66cded9 100644 --- a/core/logstash.py +++ b/core/logstash.py @@ -1,7 +1,8 @@ -import logstash import logging - from json import dumps, loads + +import logstash + import main from utils.logging.log import * diff --git a/core/parser.py b/core/parser.py index 1eb34c5..8d7c635 100644 --- a/core/parser.py +++ b/core/parser.py @@ -24,7 +24,9 @@ def parseCommand(addr, authed, data): failure("No text was sent") return if spl[0] in main.CommandMap.keys(): - main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length) + main.CommandMap[spl[0]]( + addr, authed, data, obj, spl, success, failure, info, incUsage, length + ) return incUsage(None) return diff --git a/core/relay.py b/core/relay.py index 66a9830..7f63209 100644 --- a/core/relay.py +++ b/core/relay.py @@ -1,6 +1,7 @@ -from twisted.internet.protocol import Protocol, Factory, ClientFactory -from json import dumps, loads from copy import deepcopy +from json import dumps, loads + +from twisted.internet.protocol import ClientFactory, Factory, Protocol import main from utils.logging.log import * @@ -114,8 +115,13 @@ class Relay(Protocol): def handleHello(self, parsed): if parsed["key"] in main.tokens.keys(): - if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay": - self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}) + if ( + parsed["hello"] == main.tokens[parsed["key"]]["hello"] + and main.tokens[parsed["key"]]["usage"] == "relay" + ): + self.sendMsg( + {"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]} + ) self.authed = True else: self.transport.loseConnection() @@ -130,7 +136,10 @@ class Relay(Protocol): def connectionLost(self, reason): self.authed = False - log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) + log( + "Relay connection lost from %s:%s -- %s" + % (self.addr.host, self.addr.port, reason.getErrorMessage()) + ) if self.addr in main.relayConnections.keys(): del main.relayConnections[self.addr] else: diff --git a/core/server.py b/core/server.py index 94af7a0..6b7a25d 100644 --- a/core/server.py +++ b/core/server.py @@ -1,8 +1,8 @@ -from twisted.internet.protocol import Protocol, Factory, ClientFactory -import main -from utils.logging.log import * +from twisted.internet.protocol import ClientFactory, Factory, Protocol +import main from core.parser import parseCommand +from utils.logging.log import * class Server(Protocol): @@ -34,7 +34,10 @@ class Server(Protocol): def connectionLost(self, reason): self.authed = False - log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) + log( + "Connection lost from %s:%s -- %s" + % (self.addr.host, self.addr.port, reason.getErrorMessage()) + ) if self.addr in main.connections.keys(): del main.connections[self.addr] else: diff --git a/main.py b/main.py index 473fd3e..103c15a 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,9 @@ import json import pickle -from redis import StrictRedis -from string import digits from os import urandom +from string import digits + +from redis import StrictRedis from utils.logging.log import * @@ -106,5 +107,9 @@ def initConf(): def initMain(): global r, g initConf() - r = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]) # Ephemeral - flushed on quit - g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent + r = StrictRedis( + unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] + ) # Ephemeral - flushed on quit + g = StrictRedis( + unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"] + ) # Persistent diff --git a/modules/alias.py b/modules/alias.py index 1961ec6..6417e29 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -1,7 +1,8 @@ -import main import random import re +import main + def generate_password(): return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) diff --git a/modules/chankeep.py b/modules/chankeep.py index ec43b0f..fdbe974 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -1,11 +1,13 @@ -import main -from utils.logging.log import * -from utils.logging.debug import * from copy import deepcopy from math import ceil -import modules.provision + from twisted.internet.threads import deferToThread +import main +import modules.provision +from utils.logging.debug import * +from utils.logging.log import * + def allRelaysActive(net): relayNum = len(main.network[net].relays.keys()) @@ -46,7 +48,10 @@ def emptyChanAllocate(net, flist, relay, new): if toalloc > sum(chanfree[0].values()): correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) # print("correction", correction) - warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) + warn( + "Ran out of channel spaces, provisioning additional %i relays for %s" + % (correction, net) + ) # newNums = modules.provision.provisionMultipleRelays(net, correction) return False for i in chanfree[0].keys(): @@ -99,12 +104,18 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): listinfo = minifyChans(net, listinfo) if not listinfo: return - if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels + if ( + relay <= main.config["ChanKeep"]["SigSwitch"] + ): # we can cover all of the channels coverAll = True - elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels + elif ( + relay > main.config["ChanKeep"]["SigSwitch"] + ): # we cannot cover all of the channels coverAll = False if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: - error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) + error( + "Network %s is too big to cover: %i relays required" % (net, sigrelay) + ) return if coverAll: needed = relay - len(main.network[net].relays.keys()) diff --git a/modules/counters.py b/modules/counters.py index a088cc3..5d51a52 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -1,6 +1,7 @@ -import main from twisted.internet.task import LoopingCall +import main + def event(name, eventType): if not "local" in main.counters.keys(): diff --git a/modules/monitor.py b/modules/monitor.py index f174a80..e1c16a7 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -2,10 +2,9 @@ from copy import deepcopy from json import dumps import main -from core.relay import sendRelayNotification from core.logstash import sendLogstashNotification -from modules import userinfo -from modules import regproc +from core.relay import sendRelayNotification +from modules import regproc, userinfo from utils.dedup import dedup order = [ @@ -68,7 +67,9 @@ def parsemeta(numName, c): ) -def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen +def event( + numName, c +): # yes I'm using a short variable because otherwise it goes off the screen if dedup(numName, c): return diff --git a/modules/network.py b/modules/network.py index 21f1753..5652120 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,14 +1,15 @@ -from twisted.internet.ssl import DefaultOpenSSLContextFactory import json +from twisted.internet import reactor +from twisted.internet.ssl import DefaultOpenSSLContextFactory + +import main +from core.bot import IRCBot, IRCBotFactory from modules import alias from modules.chankeep import nukeNetwork from modules.regproc import needToRegister -from twisted.internet import reactor -from core.bot import IRCBot, IRCBotFactory -import main -from utils.logging.log import * from utils.get import getRelay +from utils.logging.log import * class Network: @@ -77,7 +78,9 @@ class Network: # e.g. freenode1 for the first relay on freenode network keyFN = main.certPath + main.config["Key"] certFN = main.certPath + main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) + contextFactory = DefaultOpenSSLContextFactory( + keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace") + ) bot = IRCBotFactory(self.net, num) # host, port = self.relays[num]["host"], self.relays[num]["port"] host, port = getRelay(num) diff --git a/modules/provision.py b/modules/provision.py index 1f17a20..9e93236 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,11 +1,25 @@ +from twisted.internet import reactor + import main +import modules.regproc from core.bot import deliverRelayCommands from utils.logging.log import * -from twisted.internet import reactor -import modules.regproc -def provisionUserNetworkData(num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password): +def provisionUserNetworkData( + num, + nick, + altnick, + ident, + realname, + emails, + network, + host, + port, + security, + auth, + password, +): print("nick", nick) print("altnick", altnick) print("emails", emails) @@ -16,17 +30,25 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, emails, networ stage2commands["status"] = [] commands["controlpanel"] = [] user = nick.lower() - commands["controlpanel"].append("AddUser %s %s" % (user, main.config["Relay"]["Password"])) + commands["controlpanel"].append( + "AddUser %s %s" % (user, main.config["Relay"]["Password"]) + ) commands["controlpanel"].append("AddNetwork %s %s" % (user, network)) commands["controlpanel"].append("Set Nick %s %s" % (user, nick)) commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick)) commands["controlpanel"].append("Set Ident %s %s" % (user, ident)) commands["controlpanel"].append("Set RealName %s %s" % (user, realname)) if security == "ssl": - commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me - commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port)) + commands["controlpanel"].append( + "SetNetwork TrustAllCerts %s %s true" % (user, network) + ) # Don't judge me + commands["controlpanel"].append( + "AddServer %s %s %s +%s" % (user, network, host, port) + ) elif security == "plain": - commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port)) + commands["controlpanel"].append( + "AddServer %s %s %s %s" % (user, network, host, port) + ) if not main.config["ConnectOnCreate"]: stage2commands["status"].append("Disconnect") if main.config["Toggles"]["CycleChans"]: diff --git a/modules/regproc.py b/modules/regproc.py index ce0e5a2..353c5e6 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -1,10 +1,12 @@ -import main -from modules import provision -from utils.logging.log import * -from utils.logging.debug import * from copy import deepcopy from random import choice +import main +from modules import provision +from utils.logging.debug import * +from utils.logging.log import * + + def needToRegister(net): # Check if the network does not support authentication networkObj = main.network[net] @@ -51,7 +53,7 @@ def substitute(net, num, token=None): nickname = alias["nick"] username = nickname + "/" + net password = main.network[net].aliases[num]["password"] - #inst["email"] = inst["email"].replace("{nickname}", nickname) + # inst["email"] = inst["email"].replace("{nickname}", nickname) for i in inst.keys(): if not isinstance(inst[i], str): continue @@ -112,8 +114,12 @@ def enableAuthentication(net, num): auth = obj.auth password = obj.aliases[num]["password"] uname = main.alias[num]["nick"] + "/" + net - provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth - main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") + provision.provisionAuthenticationData( + num, nick, net, security, auth, password + ) # Set up for auth + main.IRCPool[net + str(num)].msg( + main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump" + ) if selectInst(net)["check"] == False: confirmRegistration(net, num) @@ -129,10 +135,14 @@ def registerTest(c): if not main.IRCPool[name]._negativePass == True: if c["type"] == "query" and c["nick"] == sinst["entity"]: if sinst["checknegativemsg"] in c["msg"]: - confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back + confirmRegistration( + c["net"], c["num"], negativepass=False + ) # Not passed negative check, report back return if sinst["checkendnegative"] in c["msg"]: - confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back + confirmRegistration( + c["net"], c["num"], negativepass=True + ) # Passed the negative check, report back return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: diff --git a/modules/userinfo.py b/modules/userinfo.py index 1722cb3..94c0833 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,9 +1,10 @@ -from twisted.internet.threads import deferToThread from string import digits +from twisted.internet.threads import deferToThread + import main -from utils.logging.log import * from utils.logging.debug import debug, trace +from utils.logging.log import * from utils.parsing import parsen @@ -145,7 +146,9 @@ def delUser(name, channel, nick, user): p.srem(namespace, nick) if channels == {channel.encode()}: # can we only see them on this channel? p.delete(chanspace) # remove channel tracking entry - p.hdel("live.prefix." + name + "." + channel, nick) # remove prefix tracking entry + p.hdel( + "live.prefix." + name + "." + channel, nick + ) # remove prefix tracking entry p.hdel(mapspace, nick) # remove nick mapping entry if user: p.srem(gnamespace, user) # remove global userinfo entry @@ -170,7 +173,10 @@ def getUserByNick(name, nick): if main.r.hexists(mapspace, nick): return main.r.hget(mapspace, nick) else: - warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) + warn( + "Entry doesn't exist: %s on %s - attempting auxiliary lookup" + % (nick, mapspace) + ) # return False # legacy code below - remove when map is reliable usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999) @@ -200,10 +206,18 @@ def renameUser(name, oldnick, olduser, newnick, newuser): p.sadd("live.who." + name + "." + i, newnick) p.hdel(mapspace, oldnick) p.hset(mapspace, newnick, newuser) - if main.r.exists("live.prefix." + name + "." + i): # if there's a prefix entry for the channel - if main.r.hexists("live.prefix." + name + "." + i, oldnick): # if the old nick is in it - mode = main.r.hget("live.prefix." + name + "." + i, oldnick) # retrieve old modes - p.hset("live.prefix." + name + "." + i, newnick, mode) # set old modes to new nickname + if main.r.exists( + "live.prefix." + name + "." + i + ): # if there's a prefix entry for the channel + if main.r.hexists( + "live.prefix." + name + "." + i, oldnick + ): # if the old nick is in it + mode = main.r.hget( + "live.prefix." + name + "." + i, oldnick + ) # retrieve old modes + p.hset( + "live.prefix." + name + "." + i, newnick, mode + ) # set old modes to new nickname if main.r.exists(chanspace): p.rename(chanspace, newchanspace) else: diff --git a/threshold b/threshold index dc186b4..b7c6f78 100755 --- a/threshold +++ b/threshold @@ -1,14 +1,14 @@ #!/usr/bin/env python -from twisted.internet import reactor -from twisted.internet.ssl import DefaultOpenSSLContextFactory import sys -from signal import signal, SIGINT - +from codecs import getwriter # fix printing odd shit to the terminal +from signal import SIGINT, signal # from twisted.python import log # from sys import stdout # log.startLogging(stdout) -from sys import stdout, stderr # Import again because we want to override -from codecs import getwriter # fix printing odd shit to the terminal +from sys import stderr, stdout # Import again because we want to override + +from twisted.internet import reactor +from twisted.internet.ssl import DefaultOpenSSLContextFactory stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using @@ -23,11 +23,11 @@ if "--debug" in sys.argv: # yes really main.config["Debug"] = True if "--trace" in sys.argv: main.config["Trace"] = True -from utils.logging.log import * -from utils.loaders.command_loader import loadCommands -from core.server import Server, ServerFactory -from core.relay import Relay, RelayFactory import modules.counters +from core.relay import Relay, RelayFactory +from core.server import Server, ServerFactory +from utils.loaders.command_loader import loadCommands +from utils.logging.log import * loadCommands() import core.logstash @@ -46,14 +46,20 @@ if __name__ == "__main__": ), interface=main.config["Listener"]["Address"], ) - log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) + log( + "Threshold running with SSL on %s:%s" + % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]) + ) else: reactor.listenTCP( main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"], ) - log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) + log( + "Threshold running on %s:%s" + % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]) + ) if main.config["RelayAPI"]["Enabled"]: relay = RelayFactory() if main.config["RelayAPI"]["UseSSL"] == True: @@ -76,7 +82,10 @@ if __name__ == "__main__": relay, interface=main.config["RelayAPI"]["Address"], ) - log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) + log( + "Threshold relay running on %s:%s" + % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]) + ) for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() diff --git a/utils/cleanup.py b/utils/cleanup.py index 83ed661..51eaa05 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -1,8 +1,10 @@ -import main +import sys + from twisted.internet import reactor + +import main from utils.logging.debug import debug from utils.logging.log import * -import sys def handler(sig, frame): diff --git a/utils/dedup.py b/utils/dedup.py index dc8d4c3..89e4bee 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -1,7 +1,9 @@ -from datetime import datetime -from csiphash import siphash24 from copy import deepcopy +from datetime import datetime from json import dumps + +from csiphash import siphash24 + import main from utils.logging.debug import debug @@ -10,16 +12,24 @@ def dedup(numName, b): c = deepcopy(b) if "ts" in c.keys(): del c["ts"] - c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]] + c["approxtime"] = str(datetime.utcnow().timestamp())[ + : main.config["Tweaks"]["DedupPrecision"] + ] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] - isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) + isDuplicate = any( + castHash in main.lastEvents[x] + for x in main.lastEvents.keys() + if not x == numName + ) if isDuplicate: debug("Duplicate: %s" % (c)) return True if numName in main.lastEvents.keys(): main.lastEvents[numName].insert(0, castHash) - main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] + main.lastEvents[numName] = main.lastEvents[numName][ + 0 : main.config["Tweaks"]["MaxHash"] + ] else: main.lastEvents[numName] = [castHash] return False diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 9c4095b..19517ec 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,10 +1,9 @@ from os import listdir +import commands +from main import CommandMap from utils.logging.debug import debug from utils.logging.log import * -import commands - -from main import CommandMap def loadCommands(allowDup=False): @@ -15,11 +14,15 @@ def loadCommands(allowDup=False): try: module = __import__("commands.%s" % commandName) if not commandName in CommandMap: - CommandMap[commandName] = getattr(getattr(module, commandName), className) + CommandMap[commandName] = getattr( + getattr(module, commandName), className + ) debug("Registered command: %s" % commandName) else: if allowDup: - CommandMap[commandName] = getattr(getattr(module, commandName), className) + CommandMap[commandName] = getattr( + getattr(module, commandName), className + ) debug("Registered command: %s" % commandName) error("Duplicate command: %s" % (commandName)) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index f5219ac..64b0618 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -1,12 +1,11 @@ -from os import listdir -from importlib import reload import sys +from importlib import reload +from os import listdir +import commands +from main import CommandMap from utils.logging.debug import debug from utils.logging.log import * -import commands - -from main import CommandMap def loadSingle(commandName): @@ -15,7 +14,9 @@ def loadSingle(commandName): try: if commandName in CommandMap.keys(): reload(sys.modules["commands." + commandName]) - CommandMap[commandName] = getattr(sys.modules["commands." + commandName], className) + CommandMap[commandName] = getattr( + sys.modules["commands." + commandName], className + ) debug("Reloaded command: %s" % commandName) return "RELOAD" module = __import__("commands.%s" % commandName) diff --git a/utils/logging/debug.py b/utils/logging/debug.py index 80651f7..0c73174 100644 --- a/utils/logging/debug.py +++ b/utils/logging/debug.py @@ -1,5 +1,6 @@ import main + # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main # to read the setting def debug(*data): From 3229d9b806f0cce28328ae202d1b7b3526fba4cc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:01 +0100 Subject: [PATCH 245/394] Revert "Reformat project" This reverts commit 64e3e1160aa76d191740342ab3edc68807f890fb. --- .pre-commit-config.yaml | 24 +++---- commands/admall.py | 4 +- commands/alias.py | 7 +- commands/all.py | 9 +-- commands/allc.py | 12 +--- commands/authcheck.py | 12 +--- commands/auto.py | 24 ++----- commands/blacklist.py | 7 +- commands/chans.py | 4 +- commands/cmd.py | 4 +- commands/confirm.py | 9 +-- commands/disable.py | 4 +- commands/dist.py | 7 +- commands/email.py | 52 +++------------ commands/enable.py | 4 +- commands/exec.py | 4 +- commands/help.py | 4 +- commands/join.py | 4 +- commands/list.py | 4 +- commands/load.py | 4 +- commands/loadmod.py | 4 +- commands/logout.py | 4 +- commands/mod.py | 11 +-- commands/msg.py | 9 +-- commands/network.py | 14 ++-- commands/part.py | 4 +- commands/pass.py | 4 +- commands/pending.py | 14 +--- commands/recheckauth.py | 4 +- commands/reg.py | 4 +- commands/relay.py | 21 ++---- commands/save.py | 4 +- commands/stats.py | 7 +- commands/swho.py | 4 +- commands/token.py | 10 +-- commands/users.py | 4 +- commands/who.py | 4 +- core/bot.py | 115 ++++++++++++-------------------- core/logstash.py | 5 +- core/parser.py | 4 +- core/relay.py | 19 ++---- core/server.py | 11 ++- main.py | 13 ++-- modules/alias.py | 3 +- modules/chankeep.py | 27 +++----- modules/counters.py | 3 +- modules/monitor.py | 9 ++- modules/network.py | 15 ++--- modules/provision.py | 36 ++-------- modules/regproc.py | 26 +++----- modules/userinfo.py | 30 +++------ threshold | 35 ++++------ utils/cleanup.py | 6 +- utils/dedup.py | 20 ++---- utils/loaders/command_loader.py | 13 ++-- utils/loaders/single_loader.py | 13 ++-- utils/logging/debug.py | 1 - 57 files changed, 214 insertions(+), 519 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11c38e8..98fa1cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,13 @@ repos: - - repo: https://github.com/psf/black - rev: 22.6.0 +- repo: https://github.com/ambv/black + rev: 22.3.0 hooks: - id: black - exclude: ^core/migrations - - repo: https://github.com/PyCQA/isort - rev: 5.10.1 - hooks: - - id: isort - - repo: https://github.com/PyCQA/flake8 + args: + - --line-length=120 +- repo: https://gitlab.com/pycqa/flake8 rev: 4.0.1 hooks: - - id: flake8 - args: [--max-line-length=88] - exclude: ^core/migrations - - repo: https://github.com/thibaudcolas/curlylint - rev: v0.13.1 - hooks: - - id: curlylint - files: \.(html|sls)$ + - id: flake8 + args: + - "--max-line-length=120" diff --git a/commands/admall.py b/commands/admall.py index cb0c009..8d74545 100644 --- a/commands/admall.py +++ b/commands/admall.py @@ -6,9 +6,7 @@ class AdmallCommand: def __init__(self, *args): self.admall(*args) - def admall( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def admall(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 2: for i in main.network.keys(): diff --git a/commands/alias.py b/commands/alias.py index 5bc12bf..d04cfdf 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -1,6 +1,5 @@ -from yaml import dump - import main +from yaml import dump from modules import alias @@ -8,9 +7,7 @@ class AliasCommand: def __init__(self, *args): self.alias(*args) - def alias( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def alias(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: info(dump(main.alias)) diff --git a/commands/all.py b/commands/all.py index 16a6721..a2994e9 100644 --- a/commands/all.py +++ b/commands/all.py @@ -6,9 +6,7 @@ class AllCommand: def __init__(self, *args): self.all(*args) - def all( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 2: for i in main.network.keys(): @@ -17,10 +15,7 @@ class AllCommand: net = main.network[i].relays[x]["net"] alias = main.alias[x]["nick"] commands = {spl[1]: [" ".join(spl[2:])]} - success( - "Sending commands to relay %s as user %s" - % (num, alias + "/" + net) - ) + success("Sending commands to relay %s as user %s" % (num, alias + "/" + net)) deliverRelayCommands(num, commands, user=alias + "/" + net) return else: diff --git a/commands/allc.py b/commands/allc.py index 01698e5..7a7b5fd 100644 --- a/commands/allc.py +++ b/commands/allc.py @@ -6,9 +6,7 @@ class AllcCommand: def __init__(self, *args): self.allc(*args) - def allc( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 4: targets = [] @@ -22,8 +20,7 @@ class AllcCommand: [ targets.append((i, x)) for x in main.alias.keys() - if main.alias[x]["nick"] == spl[2] - and x in main.network[i].aliases.keys() + if main.alias[x]["nick"] == spl[2] and x in main.network[i].aliases.keys() ] else: incUsage("allc") @@ -36,10 +33,7 @@ class AllcCommand: num = i[1] alias = main.alias[num]["nick"] commands = {spl[3]: [" ".join(spl[4:])]} - success( - "Sending commands to relay %i as user %s" - % (num, alias + "/" + net) - ) + success("Sending commands to relay %i as user %s" % (num, alias + "/" + net)) deliverRelayCommands(num, commands, user=alias + "/" + net) return else: diff --git a/commands/authcheck.py b/commands/authcheck.py index 7c41dd1..26cc14e 100644 --- a/commands/authcheck.py +++ b/commands/authcheck.py @@ -5,9 +5,7 @@ class AuthcheckCommand: def __init__(self, *args): self.authcheck(*args) - def authcheck( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def authcheck(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: results = [] @@ -15,9 +13,7 @@ class AuthcheckCommand: num = main.IRCPool[i].num net = main.IRCPool[i].net if not main.IRCPool[i].authenticated: - results.append( - "%s - %s: %s" % (net, num, main.alias[num]["nick"]) - ) + results.append("%s - %s: %s" % (net, num, main.alias[num]["nick"])) info("\n".join(results)) return elif length == 2: @@ -31,9 +27,7 @@ class AuthcheckCommand: if not net == spl[1]: continue if not main.IRCPool[i].authenticated: - results.append( - "%s - %s: %s" % (net, num, main.alias[num]["nick"]) - ) + results.append("%s - %s: %s" % (net, num, main.alias[num]["nick"])) info("\n".join(results)) return else: diff --git a/commands/auto.py b/commands/auto.py index 378a102..ffc6df5 100644 --- a/commands/auto.py +++ b/commands/auto.py @@ -6,9 +6,7 @@ class AutoCommand: def __init__(self, *args): self.auto(*args) - def auto( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def auto(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: for i in main.network.keys(): @@ -16,15 +14,9 @@ class AutoCommand: info("Skipping %s - first relay exists" % i) else: num, alias = main.network[i].add_relay(1) - success( - "Successfully created first relay on network %s with alias %s" - % (i, alias) - ) + success("Successfully created first relay on network %s with alias %s" % (i, alias)) provision.provisionRelay(num, i) - success( - "Started provisioning network %s on first relay for alias %s" - % (i, alias) - ) + success("Started provisioning network %s on first relay for alias %s" % (i, alias)) main.saveConf("network") return elif length == 2: @@ -35,15 +27,9 @@ class AutoCommand: failure("First relay exists on %s" % spl[1]) return num, alias = main.network[spl[1]].add_relay(1) - success( - "Successfully created relay %i on network %s with alias %s" - % (num, spl[1], alias) - ) + success("Successfully created relay %i on network %s with alias %s" % (num, spl[1], alias)) provision.provisionRelay(num, spl[1]) - success( - "Started provisioning network %s on relay %s for alias %s" - % (spl[1], num, alias) - ) + success("Started provisioning network %s on relay %s for alias %s" % (spl[1], num, alias)) main.saveConf("network") return else: diff --git a/commands/blacklist.py b/commands/blacklist.py index 858da28..42684a6 100644 --- a/commands/blacklist.py +++ b/commands/blacklist.py @@ -1,15 +1,12 @@ -from yaml import dump - import main +from yaml import dump class BlacklistCommand: def __init__(self, *args): self.blacklist(*args) - def blacklist( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def blacklist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: info(dump(main.blacklist)) diff --git a/commands/chans.py b/commands/chans.py index 8b820ff..1ccf710 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -6,9 +6,7 @@ class ChansCommand: def __init__(self, *args): self.chans(*args) - def chans( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def chans(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if len(spl) < 2: incUsage("chans") diff --git a/commands/cmd.py b/commands/cmd.py index f2706db..babdb2d 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -6,9 +6,7 @@ class CmdCommand: def __init__(self, *args): self.cmd(*args) - def cmd( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def cmd(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 4: if not spl[1].isdigit(): diff --git a/commands/confirm.py b/commands/confirm.py index 53a70a2..86d657c 100644 --- a/commands/confirm.py +++ b/commands/confirm.py @@ -6,9 +6,7 @@ class ConfirmCommand: def __init__(self, *args): self.confirm(*args) - def confirm( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def confirm(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 4: if not spl[1] in main.network.keys(): @@ -21,10 +19,7 @@ class ConfirmCommand: failure("No such relay on %s: %s" % (spl[1], spl[2])) return regproc.confirmAccount(spl[1], int(spl[2]), spl[3]) - success( - "Requested confirmation on %s - %s with token %s" - % (spl[1], spl[2], spl[3]) - ) + success("Requested confirmation on %s - %s with token %s" % (spl[1], spl[2], spl[3])) return else: incUsage("confirm") diff --git a/commands/disable.py b/commands/disable.py index 9a56fb4..9c52234 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -6,9 +6,7 @@ class DisableCommand: def __init__(self, *args): self.disable(*args) - def disable( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def disable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: if not spl[1] in main.network.keys(): diff --git a/commands/dist.py b/commands/dist.py index 7e94c4e..2348e29 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -1,15 +1,12 @@ -from subprocess import PIPE, run - import main +from subprocess import run, PIPE class DistCommand: def __init__(self, *args): self.dist(*args) - def dist( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def dist(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if main.config["Dist"]["Enabled"]: rtrn = run([main.config["Dist"]["File"]], shell=True, stdout=PIPE) diff --git a/commands/email.py b/commands/email.py index 3b78931..4817988 100644 --- a/commands/email.py +++ b/commands/email.py @@ -1,15 +1,12 @@ -from yaml import dump - import main +from yaml import dump class EmailCommand: def __init__(self, *args): self.email(*args) - def email( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def email(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 4: if spl[1] == "add": @@ -22,16 +19,10 @@ class EmailCommand: return if not domain in main.irc["_"]["domains"]: main.irc["_"]["domains"].append(domain) - success( - "Successfully added domain %s to default config" - % domain - ) + success("Successfully added domain %s to default config" % domain) main.saveConf("irc") else: - failure( - "Domain already exists in default config: %s" - % domain - ) + failure("Domain already exists in default config: %s" % domain) return elif spl[1] == "del": if not spl[2].isdigit(): @@ -41,16 +32,10 @@ class EmailCommand: if domain in main.irc["_"]["domains"]: main.irc["_"]["domains"].remove(domain) - success( - "Successfully removed domain %s to default config" - % domain - ) + success("Successfully removed domain %s to default config" % domain) main.saveConf("irc") else: - failure( - "Domain does not exist in default config: %s" - % domain - ) + failure("Domain does not exist in default config: %s" % domain) return else: @@ -62,15 +47,10 @@ class EmailCommand: if not spl[3] in main.alias[num]["emails"]: main.alias[num]["emails"].append(spl[3]) main.saveConf("alias") - success( - "Successfully added email %s to alias %i" - % (spl[3], num) - ) + success("Successfully added email %s to alias %i" % (spl[3], num)) return else: - failure( - "Email already exists in alias %i: %s" % (num, spl[3]) - ) + failure("Email already exists in alias %i: %s" % (num, spl[3])) return elif spl[1] == "del": if not num in main.alias.keys(): @@ -79,15 +59,10 @@ class EmailCommand: if spl[3] in main.alias[num]["emails"]: main.alias[num]["emails"].remove(spl[3]) main.saveConf("alias") - success( - "Successfully removed email %s from alias %i" - % (spl[3], num) - ) + success("Successfully removed email %s from alias %i" % (spl[3], num)) return else: - failure( - "Email does not exist in alias %i: %s" % (spl[3], num) - ) + failure("Email does not exist in alias %i: %s" % (spl[3], num)) return elif length == 2: if spl[1] == "list": @@ -99,12 +74,7 @@ class EmailCommand: elif length == 3: if spl[1] == "list": if spl[2] == "domain": - filtered = { - f"{k}:{k2}": v2 - for k, v in main.irc.items() - for k2, v2 in v.items() - if k2 == "domains" - } + filtered = {f"{k}:{k2}":v2 for k,v in main.irc.items() for k2,v2 in v.items() if k2 == "domains"} info(dump(filtered)) return else: diff --git a/commands/enable.py b/commands/enable.py index 1cfb0e4..439b26c 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -6,9 +6,7 @@ class EnableCommand: def __init__(self, *args): self.enable(*args) - def enable( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def enable(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: if not spl[1] in main.network.keys(): diff --git a/commands/exec.py b/commands/exec.py index 6f7a924..c2a5598 100644 --- a/commands/exec.py +++ b/commands/exec.py @@ -5,9 +5,7 @@ class ExecCommand: def __init__(self, *args): self.exec(*args) - def exec( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def exec(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length > 1: try: diff --git a/commands/help.py b/commands/help.py index 0ca3ff7..6dcf937 100644 --- a/commands/help.py +++ b/commands/help.py @@ -5,9 +5,7 @@ class HelpCommand: def __init__(self, *args): self.help(*args) - def help( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def help(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: helpMap = [] for i in main.help.keys(): diff --git a/commands/join.py b/commands/join.py index d6147a0..e6cbc20 100644 --- a/commands/join.py +++ b/commands/join.py @@ -6,9 +6,7 @@ class JoinCommand: def __init__(self, *args): self.join(*args) - def join( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def join(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: if not spl[1] in main.network.keys(): diff --git a/commands/list.py b/commands/list.py index 7fbc616..217b051 100644 --- a/commands/list.py +++ b/commands/list.py @@ -5,9 +5,7 @@ class ListCommand: def __init__(self, *args): self.list(*args) - def list( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def list(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: for i in main.network.keys(): diff --git a/commands/load.py b/commands/load.py index b62da89..31e66f4 100644 --- a/commands/load.py +++ b/commands/load.py @@ -5,9 +5,7 @@ class LoadCommand: def __init__(self, *args): self.load(*args) - def load( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def load(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: if spl[1] in main.filemap.keys(): diff --git a/commands/loadmod.py b/commands/loadmod.py index e440b58..2923fd7 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -6,9 +6,7 @@ class LoadmodCommand: def __init__(self, *args): self.loadmod(*args) - def loadmod( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def loadmod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: rtrn = loadSingle(spl[1]) diff --git a/commands/logout.py b/commands/logout.py index 06bd437..61cb834 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -5,9 +5,7 @@ class LogoutCommand: def __init__(self, *args): self.logout(*args) - def logout( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: obj.authed = False success("Logged out") diff --git a/commands/mod.py b/commands/mod.py index 94d3a7a..10a19e1 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,6 +1,5 @@ -from yaml import dump - import main +from yaml import dump class ModCommand: @@ -8,9 +7,7 @@ class ModCommand: def __init__(self, *args): self.mod(*args) - def mod( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def mod(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 4: if not spl[1] in main.network.keys(): @@ -24,9 +21,7 @@ class ModCommand: return main.saveConf("network") - success( - "Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]) - ) + success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) return # Find a better way to do this # elif length == 6: diff --git a/commands/msg.py b/commands/msg.py index 754176c..d92494a 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -5,9 +5,7 @@ class MsgCommand: def __init__(self, *args): self.msg(*args) - def msg( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def msg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length >= 5: if not spl[1] in main.network.keys(): @@ -22,10 +20,7 @@ class MsgCommand: if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels: info("Bot not on channel: %s" % spl[3]) main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:])) - success( - "Sent %s to %s on relay %s on network %s" - % (" ".join(spl[4:]), spl[3], spl[2], spl[1]) - ) + success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1])) return else: incUsage("msg") diff --git a/commands/network.py b/commands/network.py index 13b23bd..4a6bdf2 100644 --- a/commands/network.py +++ b/commands/network.py @@ -1,18 +1,14 @@ -from string import digits - -from yaml import dump - import main +from yaml import dump from modules.network import Network +from string import digits class NetworkCommand: def __init__(self, *args): self.network(*args) - def network( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def network(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 7: if spl[1] == "add": @@ -32,9 +28,7 @@ class NetworkCommand: failure("Auth must be sasl, ns or none, not %s" % spl[5]) return else: - main.network[spl[2]] = Network( - spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower() - ) + main.network[spl[2]] = Network(spl[2], spl[3], int(spl[4]), spl[5].lower(), spl[6].lower()) success("Successfully created network: %s" % spl[2]) main.saveConf("network") return diff --git a/commands/part.py b/commands/part.py index 5a2de7e..249c32a 100644 --- a/commands/part.py +++ b/commands/part.py @@ -5,9 +5,7 @@ class PartCommand: def __init__(self, *args): self.part(*args) - def part( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def part(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 4: if not spl[1] in main.network.keys(): diff --git a/commands/pass.py b/commands/pass.py index 4b9e601..e37a51e 100644 --- a/commands/pass.py +++ b/commands/pass.py @@ -5,9 +5,7 @@ class PassCommand: def __init__(self, *args): self.password(*args) - def password( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def password(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: info("You are already authenticated") return diff --git a/commands/pending.py b/commands/pending.py index 5f962dd..b65f2ad 100644 --- a/commands/pending.py +++ b/commands/pending.py @@ -5,19 +5,14 @@ class PendingCommand: def __init__(self, *args): self.pending(*args) - def pending( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def pending(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: results = [] for i in main.network.keys(): for x in main.network[i].relays.keys(): if not main.network[i].relays[x]["registered"]: - results.append( - "%s: confirm %s %s [code]" - % (main.alias[x]["nick"], i, x) - ) + results.append("%s: confirm %s %s [code]" % (main.alias[x]["nick"], i, x)) info("\n".join(results)) return elif length == 2: @@ -27,10 +22,7 @@ class PendingCommand: results = [] for x in main.network[spl[1]].relays.keys(): if not main.network[spl[1]].relays[x]["registered"]: - results.append( - "%s: confirm %s %s [code]" - % (main.alias[x]["nick"], spl[1], x) - ) + results.append("%s: confirm %s %s [code]" % (main.alias[x]["nick"], spl[1], x)) info("\n".join(results)) return else: diff --git a/commands/recheckauth.py b/commands/recheckauth.py index 6cea741..ab4b5b0 100644 --- a/commands/recheckauth.py +++ b/commands/recheckauth.py @@ -5,9 +5,7 @@ class RecheckauthCommand: def __init__(self, *args): self.recheckauth(*args) - def recheckauth( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def recheckauth(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: results = [] diff --git a/commands/reg.py b/commands/reg.py index fb3794a..41bb5f0 100644 --- a/commands/reg.py +++ b/commands/reg.py @@ -6,9 +6,7 @@ class RegCommand: def __init__(self, *args): self.reg(*args) - def reg( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def reg(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: if not spl[1] in main.network.keys(): diff --git a/commands/relay.py b/commands/relay.py index 7bfc3d8..34bbd70 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -1,24 +1,18 @@ -from yaml import dump - import main +from yaml import dump class RelayCommand: def __init__(self, *args): self.relay(*args) - def relay( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def relay(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 3: if spl[1] == "add": if spl[2] in main.network.keys(): id, alias = main.network[spl[2]].add_relay() - success( - "Successfully created relay %s on network %s with alias %s" - % (str(id), spl[2], alias) - ) + success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) main.saveConf("network") return else: @@ -41,10 +35,7 @@ class RelayCommand: failure("Must be a number, not %s" % spl[3]) return id, alias = main.network[spl[2]].add_relay(int(spl[3])) - success( - "Successfully created relay %s on network %s with alias %s" - % (str(id), spl[2], alias) - ) + success("Successfully created relay %s on network %s with alias %s" % (str(id), spl[2], alias)) main.saveConf("network") return else: @@ -61,9 +52,7 @@ class RelayCommand: failure("No such relay: %s on network %s" % (spl[3], spl[2])) return main.network[spl[2]].delete_relay(int(spl[3])) - success( - "Successfully deleted relay %s on network %s" % (spl[3], spl[2]) - ) + success("Successfully deleted relay %s on network %s" % (spl[3], spl[2])) main.saveConf("network") return else: diff --git a/commands/save.py b/commands/save.py index 0dd22b6..12624cd 100644 --- a/commands/save.py +++ b/commands/save.py @@ -5,9 +5,7 @@ class SaveCommand: def __init__(self, *args): self.save(*args) - def save( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def save(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: if spl[1] in main.filemap.keys(): diff --git a/commands/stats.py b/commands/stats.py index ab6e686..a396f12 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,17 +1,14 @@ -from string import digits - import main import modules.counters as count import modules.userinfo as userinfo +from string import digits class StatsCommand: def __init__(self, *args): self.stats(*args) - def stats( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def stats(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: stats = [] diff --git a/commands/swho.py b/commands/swho.py index aa3a78c..18dbc12 100644 --- a/commands/swho.py +++ b/commands/swho.py @@ -5,9 +5,7 @@ class SwhoCommand: def __init__(self, *args): self.swho(*args) - def swho( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def swho(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: if not spl[1] in main.network.keys(): diff --git a/commands/token.py b/commands/token.py index 8d95394..7d38eac 100644 --- a/commands/token.py +++ b/commands/token.py @@ -1,17 +1,13 @@ -from uuid import uuid4 - -from yaml import dump - import main +from yaml import dump +from uuid import uuid4 class TokenCommand: def __init__(self, *args): self.token(*args) - def token( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def token(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: if spl[1] == "list": diff --git a/commands/users.py b/commands/users.py index 6ba7563..7d80e84 100644 --- a/commands/users.py +++ b/commands/users.py @@ -6,9 +6,7 @@ class UsersCommand: def __init__(self, *args): self.users(*args) - def users( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def users(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if len(spl) < 2: incUsage("users") diff --git a/commands/who.py b/commands/who.py index 8273ef4..e80f488 100644 --- a/commands/who.py +++ b/commands/who.py @@ -6,9 +6,7 @@ class WhoCommand: def __init__(self, *args): self.who(*args) - def who( - self, addr, authed, data, obj, spl, success, failure, info, incUsage, length - ): + def who(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 2: result = userinfo.getWho(spl[1]) diff --git a/core/bot.py b/core/bot.py index 31ff5d5..375703d 100644 --- a/core/bot.py +++ b/core/bot.py @@ -1,38 +1,45 @@ +from twisted.internet.protocol import ReconnectingClientFactory +from twisted.words.protocols.irc import IRCClient +from twisted.internet.defer import Deferred +from twisted.internet.task import LoopingCall +from twisted.internet import reactor, task +from twisted.words.protocols.irc import ( + symbolic_to_numeric, + numeric_to_symbolic, + lowDequote, + IRCBadMessage, +) + import sys +from string import digits +from random import randint from copy import deepcopy from datetime import datetime -from random import randint -from string import digits -from twisted.internet import reactor, task -from twisted.internet.defer import Deferred -from twisted.internet.protocol import ReconnectingClientFactory -from twisted.internet.ssl import DefaultOpenSSLContextFactory -from twisted.internet.task import LoopingCall -from twisted.words.protocols.irc import (IRCBadMessage, IRCClient, lowDequote, - numeric_to_symbolic, - symbolic_to_numeric) +from modules import userinfo +from modules import counters +from modules import monitor +from modules import chankeep +from modules import regproc -import main from core.relay import sendRelayNotification -from modules import chankeep, counters, monitor, regproc, userinfo from utils.dedup import dedup from utils.get import getRelay -from utils.logging.debug import * + +import main from utils.logging.log import * +from utils.logging.debug import * from utils.logging.send import * from utils.parsing import parsen +from twisted.internet.ssl import DefaultOpenSSLContextFactory + def deliverRelayCommands(num, relayCommands, user=None, stage2=None): keyFN = main.certPath + main.config["Key"] certFN = main.certPath + main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory( - keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace") - ) - bot = IRCBotFactory( - net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2 - ) + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) + bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) host, port = getRelay(num) rct = reactor.connectSSL(host, port, bot, contextFactory) @@ -158,9 +165,7 @@ class IRCBot(IRCClient): self.listAttempted = False # we asked for a list self.listSimple = False # after asking again we got the list, so use the simple # syntax from now on - self.wantList = ( - False # we want to send a LIST, but not all relays are active yet - ) + self.wantList = False # we want to send a LIST, but not all relays are active yet self.chanlimit = 0 self.prefix = {} self.servername = None @@ -192,14 +197,9 @@ class IRCBot(IRCClient): if not i in self.channels: if self.net in main.blacklist.keys(): if i in main.blacklist[self.net]: - debug( - "Not joining blacklisted channel %s on %s - %i" - % (i, self.net, self.num) - ) + debug("Not joining blacklisted channel %s on %s - %i" % (i, self.net, self.num)) continue - debug( - self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds" - ) + debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds") reactor.callLater(sleeptime, self.join, i) sleeptime += increment if sleeptime == 10: @@ -207,17 +207,11 @@ class IRCBot(IRCClient): increment = 0.7 increment += 0.1 else: - error( - "%s - Cannot join channel we are already on: %s - %i" - % (i, self.net, self.num) - ) + error("%s - Cannot join channel we are already on: %s - %i" % (i, self.net, self.num)) def checkChannels(self): if not chankeep.allRelaysActive(self.net): - debug( - "Skipping channel check as we have inactive relays: %s - %i" - % (self.net, self.num) - ) + debug("Skipping channel check as we have inactive relays: %s - %i" % (self.net, self.num)) return if self.net in main.TempChan.keys(): if self.num in main.TempChan[self.net].keys(): @@ -231,9 +225,7 @@ class IRCBot(IRCClient): cast["ts"] = str(datetime.now().isoformat()) # remove odd stuff - for i in list( - cast.keys() - ): # Make a copy of the .keys() as Python 3 cannot handle iterating over + for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] # remove server stuff @@ -367,9 +359,7 @@ class IRCBot(IRCClient): def irc_ERR_PASSWDMISMATCH(self, prefix, params): log("%s - %i: password mismatch as %s" % (self.net, self.num, self.username)) - sendAll( - "%s - %i: password mismatch as %s" % (self.net, self.num, self.username) - ) + sendAll("%s - %i: password mismatch as %s" % (self.net, self.num, self.username)) def _who(self, channel): d = Deferred() @@ -484,17 +474,12 @@ class IRCBot(IRCClient): debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num)) return if self.listAttempted: - debug( - "List request dropped, already asked for LIST - %s - %i" - % (self.net, self.num) - ) + debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num)) return else: self.listAttempted = True if self.listOngoing: - debug( - "LIST request dropped, already ongoing - %s - %i" % (self.net, self.num) - ) + debug("LIST request dropped, already ongoing - %s - %i" % (self.net, self.num)) return else: if nocheck: @@ -519,9 +504,7 @@ class IRCBot(IRCClient): self._tempList[1].append([channel, users, topic]) def irc_RPL_LISTEND(self, prefix, params): - if ( - not len(self._tempList[0]) > 0 - ): # there are no callbacks, can't do anything there + if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there debug("We didn't ask for this LIST, discarding") self._tempList[0].clear() self._tempList[1].clear() @@ -546,10 +529,7 @@ class IRCBot(IRCClient): else: if self.listRetried: self.listRetried = False - debug( - "List received after retry - defaulting to simple list syntax: %s - %i" - % (self.net, self.num) - ) + debug("List received after retry - defaulting to simple list syntax: %s - %i" % (self.net, self.num)) self.listSimple = True def got_list(self, listinfo): @@ -563,10 +543,7 @@ class IRCBot(IRCClient): name = self.net + "1" if main.IRCPool[name].wantList == True: main.IRCPool[name].list(nocheck=True) - debug( - "Asking for a list for %s after final relay %i connected" - % (self.net, self.num) - ) + debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) if self.num == 1: # Only one instance should do a list if self.chanlimit: if allRelays: @@ -583,9 +560,7 @@ class IRCBot(IRCClient): ]: # TODO: add check for register request sent, only send it once if main.config["AutoReg"]: if not self.authenticated: - self._regAttempt = reactor.callLater( - 5, regproc.registerAccount, self.net, self.num - ) + self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) # regproc.registerAccount(self.net, self.num) try: self.chanlimit = int(chanlimit) @@ -593,9 +568,7 @@ class IRCBot(IRCClient): warn("Invalid chanlimit: %s" % i) if self.chanlimit == 0: self.chanlimit = 200 # don't take the piss if it's not limited - if not regproc.needToRegister( - self.net - ): # if we need to register, only recheck on auth confirmation + if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation self.recheckList() def seed_prefix(self, prefix): @@ -696,9 +669,7 @@ class IRCBot(IRCClient): if negativepass == True: if self._negativePass == None: self._negativePass = True - debug( - "Positive registration check - %s - %i" % (self.net, self.num) - ) + debug("Positive registration check - %s - %i" % (self.net, self.num)) if sinst["ping"]: debug("Sending ping - %s - %i" % (self.net, self.num)) self.msg(sinst["entity"], sinst["pingmsg"]) @@ -754,9 +725,7 @@ class IRCBot(IRCClient): lc = self._getWho[channel] lc.stop() del self._getWho[channel] - userinfo.delChannels( - self.net, [channel] - ) # < we do not need to deduplicate this + userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this # log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so diff --git a/core/logstash.py b/core/logstash.py index 66cded9..17e8dfd 100644 --- a/core/logstash.py +++ b/core/logstash.py @@ -1,8 +1,7 @@ -import logging -from json import dumps, loads - import logstash +import logging +from json import dumps, loads import main from utils.logging.log import * diff --git a/core/parser.py b/core/parser.py index 8d7c635..1eb34c5 100644 --- a/core/parser.py +++ b/core/parser.py @@ -24,9 +24,7 @@ def parseCommand(addr, authed, data): failure("No text was sent") return if spl[0] in main.CommandMap.keys(): - main.CommandMap[spl[0]]( - addr, authed, data, obj, spl, success, failure, info, incUsage, length - ) + main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length) return incUsage(None) return diff --git a/core/relay.py b/core/relay.py index 7f63209..66a9830 100644 --- a/core/relay.py +++ b/core/relay.py @@ -1,7 +1,6 @@ -from copy import deepcopy +from twisted.internet.protocol import Protocol, Factory, ClientFactory from json import dumps, loads - -from twisted.internet.protocol import ClientFactory, Factory, Protocol +from copy import deepcopy import main from utils.logging.log import * @@ -115,13 +114,8 @@ class Relay(Protocol): def handleHello(self, parsed): if parsed["key"] in main.tokens.keys(): - if ( - parsed["hello"] == main.tokens[parsed["key"]]["hello"] - and main.tokens[parsed["key"]]["usage"] == "relay" - ): - self.sendMsg( - {"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]} - ) + if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay": + self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}) self.authed = True else: self.transport.loseConnection() @@ -136,10 +130,7 @@ class Relay(Protocol): def connectionLost(self, reason): self.authed = False - log( - "Relay connection lost from %s:%s -- %s" - % (self.addr.host, self.addr.port, reason.getErrorMessage()) - ) + log("Relay connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) if self.addr in main.relayConnections.keys(): del main.relayConnections[self.addr] else: diff --git a/core/server.py b/core/server.py index 6b7a25d..94af7a0 100644 --- a/core/server.py +++ b/core/server.py @@ -1,9 +1,9 @@ -from twisted.internet.protocol import ClientFactory, Factory, Protocol - +from twisted.internet.protocol import Protocol, Factory, ClientFactory import main -from core.parser import parseCommand from utils.logging.log import * +from core.parser import parseCommand + class Server(Protocol): def __init__(self, addr): @@ -34,10 +34,7 @@ class Server(Protocol): def connectionLost(self, reason): self.authed = False - log( - "Connection lost from %s:%s -- %s" - % (self.addr.host, self.addr.port, reason.getErrorMessage()) - ) + log("Connection lost from %s:%s -- %s" % (self.addr.host, self.addr.port, reason.getErrorMessage())) if self.addr in main.connections.keys(): del main.connections[self.addr] else: diff --git a/main.py b/main.py index 103c15a..473fd3e 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,8 @@ import json import pickle -from os import urandom -from string import digits - from redis import StrictRedis +from string import digits +from os import urandom from utils.logging.log import * @@ -107,9 +106,5 @@ def initConf(): def initMain(): global r, g initConf() - r = StrictRedis( - unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] - ) # Ephemeral - flushed on quit - g = StrictRedis( - unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"] - ) # Persistent + r = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]) # Ephemeral - flushed on quit + g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent diff --git a/modules/alias.py b/modules/alias.py index 6417e29..1961ec6 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -1,8 +1,7 @@ +import main import random import re -import main - def generate_password(): return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) diff --git a/modules/chankeep.py b/modules/chankeep.py index fdbe974..ec43b0f 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -1,12 +1,10 @@ +import main +from utils.logging.log import * +from utils.logging.debug import * from copy import deepcopy from math import ceil - -from twisted.internet.threads import deferToThread - -import main import modules.provision -from utils.logging.debug import * -from utils.logging.log import * +from twisted.internet.threads import deferToThread def allRelaysActive(net): @@ -48,10 +46,7 @@ def emptyChanAllocate(net, flist, relay, new): if toalloc > sum(chanfree[0].values()): correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) # print("correction", correction) - warn( - "Ran out of channel spaces, provisioning additional %i relays for %s" - % (correction, net) - ) + warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) # newNums = modules.provision.provisionMultipleRelays(net, correction) return False for i in chanfree[0].keys(): @@ -104,18 +99,12 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): listinfo = minifyChans(net, listinfo) if not listinfo: return - if ( - relay <= main.config["ChanKeep"]["SigSwitch"] - ): # we can cover all of the channels + if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels coverAll = True - elif ( - relay > main.config["ChanKeep"]["SigSwitch"] - ): # we cannot cover all of the channels + elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels coverAll = False if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: - error( - "Network %s is too big to cover: %i relays required" % (net, sigrelay) - ) + error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) return if coverAll: needed = relay - len(main.network[net].relays.keys()) diff --git a/modules/counters.py b/modules/counters.py index 5d51a52..a088cc3 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -1,6 +1,5 @@ -from twisted.internet.task import LoopingCall - import main +from twisted.internet.task import LoopingCall def event(name, eventType): diff --git a/modules/monitor.py b/modules/monitor.py index e1c16a7..f174a80 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -2,9 +2,10 @@ from copy import deepcopy from json import dumps import main -from core.logstash import sendLogstashNotification from core.relay import sendRelayNotification -from modules import regproc, userinfo +from core.logstash import sendLogstashNotification +from modules import userinfo +from modules import regproc from utils.dedup import dedup order = [ @@ -67,9 +68,7 @@ def parsemeta(numName, c): ) -def event( - numName, c -): # yes I'm using a short variable because otherwise it goes off the screen +def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen if dedup(numName, c): return diff --git a/modules/network.py b/modules/network.py index 5652120..21f1753 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,15 +1,14 @@ +from twisted.internet.ssl import DefaultOpenSSLContextFactory import json -from twisted.internet import reactor -from twisted.internet.ssl import DefaultOpenSSLContextFactory - -import main -from core.bot import IRCBot, IRCBotFactory from modules import alias from modules.chankeep import nukeNetwork from modules.regproc import needToRegister -from utils.get import getRelay +from twisted.internet import reactor +from core.bot import IRCBot, IRCBotFactory +import main from utils.logging.log import * +from utils.get import getRelay class Network: @@ -78,9 +77,7 @@ class Network: # e.g. freenode1 for the first relay on freenode network keyFN = main.certPath + main.config["Key"] certFN = main.certPath + main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory( - keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace") - ) + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) bot = IRCBotFactory(self.net, num) # host, port = self.relays[num]["host"], self.relays[num]["port"] host, port = getRelay(num) diff --git a/modules/provision.py b/modules/provision.py index 9e93236..1f17a20 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,25 +1,11 @@ -from twisted.internet import reactor - import main -import modules.regproc from core.bot import deliverRelayCommands from utils.logging.log import * +from twisted.internet import reactor +import modules.regproc -def provisionUserNetworkData( - num, - nick, - altnick, - ident, - realname, - emails, - network, - host, - port, - security, - auth, - password, -): +def provisionUserNetworkData(num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password): print("nick", nick) print("altnick", altnick) print("emails", emails) @@ -30,25 +16,17 @@ def provisionUserNetworkData( stage2commands["status"] = [] commands["controlpanel"] = [] user = nick.lower() - commands["controlpanel"].append( - "AddUser %s %s" % (user, main.config["Relay"]["Password"]) - ) + commands["controlpanel"].append("AddUser %s %s" % (user, main.config["Relay"]["Password"])) commands["controlpanel"].append("AddNetwork %s %s" % (user, network)) commands["controlpanel"].append("Set Nick %s %s" % (user, nick)) commands["controlpanel"].append("Set Altnick %s %s" % (user, altnick)) commands["controlpanel"].append("Set Ident %s %s" % (user, ident)) commands["controlpanel"].append("Set RealName %s %s" % (user, realname)) if security == "ssl": - commands["controlpanel"].append( - "SetNetwork TrustAllCerts %s %s true" % (user, network) - ) # Don't judge me - commands["controlpanel"].append( - "AddServer %s %s %s +%s" % (user, network, host, port) - ) + commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me + commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port)) elif security == "plain": - commands["controlpanel"].append( - "AddServer %s %s %s %s" % (user, network, host, port) - ) + commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port)) if not main.config["ConnectOnCreate"]: stage2commands["status"].append("Disconnect") if main.config["Toggles"]["CycleChans"]: diff --git a/modules/regproc.py b/modules/regproc.py index 353c5e6..ce0e5a2 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -1,11 +1,9 @@ -from copy import deepcopy -from random import choice - import main from modules import provision -from utils.logging.debug import * from utils.logging.log import * - +from utils.logging.debug import * +from copy import deepcopy +from random import choice def needToRegister(net): # Check if the network does not support authentication @@ -53,7 +51,7 @@ def substitute(net, num, token=None): nickname = alias["nick"] username = nickname + "/" + net password = main.network[net].aliases[num]["password"] - # inst["email"] = inst["email"].replace("{nickname}", nickname) + #inst["email"] = inst["email"].replace("{nickname}", nickname) for i in inst.keys(): if not isinstance(inst[i], str): continue @@ -114,12 +112,8 @@ def enableAuthentication(net, num): auth = obj.auth password = obj.aliases[num]["password"] uname = main.alias[num]["nick"] + "/" + net - provision.provisionAuthenticationData( - num, nick, net, security, auth, password - ) # Set up for auth - main.IRCPool[net + str(num)].msg( - main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump" - ) + provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth + main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") if selectInst(net)["check"] == False: confirmRegistration(net, num) @@ -135,14 +129,10 @@ def registerTest(c): if not main.IRCPool[name]._negativePass == True: if c["type"] == "query" and c["nick"] == sinst["entity"]: if sinst["checknegativemsg"] in c["msg"]: - confirmRegistration( - c["net"], c["num"], negativepass=False - ) # Not passed negative check, report back + confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back return if sinst["checkendnegative"] in c["msg"]: - confirmRegistration( - c["net"], c["num"], negativepass=True - ) # Passed the negative check, report back + confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: diff --git a/modules/userinfo.py b/modules/userinfo.py index 94c0833..1722cb3 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,10 +1,9 @@ +from twisted.internet.threads import deferToThread from string import digits -from twisted.internet.threads import deferToThread - import main -from utils.logging.debug import debug, trace from utils.logging.log import * +from utils.logging.debug import debug, trace from utils.parsing import parsen @@ -146,9 +145,7 @@ def delUser(name, channel, nick, user): p.srem(namespace, nick) if channels == {channel.encode()}: # can we only see them on this channel? p.delete(chanspace) # remove channel tracking entry - p.hdel( - "live.prefix." + name + "." + channel, nick - ) # remove prefix tracking entry + p.hdel("live.prefix." + name + "." + channel, nick) # remove prefix tracking entry p.hdel(mapspace, nick) # remove nick mapping entry if user: p.srem(gnamespace, user) # remove global userinfo entry @@ -173,10 +170,7 @@ def getUserByNick(name, nick): if main.r.hexists(mapspace, nick): return main.r.hget(mapspace, nick) else: - warn( - "Entry doesn't exist: %s on %s - attempting auxiliary lookup" - % (nick, mapspace) - ) + warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) # return False # legacy code below - remove when map is reliable usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999) @@ -206,18 +200,10 @@ def renameUser(name, oldnick, olduser, newnick, newuser): p.sadd("live.who." + name + "." + i, newnick) p.hdel(mapspace, oldnick) p.hset(mapspace, newnick, newuser) - if main.r.exists( - "live.prefix." + name + "." + i - ): # if there's a prefix entry for the channel - if main.r.hexists( - "live.prefix." + name + "." + i, oldnick - ): # if the old nick is in it - mode = main.r.hget( - "live.prefix." + name + "." + i, oldnick - ) # retrieve old modes - p.hset( - "live.prefix." + name + "." + i, newnick, mode - ) # set old modes to new nickname + if main.r.exists("live.prefix." + name + "." + i): # if there's a prefix entry for the channel + if main.r.hexists("live.prefix." + name + "." + i, oldnick): # if the old nick is in it + mode = main.r.hget("live.prefix." + name + "." + i, oldnick) # retrieve old modes + p.hset("live.prefix." + name + "." + i, newnick, mode) # set old modes to new nickname if main.r.exists(chanspace): p.rename(chanspace, newchanspace) else: diff --git a/threshold b/threshold index b7c6f78..dc186b4 100755 --- a/threshold +++ b/threshold @@ -1,14 +1,14 @@ #!/usr/bin/env python +from twisted.internet import reactor +from twisted.internet.ssl import DefaultOpenSSLContextFactory import sys -from codecs import getwriter # fix printing odd shit to the terminal -from signal import SIGINT, signal +from signal import signal, SIGINT + # from twisted.python import log # from sys import stdout # log.startLogging(stdout) -from sys import stderr, stdout # Import again because we want to override - -from twisted.internet import reactor -from twisted.internet.ssl import DefaultOpenSSLContextFactory +from sys import stdout, stderr # Import again because we want to override +from codecs import getwriter # fix printing odd shit to the terminal stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using @@ -23,11 +23,11 @@ if "--debug" in sys.argv: # yes really main.config["Debug"] = True if "--trace" in sys.argv: main.config["Trace"] = True -import modules.counters -from core.relay import Relay, RelayFactory -from core.server import Server, ServerFactory -from utils.loaders.command_loader import loadCommands from utils.logging.log import * +from utils.loaders.command_loader import loadCommands +from core.server import Server, ServerFactory +from core.relay import Relay, RelayFactory +import modules.counters loadCommands() import core.logstash @@ -46,20 +46,14 @@ if __name__ == "__main__": ), interface=main.config["Listener"]["Address"], ) - log( - "Threshold running with SSL on %s:%s" - % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]) - ) + log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) else: reactor.listenTCP( main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"], ) - log( - "Threshold running on %s:%s" - % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]) - ) + log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) if main.config["RelayAPI"]["Enabled"]: relay = RelayFactory() if main.config["RelayAPI"]["UseSSL"] == True: @@ -82,10 +76,7 @@ if __name__ == "__main__": relay, interface=main.config["RelayAPI"]["Address"], ) - log( - "Threshold relay running on %s:%s" - % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]) - ) + log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() diff --git a/utils/cleanup.py b/utils/cleanup.py index 51eaa05..83ed661 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -1,10 +1,8 @@ -import sys - -from twisted.internet import reactor - import main +from twisted.internet import reactor from utils.logging.debug import debug from utils.logging.log import * +import sys def handler(sig, frame): diff --git a/utils/dedup.py b/utils/dedup.py index 89e4bee..dc8d4c3 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -1,9 +1,7 @@ -from copy import deepcopy from datetime import datetime -from json import dumps - from csiphash import siphash24 - +from copy import deepcopy +from json import dumps import main from utils.logging.debug import debug @@ -12,24 +10,16 @@ def dedup(numName, b): c = deepcopy(b) if "ts" in c.keys(): del c["ts"] - c["approxtime"] = str(datetime.utcnow().timestamp())[ - : main.config["Tweaks"]["DedupPrecision"] - ] + c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] - isDuplicate = any( - castHash in main.lastEvents[x] - for x in main.lastEvents.keys() - if not x == numName - ) + isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) if isDuplicate: debug("Duplicate: %s" % (c)) return True if numName in main.lastEvents.keys(): main.lastEvents[numName].insert(0, castHash) - main.lastEvents[numName] = main.lastEvents[numName][ - 0 : main.config["Tweaks"]["MaxHash"] - ] + main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] else: main.lastEvents[numName] = [castHash] return False diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 19517ec..9c4095b 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,9 +1,10 @@ from os import listdir -import commands -from main import CommandMap from utils.logging.debug import debug from utils.logging.log import * +import commands + +from main import CommandMap def loadCommands(allowDup=False): @@ -14,15 +15,11 @@ def loadCommands(allowDup=False): try: module = __import__("commands.%s" % commandName) if not commandName in CommandMap: - CommandMap[commandName] = getattr( - getattr(module, commandName), className - ) + CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) else: if allowDup: - CommandMap[commandName] = getattr( - getattr(module, commandName), className - ) + CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) error("Duplicate command: %s" % (commandName)) diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index 64b0618..f5219ac 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -1,11 +1,12 @@ -import sys -from importlib import reload from os import listdir +from importlib import reload +import sys -import commands -from main import CommandMap from utils.logging.debug import debug from utils.logging.log import * +import commands + +from main import CommandMap def loadSingle(commandName): @@ -14,9 +15,7 @@ def loadSingle(commandName): try: if commandName in CommandMap.keys(): reload(sys.modules["commands." + commandName]) - CommandMap[commandName] = getattr( - sys.modules["commands." + commandName], className - ) + CommandMap[commandName] = getattr(sys.modules["commands." + commandName], className) debug("Reloaded command: %s" % commandName) return "RELOAD" module = __import__("commands.%s" % commandName) diff --git a/utils/logging/debug.py b/utils/logging/debug.py index 0c73174..80651f7 100644 --- a/utils/logging/debug.py +++ b/utils/logging/debug.py @@ -1,6 +1,5 @@ import main - # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main # to read the setting def debug(*data): From 6c7d0d5c4575f2a4b5e97347f355c3ff17a3b87c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:03 +0100 Subject: [PATCH 246/394] Reformat and fix circular import --- commands/admall.py | 2 +- commands/email.py | 4 +++- core/bot.py | 8 +------- core/relay.py | 5 ++++- main.py | 4 +++- modules/network.py | 4 +++- modules/provision.py | 6 ++++-- modules/regproc.py | 11 ++++++++--- threshold | 10 ++++++++-- utils/deliver_relay_commands.py | 7 +++++++ utils/loaders/command_loader.py | 20 ++++++++++---------- 11 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 utils/deliver_relay_commands.py diff --git a/commands/admall.py b/commands/admall.py index 8d74545..9ae8636 100644 --- a/commands/admall.py +++ b/commands/admall.py @@ -1,5 +1,5 @@ import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands class AdmallCommand: diff --git a/commands/email.py b/commands/email.py index 4817988..8905813 100644 --- a/commands/email.py +++ b/commands/email.py @@ -74,7 +74,9 @@ class EmailCommand: elif length == 3: if spl[1] == "list": if spl[2] == "domain": - filtered = {f"{k}:{k2}":v2 for k,v in main.irc.items() for k2,v2 in v.items() if k2 == "domains"} + filtered = { + f"{k}:{k2}": v2 for k, v in main.irc.items() for k2, v2 in v.items() if k2 == "domains" + } info(dump(filtered)) return else: diff --git a/core/bot.py b/core/bot.py index 375703d..054e8e0 100644 --- a/core/bot.py +++ b/core/bot.py @@ -35,13 +35,7 @@ from utils.parsing import parsen from twisted.internet.ssl import DefaultOpenSSLContextFactory -def deliverRelayCommands(num, relayCommands, user=None, stage2=None): - keyFN = main.certPath + main.config["Key"] - certFN = main.certPath + main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) - host, port = getRelay(num) - rct = reactor.connectSSL(host, port, bot, contextFactory) +from utils.deliver_relay_commands import deliverRelayCommands # Copied from the Twisted source so we can fix a bug diff --git a/core/relay.py b/core/relay.py index 66a9830..8ea6c92 100644 --- a/core/relay.py +++ b/core/relay.py @@ -114,7 +114,10 @@ class Relay(Protocol): def handleHello(self, parsed): if parsed["key"] in main.tokens.keys(): - if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay": + if ( + parsed["hello"] == main.tokens[parsed["key"]]["hello"] + and main.tokens[parsed["key"]]["usage"] == "relay" + ): self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]}) self.authed = True else: diff --git a/main.py b/main.py index 473fd3e..1b01f4d 100644 --- a/main.py +++ b/main.py @@ -106,5 +106,7 @@ def initConf(): def initMain(): global r, g initConf() - r = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]) # Ephemeral - flushed on quit + r = StrictRedis( + unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] + ) # Ephemeral - flushed on quit g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent diff --git a/modules/network.py b/modules/network.py index 21f1753..71fd3dc 100644 --- a/modules/network.py +++ b/modules/network.py @@ -77,7 +77,9 @@ class Network: # e.g. freenode1 for the first relay on freenode network keyFN = main.certPath + main.config["Key"] certFN = main.certPath + main.config["Certificate"] - contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) + contextFactory = DefaultOpenSSLContextFactory( + keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace") + ) bot = IRCBotFactory(self.net, num) # host, port = self.relays[num]["host"], self.relays[num]["port"] host, port = getRelay(num) diff --git a/modules/provision.py b/modules/provision.py index 1f17a20..c32aaca 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,11 +1,13 @@ import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands from utils.logging.log import * from twisted.internet import reactor import modules.regproc -def provisionUserNetworkData(num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password): +def provisionUserNetworkData( + num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password +): print("nick", nick) print("altnick", altnick) print("emails", emails) diff --git a/modules/regproc.py b/modules/regproc.py index ce0e5a2..f0a8335 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -5,6 +5,7 @@ from utils.logging.debug import * from copy import deepcopy from random import choice + def needToRegister(net): # Check if the network does not support authentication networkObj = main.network[net] @@ -51,7 +52,7 @@ def substitute(net, num, token=None): nickname = alias["nick"] username = nickname + "/" + net password = main.network[net].aliases[num]["password"] - #inst["email"] = inst["email"].replace("{nickname}", nickname) + # inst["email"] = inst["email"].replace("{nickname}", nickname) for i in inst.keys(): if not isinstance(inst[i], str): continue @@ -129,10 +130,14 @@ def registerTest(c): if not main.IRCPool[name]._negativePass == True: if c["type"] == "query" and c["nick"] == sinst["entity"]: if sinst["checknegativemsg"] in c["msg"]: - confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back + confirmRegistration( + c["net"], c["num"], negativepass=False + ) # Not passed negative check, report back return if sinst["checkendnegative"] in c["msg"]: - confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back + confirmRegistration( + c["net"], c["num"], negativepass=True + ) # Passed the negative check, report back return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: diff --git a/threshold b/threshold index dc186b4..e844b23 100755 --- a/threshold +++ b/threshold @@ -46,7 +46,10 @@ if __name__ == "__main__": ), interface=main.config["Listener"]["Address"], ) - log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) + log( + "Threshold running with SSL on %s:%s" + % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]) + ) else: reactor.listenTCP( main.config["Listener"]["Port"], @@ -76,7 +79,10 @@ if __name__ == "__main__": relay, interface=main.config["RelayAPI"]["Address"], ) - log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) + log( + "Threshold relay running on %s:%s" + % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]) + ) for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() diff --git a/utils/deliver_relay_commands.py b/utils/deliver_relay_commands.py new file mode 100644 index 0000000..c0b44df --- /dev/null +++ b/utils/deliver_relay_commands.py @@ -0,0 +1,7 @@ +def deliverRelayCommands(num, relayCommands, user=None, stage2=None): + keyFN = main.certPath + main.config["Key"] + certFN = main.certPath + main.config["Certificate"] + contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) + bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) + host, port = getRelay(num) + rct = reactor.connectSSL(host, port, bot, contextFactory) \ No newline at end of file diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 9c4095b..0fa6d5c 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -12,16 +12,16 @@ def loadCommands(allowDup=False): if filename.endswith(".py") and filename != "__init__.py": commandName = filename[0:-3] className = commandName.capitalize() + "Command" - try: - module = __import__("commands.%s" % commandName) - if not commandName in CommandMap: + # try: + module = __import__("commands.%s" % commandName) + if not commandName in CommandMap: + CommandMap[commandName] = getattr(getattr(module, commandName), className) + debug("Registered command: %s" % commandName) + else: + if allowDup: CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) - else: - if allowDup: - CommandMap[commandName] = getattr(getattr(module, commandName), className) - debug("Registered command: %s" % commandName) - error("Duplicate command: %s" % (commandName)) - except Exception as err: - error("Exception while loading command %s:\n%s" % (commandName, err)) + error("Duplicate command: %s" % (commandName)) + # except Exception as err: + # error("Exception while loading command %s:\n%s" % (commandName, err)) From f4c5323de1ad1c9803fd9f27878aa2fe3f4ac730 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:05 +0100 Subject: [PATCH 247/394] Reformat project --- commands/alias.py | 2 +- commands/all.py | 2 +- commands/allc.py | 2 +- commands/chans.py | 1 - commands/cmd.py | 3 +- commands/disable.py | 6 +- commands/email.py | 6 +- commands/enable.py | 2 +- commands/exec.py | 3 - commands/list.py | 6 +- commands/loadmod.py | 3 +- commands/logout.py | 3 - commands/mod.py | 5 +- commands/msg.py | 2 +- commands/recheckauth.py | 5 +- commands/stats.py | 6 +- commands/users.py | 1 - commands/who.py | 1 - conf/example/config.json | 6 +- conf/example/irc.json | 1 - core/bot.py | 124 ++++++------------------- core/factory.py | 77 ++++++++++++++++ core/logstash.py | 5 +- core/parser.py | 17 ++-- core/relay.py | 16 ++-- core/server.py | 6 +- main.py | 9 +- modules/alias.py | 4 +- modules/chankeep.py | 6 +- modules/counters.py | 8 +- modules/monitor.py | 7 +- modules/network.py | 7 +- modules/provision.py | 3 +- modules/regproc.py | 24 ++--- modules/userinfo.py | 11 +-- threshold | 31 ++++--- utils/cleanup.py | 3 +- utils/dedup.py | 2 +- utils/deliver_relay_commands.py | 159 +++++++++++++++++++++++++++++++- utils/get.py | 4 +- utils/loaders/command_loader.py | 5 +- utils/loaders/single_loader.py | 2 - utils/logging/debug.py | 1 + utils/logging/send.py | 2 +- 44 files changed, 363 insertions(+), 236 deletions(-) create mode 100644 core/factory.py diff --git a/commands/alias.py b/commands/alias.py index d04cfdf..ce55037 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -32,7 +32,7 @@ class AliasCommand: failure("Must be a number, not %s" % spl[2]) return num = int(spl[2]) - if not num in main.alias.keys(): + if num not in main.alias.keys(): failure("No such alias: %i" % num) return failed = False diff --git a/commands/all.py b/commands/all.py index a2994e9..509f0f5 100644 --- a/commands/all.py +++ b/commands/all.py @@ -1,5 +1,5 @@ import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands class AllCommand: diff --git a/commands/allc.py b/commands/allc.py index 7a7b5fd..cd55596 100644 --- a/commands/allc.py +++ b/commands/allc.py @@ -1,5 +1,5 @@ import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands class AllcCommand: diff --git a/commands/chans.py b/commands/chans.py index 1ccf710..663d82a 100644 --- a/commands/chans.py +++ b/commands/chans.py @@ -1,4 +1,3 @@ -import main import modules.userinfo as userinfo diff --git a/commands/cmd.py b/commands/cmd.py index babdb2d..e269da2 100644 --- a/commands/cmd.py +++ b/commands/cmd.py @@ -1,5 +1,4 @@ -import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands class CmdCommand: diff --git a/commands/disable.py b/commands/disable.py index 9c52234..78b0984 100644 --- a/commands/disable.py +++ b/commands/disable.py @@ -1,5 +1,5 @@ import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands class DisableCommand: @@ -19,13 +19,13 @@ class DisableCommand: name = spl[1] + spl[2] if not spl[1] in main.IRCPool.keys(): info("Note - instance not running, proceeding anyway") - if not relayNum in main.network[spl[1]].relays.keys(): + if relayNum not in main.network[spl[1]].relays.keys(): failure("No such relay: %s in network %s" % (spl[2], spl[1])) return main.network[spl[1]].relays[relayNum]["enabled"] = False user = main.alias[relayNum]["nick"] network = spl[1] - relay = main.network[spl[1]].relays[relayNum] + # relay = main.network[spl[1]].relays[relayNum] commands = {"status": ["Disconnect"]} deliverRelayCommands(relayNum, commands, user=user + "/" + network) main.saveConf("network") diff --git a/commands/email.py b/commands/email.py index 8905813..ea36da0 100644 --- a/commands/email.py +++ b/commands/email.py @@ -17,7 +17,7 @@ class EmailCommand: if "@" in domain: failure("Not a domain: %s" % domain) return - if not domain in main.irc["_"]["domains"]: + if domain not in main.irc["_"]["domains"]: main.irc["_"]["domains"].append(domain) success("Successfully added domain %s to default config" % domain) main.saveConf("irc") @@ -41,7 +41,7 @@ class EmailCommand: else: num = int(spl[2]) if spl[1] == "add": - if not num in main.alias.keys(): + if num not in main.alias.keys(): failure("No such alias: %i" % num) return if not spl[3] in main.alias[num]["emails"]: @@ -53,7 +53,7 @@ class EmailCommand: failure("Email already exists in alias %i: %s" % (num, spl[3])) return elif spl[1] == "del": - if not num in main.alias.keys(): + if num not in main.alias.keys(): failure("No such alias: %i" % num) return if spl[3] in main.alias[num]["emails"]: diff --git a/commands/enable.py b/commands/enable.py index 439b26c..3553446 100644 --- a/commands/enable.py +++ b/commands/enable.py @@ -1,5 +1,5 @@ import main -from core.bot import deliverRelayCommands +from utils.deliver_relay_commands import deliverRelayCommands class EnableCommand: diff --git a/commands/exec.py b/commands/exec.py index c2a5598..9b7ea28 100644 --- a/commands/exec.py +++ b/commands/exec.py @@ -1,6 +1,3 @@ -import main - - class ExecCommand: def __init__(self, *args): self.exec(*args) diff --git a/commands/list.py b/commands/list.py index 217b051..e72cc2e 100644 --- a/commands/list.py +++ b/commands/list.py @@ -9,7 +9,7 @@ class ListCommand: if authed: if length == 1: for i in main.network.keys(): - if not 1 in main.network[i].relays.keys(): + if 1 not in main.network[i].relays.keys(): info("Network has no first instance: %s" % i) continue if not i + "1" in main.IRCPool.keys(): @@ -22,10 +22,10 @@ class ListCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - if not 1 in main.network[spl[1]].relays.keys(): + if 1 not in main.network[spl[1]].relays.keys(): failure("Network has no first instance") return - if not spl[1] + "1" in main.IRCPool.keys(): + if spl[1] + "1" not in main.IRCPool.keys(): failure("No IRC instance: %s - 1" % spl[1]) return main.IRCPool[spl[1] + "1"].list() diff --git a/commands/loadmod.py b/commands/loadmod.py index 2923fd7..0dbc8a8 100644 --- a/commands/loadmod.py +++ b/commands/loadmod.py @@ -1,4 +1,3 @@ -import main from utils.loaders.single_loader import loadSingle @@ -10,7 +9,7 @@ class LoadmodCommand: if authed: if length == 2: rtrn = loadSingle(spl[1]) - if rtrn == True: + if rtrn is True: success("Loaded module: %s" % spl[1]) return elif rtrn == "RELOAD": diff --git a/commands/logout.py b/commands/logout.py index 61cb834..776dfd0 100644 --- a/commands/logout.py +++ b/commands/logout.py @@ -1,6 +1,3 @@ -import main - - class LogoutCommand: def __init__(self, *args): self.logout(*args) diff --git a/commands/mod.py b/commands/mod.py index 10a19e1..85e7814 100644 --- a/commands/mod.py +++ b/commands/mod.py @@ -1,5 +1,4 @@ import main -from yaml import dump class ModCommand: @@ -16,8 +15,8 @@ class ModCommand: try: setattr(main.network[spl[1]], spl[2], spl[3]) - except e: - failure("Something went wrong.") + except Exception as e: + failure(f"Something went wrong: {e}") return main.saveConf("network") diff --git a/commands/msg.py b/commands/msg.py index d92494a..452a532 100644 --- a/commands/msg.py +++ b/commands/msg.py @@ -12,7 +12,7 @@ class MsgCommand: failure("Network does not exist: %s" % spl[1]) return if not int(spl[2]) in main.network[spl[1]].relays.keys(): - failure("Relay % does not exist on network %" % (spl[2], spl[1])) + failure("Relay %s does not exist on network %s" % (spl[2], spl[1])) return if not spl[1] + spl[2] in main.IRCPool.keys(): failure("Name has no instance: %s" % spl[1]) diff --git a/commands/recheckauth.py b/commands/recheckauth.py index ab4b5b0..7cdf527 100644 --- a/commands/recheckauth.py +++ b/commands/recheckauth.py @@ -8,9 +8,7 @@ class RecheckauthCommand: def recheckauth(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): if authed: if length == 1: - results = [] for i in main.IRCPool.keys(): - num = main.IRCPool[i].num net = main.IRCPool[i].net main.IRCPool[i].authenticated = False main.IRCPool[i].regPing() @@ -20,9 +18,8 @@ class RecheckauthCommand: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - results = [] for i in main.IRCPool.keys(): - num = main.IRCPool[i].num + # num = main.IRCPool[i].num net = main.IRCPool[i].net if not net == spl[1]: continue diff --git a/commands/stats.py b/commands/stats.py index a396f12..7102dd1 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -26,7 +26,7 @@ class StatsCommand: stats.append("User records: %s" % numWhoEntries) stats.append("Events/min: %s" % main.lastMinuteSample) counterEvents = count.getEvents() - if counterEvents == None: + if counterEvents is None: stats.append("No counters records") else: stats.append("Counters:") @@ -43,7 +43,7 @@ class StatsCommand: numNodes = 0 for i in main.IRCPool.keys(): - if "".join([x for x in i if not x in digits]) == spl[1]: + if "".join([x for x in i if x not in digits]) == spl[1]: numChannels += len(main.IRCPool[i].channels) found = True numNodes += 1 @@ -54,7 +54,7 @@ class StatsCommand: stats.append("User records: %s" % numWhoEntries) stats.append("Endpoints: %s" % numNodes) counterEvents = count.getEvents(spl[1]) - if counterEvents == None: + if counterEvents is None: stats.append("No counters records") else: stats.append("Counters:") diff --git a/commands/users.py b/commands/users.py index 7d80e84..be7a7b1 100644 --- a/commands/users.py +++ b/commands/users.py @@ -1,4 +1,3 @@ -import main import modules.userinfo as userinfo diff --git a/commands/who.py b/commands/who.py index e80f488..10faa93 100644 --- a/commands/who.py +++ b/commands/who.py @@ -1,4 +1,3 @@ -import main import modules.userinfo as userinfo diff --git a/conf/example/config.json b/conf/example/config.json index 3846c71..9471a47 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -12,14 +12,12 @@ }, "Key": "key.pem", "Certificate": "cert.pem", - "RedisSocket": "/var/run/redis/redis.sock", - "RedisDBEphemeral": 2, - "RedisDBPersistent": 3, + "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, "AutoReg": false, "Debug": false, - "Trace": false, + "Trace", false, "Relay": { "Host": "127.0.0.1", "Port": "201x", diff --git a/conf/example/irc.json b/conf/example/irc.json index a62dd51..8270e2f 100644 --- a/conf/example/irc.json +++ b/conf/example/irc.json @@ -3,7 +3,6 @@ "register": true, "entity": "NickServ", "email": "{nickname}@domain.com", - "domains": [], "registermsg": "REGISTER {password} {email}", "confirm": "CONFIRM {token}", "check": true, diff --git a/core/bot.py b/core/bot.py index 054e8e0..4f20576 100644 --- a/core/bot.py +++ b/core/bot.py @@ -2,16 +2,14 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall -from twisted.internet import reactor, task +from twisted.internet import reactor from twisted.words.protocols.irc import ( - symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage, ) import sys -from string import digits from random import randint from copy import deepcopy from datetime import datetime @@ -24,19 +22,13 @@ from modules import regproc from core.relay import sendRelayNotification from utils.dedup import dedup -from utils.get import getRelay import main -from utils.logging.log import * -from utils.logging.debug import * -from utils.logging.send import * +from utils.logging.log import log, warn, error +from utils.logging.debug import debug +from utils.logging.send import sendAll from utils.parsing import parsen -from twisted.internet.ssl import DefaultOpenSSLContextFactory - - -from utils.deliver_relay_commands import deliverRelayCommands - # Copied from the Twisted source so we can fix a bug def parsemsg(s): @@ -64,69 +56,6 @@ def parsemsg(s): return prefix, command, args -class IRCRelay(IRCClient): - def __init__(self, num, relayCommands, user, stage2): - self.isconnected = False - self.buffer = "" - if user == None: - self.user = main.config["Relay"]["User"] - else: - self.user = user.lower() - password = main.config["Relay"]["Password"] - self.nickname = "relay" - self.realname = "relay" - self.username = self.user - self.password = self.user + ":" + password - - self.relayCommands = relayCommands - self.num = num - self.stage2 = stage2 - self.loop = None - - def privmsg(self, user, channel, msg): - nick, ident, host = parsen(user) - for i in main.ZNCErrors: - if i in msg: - error("ZNC issue:", msg) - if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]: - nick = nick[1:] - if nick in self.relayCommands.keys(): - sendAll("[%s] %s -> %s" % (self.num, nick, msg)) - - def irc_ERR_PASSWDMISMATCH(self, prefix, params): - log("%s: relay password mismatch" % self.num) - sendAll("%s: relay password mismatch" % self.num) - - def sendStage2(self): - # [["user", {"sasl": ["message1", "message2"]}], []] - if not len(self.stage2) == 0: - user = self.stage2[0].pop(0) - commands = self.stage2[0].pop(0) - del self.stage2[0] - deliverRelayCommands(self.num, commands, user, self.stage2) - - def signedOn(self): - if not self.isconnected: - self.isconnected = True - # log("signed on as a relay: %s" % self.num) - sleeptime = 0 - increment = 0.8 - for i in self.relayCommands.keys(): - for x in self.relayCommands[i]: - reactor.callLater( - sleeptime, - self.msg, - main.config["Tweaks"]["ZNC"]["Prefix"] + i, - x, - ) - sleeptime += increment - increment += 0.8 - if not self.stage2 == None: - reactor.callLater(sleeptime, self.sendStage2) - reactor.callLater(sleeptime + 5, self.transport.loseConnection) - return - - class IRCBot(IRCClient): def __init__(self, net, num): self.isconnected = False @@ -188,7 +117,7 @@ class IRCBot(IRCClient): sleeptime = 0.0 increment = 0.8 for i in channels: - if not i in self.channels: + if i not in self.channels: if self.net in main.blacklist.keys(): if i in main.blacklist[self.net]: debug("Not joining blacklisted channel %s on %s - %i" % (i, self.net, self.num)) @@ -215,12 +144,12 @@ class IRCBot(IRCClient): del main.TempChan[self.net] def event(self, **cast): - if not "ts" in cast.keys(): + if "ts" not in cast.keys(): cast["ts"] = str(datetime.now().isoformat()) # remove odd stuff for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over - if cast[i] == "": # a dictionary that changes length with each iteration + if cast[i] == "": # a dictionary that changes length with each iteration del cast[i] # remove server stuff if "muser" in cast.keys(): @@ -302,16 +231,16 @@ class IRCBot(IRCClient): # we have been mentioned in a msg/notice/action/part/quit/topic message if "msg" in cast.keys(): # Don't highlight queries - if not cast["msg"] == None: + if cast["msg"] is not None: if self.nickname.lower() in cast["msg"].lower(): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "highlight" self.event(**castDup) - if not "net" in cast.keys(): + if "net" not in cast.keys(): cast["net"] = self.net - if not "num" in cast.keys(): + if "num" not in cast.keys(): cast["num"] = self.num if not self.authenticated: regproc.registerTest(cast) @@ -478,7 +407,7 @@ class IRCBot(IRCClient): else: if nocheck: allRelays = True # override the system - if this is - else: # specified, we already did this + else: # specified, we already did this allRelays = chankeep.allRelaysActive(self.net) if not allRelays: self.wantList = True @@ -535,7 +464,7 @@ class IRCBot(IRCClient): allRelays = chankeep.allRelaysActive(self.net) if allRelays: name = self.net + "1" - if main.IRCPool[name].wantList == True: + if main.IRCPool[name].wantList is True: main.IRCPool[name].list(nocheck=True) debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) if self.num == 1: # Only one instance should do a list @@ -559,7 +488,7 @@ class IRCBot(IRCClient): try: self.chanlimit = int(chanlimit) except TypeError: - warn("Invalid chanlimit: %s" % i) + warn("Invalid chanlimit: %s" % chanlimit) if self.chanlimit == 0: self.chanlimit = 200 # don't take the piss if it's not limited if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation @@ -625,7 +554,7 @@ class IRCBot(IRCClient): self.userLeft(prefix, channel, message) def irc_QUIT(self, prefix, params): - nick = prefix.split("!")[0] + # nick = prefix.split("!")[0] self.userQuit(prefix, params[0]) def irc_NICK(self, prefix, params): @@ -656,12 +585,12 @@ class IRCBot(IRCClient): if not sinst: error(f"Registration ping failed for {self.net} - {self.num}") return - if not self._negativePass == True: - if negativepass == False: + if self._negativePass is not True: + if negativepass is False: self._negativePass = False return - if negativepass == True: - if self._negativePass == None: + if negativepass is True: + if self._negativePass is None: self._negativePass = True debug("Positive registration check - %s - %i" % (self.net, self.num)) if sinst["ping"]: @@ -701,7 +630,7 @@ class IRCBot(IRCClient): reactor.callLater(10, self.regPing) def joined(self, channel): - if not channel in self.channels: + if channel not in self.channels: self.channels.append(channel) self.names(channel).addCallback(self.got_names) if main.config["Toggles"]["Who"]: @@ -723,7 +652,7 @@ class IRCBot(IRCClient): # log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so - def left(self, user, channel, message): # even if they saw it, they wouldn't react + def left(self, user, channel, message): # even if they saw it, they wouldn't react self.event(type="part", muser=user, channel=channel, msg=message) self.botLeft(channel) @@ -749,7 +678,7 @@ class IRCBot(IRCClient): # format return # stop right there sir! chans = userinfo.getChanList(self.net, cast["nick"]) - if chans == None: + if chans is None: error("No channels returned for chanless event: %s" % cast) # self.event(**cast) -- no, should NEVER happen return @@ -780,9 +709,10 @@ class IRCBot(IRCClient): ) +# TODO: strip out relay functionality class IRCBotFactory(ReconnectingClientFactory): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): - if net == None: + if net is None: self.num = num self.net = None self.name = "relay - %i" % num @@ -801,11 +731,9 @@ class IRCBotFactory(ReconnectingClientFactory): self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 def buildProtocol(self, addr): - if self.relay == False: + if self.relay is False: entry = IRCBot(self.net, self.num) main.IRCPool[self.name] = entry - else: - entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) self.client = entry return entry @@ -813,7 +741,7 @@ class IRCBotFactory(ReconnectingClientFactory): def clientConnectionLost(self, connector, reason): if not self.relay: userinfo.delChannels(self.net, self.client.channels) - if not self.client == None: + if self.client is not None: self.client.isconnected = False self.client.authenticated = False self.client.channels = [] @@ -836,7 +764,7 @@ class IRCBotFactory(ReconnectingClientFactory): # ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): - if not self.client == None: + if self.client is not None: self.client.isconnected = False self.client.authenticated = False self.client.channels = [] diff --git a/core/factory.py b/core/factory.py new file mode 100644 index 0000000..0d63e32 --- /dev/null +++ b/core/factory.py @@ -0,0 +1,77 @@ +class IRCBotFactory(ReconnectingClientFactory): + def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): + if net is None: + self.num = num + self.net = None + self.name = "relay - %i" % num + self.relay = True + else: + self.name = net + str(num) + self.num = num + self.net = net + self.relay = False + self.client = None + self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"] + self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"] + self.factor = main.config["Tweaks"]["Delays"]["Factor"] + self.jitter = main.config["Tweaks"]["Delays"]["Jitter"] + + self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 + + def buildProtocol(self, addr): + if self.relay is False: + entry = IRCBot(self.net, self.num) + main.IRCPool[self.name] = entry + else: + entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) + + self.client = entry + return entry + + def clientConnectionLost(self, connector, reason): + if not self.relay: + userinfo.delChannels(self.net, self.client.channels) + if self.client is not None: + self.client.isconnected = False + self.client.authenticated = False + self.client.channels = [] + error = reason.getErrorMessage() + if not self.relay: + log("%s - %i: connection lost: %s" % (self.net, self.num, error)) + sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) + ctime = str(datetime.now().isoformat()) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "lost", + "message": error, + "ts": ctime, + } + ) + self.retry(connector) + # ReconnectingClientFactory.clientConnectionLost(self, connector, reason) + + def clientConnectionFailed(self, connector, reason): + if self.client is not None: + self.client.isconnected = False + self.client.authenticated = False + self.client.channels = [] + error = reason.getErrorMessage() + log("%s - %i: connection failed: %s" % (self.net, self.num, error)) + if not self.relay: + sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) + ctime = str(datetime.now().isoformat()) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "failed", + "message": error, + "ts": ctime, + } + ) + self.retry(connector) + # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/logstash.py b/core/logstash.py index 17e8dfd..94429bc 100644 --- a/core/logstash.py +++ b/core/logstash.py @@ -1,9 +1,8 @@ import logstash import logging -from json import dumps, loads +from json import dumps import main -from utils.logging.log import * logger = None @@ -22,7 +21,7 @@ def init_logstash(): def sendLogstashNotification(text): - if not logger == None: + if logger is not None: logger.info(dumps(text)) return True return False diff --git a/core/parser.py b/core/parser.py index 1eb34c5..2f2383d 100644 --- a/core/parser.py +++ b/core/parser.py @@ -1,6 +1,6 @@ import main -from utils.logging.log import * -from utils.logging.send import * +from utils.logging.log import warn +from utils.logging.send import sendSuccess, sendFailure, sendInfo, incorrectUsage def parseCommand(addr, authed, data): @@ -12,17 +12,12 @@ def parseCommand(addr, authed, data): warn("Got connection object with no instance in the address pool") return - success = lambda data: sendSuccess(addr, data) - failure = lambda data: sendFailure(addr, data) - info = lambda data: sendInfo(addr, data) + success = lambda data: sendSuccess(addr, data) # noqa: E731 + failure = lambda data: sendFailure(addr, data) # noqa: E731 + info = lambda data: sendInfo(addr, data) # noqa: E731 - incUsage = lambda mode: incorrectUsage(addr, mode) + incUsage = lambda mode: incorrectUsage(addr, mode) # noqa: E731 length = len(spl) - if len(spl) > 0: - cmd = spl[0] - else: - failure("No text was sent") - return if spl[0] in main.CommandMap.keys(): main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length) return diff --git a/core/relay.py b/core/relay.py index 8ea6c92..acd64cd 100644 --- a/core/relay.py +++ b/core/relay.py @@ -1,9 +1,7 @@ -from twisted.internet.protocol import Protocol, Factory, ClientFactory +from twisted.internet.protocol import Protocol, Factory from json import dumps, loads -from copy import deepcopy - import main -from utils.logging.log import * +from utils.logging.log import log, warn validTypes = [ "msg", @@ -47,10 +45,10 @@ class Relay(Protocol): data = data.decode("utf-8", "replace") try: parsed = loads(data) - except: + except: # noqa: E722 self.sendErr("MALFORMED") return - if not "type" in parsed.keys(): + if "type" not in parsed.keys(): self.sendErr("NOTYPE") return if parsed["type"] == "hello": @@ -87,7 +85,7 @@ class Relay(Protocol): self.sendErr("NOTLIST") return for i in lst: - if not i in validTypes: + if i not in validTypes: self.sendErr("NONEXISTANT") return if i in self.subscriptions: @@ -102,10 +100,10 @@ class Relay(Protocol): self.sendErr("NOTLIST") return for i in lst: - if not i in validTypes: + if i not in validTypes: self.sendErr("NONEXISTANT") return - if not i in self.subscriptions: + if i not in self.subscriptions: self.sendErr("NOTSUBSCRIBED") return del self.subscriptions[i] diff --git a/core/server.py b/core/server.py index 94af7a0..eda59e9 100644 --- a/core/server.py +++ b/core/server.py @@ -1,6 +1,6 @@ -from twisted.internet.protocol import Protocol, Factory, ClientFactory +from twisted.internet.protocol import Protocol, Factory import main -from utils.logging.log import * +from utils.logging.log import log, warn from core.parser import parseCommand @@ -9,7 +9,7 @@ class Server(Protocol): def __init__(self, addr): self.addr = addr self.authed = False - if main.config["UsePassword"] == False: + if main.config["UsePassword"] is False: self.authed = True def send(self, data): diff --git a/main.py b/main.py index 1b01f4d..be8e446 100644 --- a/main.py +++ b/main.py @@ -4,8 +4,6 @@ from redis import StrictRedis from string import digits from os import urandom -from utils.logging.log import * - # List of errors ZNC can give us ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] @@ -59,11 +57,12 @@ lastMinuteSample = 0 hashKey = urandom(16) lastEvents = {} + # Get networks that are currently online and dedupliate def liveNets(): networks = set() for i in IRCPool.keys(): - networks.add("".join([x for x in i if not x in digits])) + networks.add("".join([x for x in i if x not in digits])) return networks @@ -107,6 +106,6 @@ def initMain(): global r, g initConf() r = StrictRedis( - unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] + unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] # noqa ) # Ephemeral - flushed on quit - g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent + g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # noqa diff --git a/modules/alias.py b/modules/alias.py index 1961ec6..84b0adc 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -15,7 +15,7 @@ def generate_alias(): rand = random.randint(1, 4) while rand == 1: split = random.randint(0, len(nick) - 1) - nick = nick[:split] + nick[split + 1 :] + nick = nick[:split] + nick[split + 1 :] # noqa: E203 rand = random.randint(1, 4) rand = random.randint(1, 3) if rand == 1 or rand == 4: @@ -53,7 +53,7 @@ def generate_alias(): ident = namebase.split(" ")[0] ident = ident[:10] elif rand == 6: - ident = re.sub("\s", "", namebase).lower() + ident = re.sub("\s", "", namebase).lower() # noqa: W605 ident = ident[:10] realname = nick diff --git a/modules/chankeep.py b/modules/chankeep.py index ec43b0f..66c1764 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -1,6 +1,6 @@ import main -from utils.logging.log import * -from utils.logging.debug import * +from utils.logging.log import log, warn, error +from utils.logging.debug import debug from copy import deepcopy from math import ceil import modules.provision @@ -166,7 +166,7 @@ def _initialList(net, num, listinfo, chanlimit): sigrelay = ceil(siglength / chanlimit) relay = ceil(listlength / chanlimit) - netbase = "list.%s" % net + # netbase = "list.%s" % net abase = "analytics.list.%s" % net p = main.g.pipeline() p.hset(abase, "mean", mean) diff --git a/modules/counters.py b/modules/counters.py index a088cc3..21ad775 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -3,11 +3,11 @@ from twisted.internet.task import LoopingCall def event(name, eventType): - if not "local" in main.counters.keys(): + if "local" not in main.counters.keys(): main.counters["local"] = {} - if not "global" in main.counters.keys(): + if "global" not in main.counters.keys(): main.counters["global"] = {} - if not name in main.counters["local"].keys(): + if name not in main.counters["local"].keys(): main.counters["local"][name] = {} if eventType not in main.counters["local"][name].keys(): main.counters["local"][name][eventType] = 0 @@ -21,7 +21,7 @@ def event(name, eventType): def getEvents(name=None): - if name == None: + if name is None: if "global" in main.counters.keys(): return main.counters["global"] else: diff --git a/modules/monitor.py b/modules/monitor.py index f174a80..34d6c8c 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,11 +1,6 @@ -from copy import deepcopy -from json import dumps - -import main from core.relay import sendRelayNotification from core.logstash import sendLogstashNotification from modules import userinfo -from modules import regproc from utils.dedup import dedup order = [ @@ -29,7 +24,7 @@ order = [ def parsemeta(numName, c): - if not "channel" in c.keys(): + if "channel" not in c.keys(): c["channel"] = None # metadata scraping # need to check if this was received from a relay diff --git a/modules/network.py b/modules/network.py index 71fd3dc..be70f48 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,13 +1,12 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory -import json from modules import alias from modules.chankeep import nukeNetwork from modules.regproc import needToRegister from twisted.internet import reactor -from core.bot import IRCBot, IRCBotFactory +from core.bot import IRCBotFactory import main -from utils.logging.log import * +from utils.logging.log import log from utils.get import getRelay @@ -41,7 +40,7 @@ class Network: "registered": registered, } password = alias.generate_password() - if not num in main.alias.keys(): + if num not in main.alias.keys(): main.alias[num] = alias.generate_alias() main.saveConf("alias") self.aliases[num] = {"password": password} diff --git a/modules/provision.py b/modules/provision.py index c32aaca..e4a6c9d 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,6 +1,5 @@ import main from utils.deliver_relay_commands import deliverRelayCommands -from utils.logging.log import * from twisted.internet import reactor import modules.regproc @@ -64,7 +63,7 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): def provisionRelay(num, network): # provision user and network data aliasObj = main.alias[num] print("ALIASOBJ FALUES", aliasObj.values()) - alias = aliasObj["nick"] + # alias = aliasObj["nick"] provisionUserNetworkData( num, *aliasObj.values(), diff --git a/modules/regproc.py b/modules/regproc.py index f0a8335..6926ad6 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -1,7 +1,7 @@ import main from modules import provision -from utils.logging.log import * -from utils.logging.debug import * +from utils.logging.log import error +from utils.logging.debug import debug from copy import deepcopy from random import choice @@ -24,7 +24,7 @@ def selectInst(net): if net in main.irc.keys(): inst = deepcopy(main.irc[net]) for i in main.irc["_"].keys(): - if not i in inst: + if i not in inst: inst[i] = main.irc["_"][i] else: inst = main.irc["_"] @@ -50,7 +50,7 @@ def substitute(net, num, token=None): error(f"Could not get email for {net} - {num}") return False nickname = alias["nick"] - username = nickname + "/" + net + # username = nickname + "/" + net password = main.network[net].aliases[num]["password"] # inst["email"] = inst["email"].replace("{nickname}", nickname) for i in inst.keys(): @@ -88,7 +88,7 @@ def confirmRegistration(net, num, negativepass=None): obj = main.network[net] name = net + str(num) if name in main.IRCPool.keys(): - if not negativepass == None: + if negativepass is not None: main.IRCPool[name].regPing(negativepass=negativepass) return debug("Relay authenticated: %s - %i" % (net, num)) @@ -100,7 +100,7 @@ def confirmRegistration(net, num, negativepass=None): if main.IRCPool[name]._regAttempt: try: main.IRCPool[name]._regAttempt.cancel() - except: + except: # noqa pass obj.relays[num]["registered"] = True main.saveConf("network") @@ -112,22 +112,22 @@ def enableAuthentication(net, num): security = obj.security auth = obj.auth password = obj.aliases[num]["password"] - uname = main.alias[num]["nick"] + "/" + net + # uname = main.alias[num]["nick"] + "/" + net provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") - if selectInst(net)["check"] == False: + if selectInst(net)["check"] is False: confirmRegistration(net, num) def registerTest(c): sinst = substitute(c["net"], c["num"]) name = c["net"] + str(c["num"]) - if sinst["check"] == False: + if sinst["check"] is False: return - if "msg" in c.keys() and not c["msg"] == None: + if "msg" in c.keys() and not c["msg"] is None: if sinst["negative"]: if name in main.IRCPool.keys(): - if not main.IRCPool[name]._negativePass == True: + if main.IRCPool[name]._negativePass is not True: if c["type"] == "query" and c["nick"] == sinst["entity"]: if sinst["checknegativemsg"] in c["msg"]: confirmRegistration( @@ -150,6 +150,6 @@ def registerTest(c): elif sinst["checktype"] == "mode": if c["type"] == "self": if c["mtype"] == "mode": - if sinst["checkmode"] in c["mode"] and c["status"] == True: + if sinst["checkmode"] in c["mode"] and c["status"] is True: confirmRegistration(c["net"], c["num"]) return diff --git a/modules/userinfo.py b/modules/userinfo.py index 1722cb3..9e66463 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,9 +1,8 @@ from twisted.internet.threads import deferToThread -from string import digits import main -from utils.logging.log import * -from utils.logging.debug import debug, trace +from utils.logging.log import warn +from utils.logging.debug import trace from utils.parsing import parsen @@ -97,7 +96,7 @@ def _initialUsers(name, channel, users): def initialUsers(name, channel, users): trace("Initialising WHO records for %s on %s" % (channel, name)) - d = deferToThread(_initialUsers, name, channel, users) + deferToThread(_initialUsers, name, channel, users) # d.addCallback(testCallback) @@ -114,7 +113,7 @@ def _initialNames(name, channel, names): def initialNames(name, channel, names): trace("Initialising NAMES records for %s on %s" % (channel, name)) - d = deferToThread(_initialNames, name, channel, names) + deferToThread(_initialNames, name, channel, names) # d.addCallback(testCallback) @@ -260,5 +259,5 @@ def _delChannels(net, channels): def delChannels(net, channels): # we have left a channel trace("Purging channel %s for %s" % (", ".join(channels), net)) - d = deferToThread(_delChannels, net, channels) + deferToThread(_delChannels, net, channels) # d.addCallback(testCallback) diff --git a/threshold b/threshold index e844b23..893b455 100755 --- a/threshold +++ b/threshold @@ -10,33 +10,38 @@ from signal import signal, SIGINT from sys import stdout, stderr # Import again because we want to override from codecs import getwriter # fix printing odd shit to the terminal -stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know -stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using -# unicode quit messages for no reason + import main -main.initMain() from utils.cleanup import handler -signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine + +from utils.logging.log import log +from utils.loaders.command_loader import loadCommands +from core.server import ServerFactory +from core.relay import RelayFactory +import modules.counters +import core.logstash + +main.initMain() + if "--debug" in sys.argv: # yes really main.config["Debug"] = True if "--trace" in sys.argv: main.config["Trace"] = True -from utils.logging.log import * -from utils.loaders.command_loader import loadCommands -from core.server import Server, ServerFactory -from core.relay import Relay, RelayFactory -import modules.counters loadCommands() -import core.logstash + core.logstash.init_logstash() +signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine +stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know +stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using +# unicode quit messages for no reason if __name__ == "__main__": listener = ServerFactory() - if main.config["Listener"]["UseSSL"] == True: + if main.config["Listener"]["UseSSL"] is True: reactor.listenSSL( main.config["Listener"]["Port"], listener, @@ -59,7 +64,7 @@ if __name__ == "__main__": log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) if main.config["RelayAPI"]["Enabled"]: relay = RelayFactory() - if main.config["RelayAPI"]["UseSSL"] == True: + if main.config["RelayAPI"]["UseSSL"] is True: reactor.listenSSL( main.config["RelayAPI"]["Port"], relay, diff --git a/utils/cleanup.py b/utils/cleanup.py index 83ed661..aef11bc 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -1,8 +1,7 @@ import main from twisted.internet import reactor from utils.logging.debug import debug -from utils.logging.log import * -import sys +from utils.logging.log import log def handler(sig, frame): diff --git a/utils/dedup.py b/utils/dedup.py index dc8d4c3..89654f1 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -19,7 +19,7 @@ def dedup(numName, b): return True if numName in main.lastEvents.keys(): main.lastEvents[numName].insert(0, castHash) - main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] + main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] # noqa else: main.lastEvents[numName] = [castHash] return False diff --git a/utils/deliver_relay_commands.py b/utils/deliver_relay_commands.py index c0b44df..e02a70d 100644 --- a/utils/deliver_relay_commands.py +++ b/utils/deliver_relay_commands.py @@ -1,7 +1,162 @@ +import main +from twisted.internet.ssl import DefaultOpenSSLContextFactory +from twisted.internet import reactor +from twisted.words.protocols.irc import IRCClient + +from twisted.internet.protocol import ReconnectingClientFactory +from utils.parsing import parsen +from utils.logging.log import log, error +from utils.logging.send import sendAll +from modules import userinfo +from datetime import datetime +from core.relay import sendRelayNotification +from utils.get import getRelay + + +# TODO: strip out non-relay functionality +class IRCRelay(IRCClient): + def __init__(self, num, relayCommands, user, stage2): + self.isconnected = False + self.buffer = "" + if user is None: + self.user = main.config["Relay"]["User"] + else: + self.user = user.lower() + password = main.config["Relay"]["Password"] + self.nickname = "relay" + self.realname = "relay" + self.username = self.user + self.password = self.user + ":" + password + + self.relayCommands = relayCommands + self.num = num + self.stage2 = stage2 + self.loop = None + + def privmsg(self, user, channel, msg): + nick, ident, host = parsen(user) + for i in main.ZNCErrors: + if i in msg: + error("ZNC issue:", msg) + if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]: + nick = nick[1:] + if nick in self.relayCommands.keys(): + sendAll("[%s] %s -> %s" % (self.num, nick, msg)) + + def irc_ERR_PASSWDMISMATCH(self, prefix, params): + log("%s: relay password mismatch" % self.num) + sendAll("%s: relay password mismatch" % self.num) + + def sendStage2(self): + # [["user", {"sasl": ["message1", "message2"]}], []] + if not len(self.stage2) == 0: + user = self.stage2[0].pop(0) + commands = self.stage2[0].pop(0) + del self.stage2[0] + deliverRelayCommands(self.num, commands, user, self.stage2) + + def signedOn(self): + if not self.isconnected: + self.isconnected = True + # log("signed on as a relay: %s" % self.num) + sleeptime = 0 + increment = 0.8 + for i in self.relayCommands.keys(): + for x in self.relayCommands[i]: + reactor.callLater( + sleeptime, + self.msg, + main.config["Tweaks"]["ZNC"]["Prefix"] + i, + x, + ) + sleeptime += increment + increment += 0.8 + if self.stage2 is not None: + reactor.callLater(sleeptime, self.sendStage2) + reactor.callLater(sleeptime + 5, self.transport.loseConnection) + return + + +class IRCRelayFactory(ReconnectingClientFactory): + def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): + if net is None: + self.num = num + self.net = None + self.name = "relay - %i" % num + self.relay = True + else: + self.name = net + str(num) + self.num = num + self.net = net + self.relay = False + self.client = None + self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"] + self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"] + self.factor = main.config["Tweaks"]["Delays"]["Factor"] + self.jitter = main.config["Tweaks"]["Delays"]["Jitter"] + + self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 + + def buildProtocol(self, addr): + + entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) + + self.client = entry + return entry + + def clientConnectionLost(self, connector, reason): + if not self.relay: + userinfo.delChannels(self.net, self.client.channels) + if self.client is not None: + self.client.isconnected = False + self.client.authenticated = False + self.client.channels = [] + error = reason.getErrorMessage() + if not self.relay: + log("%s - %i: connection lost: %s" % (self.net, self.num, error)) + sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) + ctime = str(datetime.now().isoformat()) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "lost", + "message": error, + "ts": ctime, + } + ) + self.retry(connector) + # ReconnectingClientFactory.clientConnectionLost(self, connector, reason) + + def clientConnectionFailed(self, connector, reason): + if self.client is not None: + self.client.isconnected = False + self.client.authenticated = False + self.client.channels = [] + error = reason.getErrorMessage() + log("%s - %i: connection failed: %s" % (self.net, self.num, error)) + if not self.relay: + sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) + ctime = str(datetime.now().isoformat()) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "failed", + "message": error, + "ts": ctime, + } + ) + self.retry(connector) + # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) + + def deliverRelayCommands(num, relayCommands, user=None, stage2=None): keyFN = main.certPath + main.config["Key"] certFN = main.certPath + main.config["Certificate"] contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) - bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) + bot = IRCRelayFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) host, port = getRelay(num) - rct = reactor.connectSSL(host, port, bot, contextFactory) \ No newline at end of file + reactor.connectSSL(host, port, bot, contextFactory) diff --git a/utils/get.py b/utils/get.py index f455b6c..d5c2ec5 100644 --- a/utils/get.py +++ b/utils/get.py @@ -4,8 +4,8 @@ import main def getRelay(num): host = main.config["Relay"]["Host"].replace("x", str(num)) port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2))) - user = main.config["Relay"]["User"] - password = main.config["Relay"]["Password"] + # user = main.config["Relay"]["User"] + # password = main.config["Relay"]["Password"] try: port = int(port) except ValueError: diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 0fa6d5c..1181119 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,8 +1,7 @@ from os import listdir from utils.logging.debug import debug -from utils.logging.log import * -import commands +from utils.logging.log import error from main import CommandMap @@ -14,7 +13,7 @@ def loadCommands(allowDup=False): className = commandName.capitalize() + "Command" # try: module = __import__("commands.%s" % commandName) - if not commandName in CommandMap: + if commandName not in CommandMap: CommandMap[commandName] = getattr(getattr(module, commandName), className) debug("Registered command: %s" % commandName) else: diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index f5219ac..c87a963 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -3,8 +3,6 @@ from importlib import reload import sys from utils.logging.debug import debug -from utils.logging.log import * -import commands from main import CommandMap diff --git a/utils/logging/debug.py b/utils/logging/debug.py index 80651f7..0c73174 100644 --- a/utils/logging/debug.py +++ b/utils/logging/debug.py @@ -1,5 +1,6 @@ import main + # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main # to read the setting def debug(*data): diff --git a/utils/logging/send.py b/utils/logging/send.py index 0623243..e1a62af 100644 --- a/utils/logging/send.py +++ b/utils/logging/send.py @@ -32,7 +32,7 @@ def sendAll(data): def incorrectUsage(addr, mode): - if mode == None: + if mode is None: sendFailure(addr, "Incorrect usage") return if mode in main.help.keys(): From a5fd7d60fd81af5ba5d75c4e7425c9dd00df379e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:07 +0100 Subject: [PATCH 248/394] Remove some legacy code --- core/bot.py | 34 ++++++++++------------ core/factory.py | 77 ------------------------------------------------- 2 files changed, 16 insertions(+), 95 deletions(-) delete mode 100644 core/factory.py diff --git a/core/bot.py b/core/bot.py index 4f20576..7edec16 100644 --- a/core/bot.py +++ b/core/bot.py @@ -731,9 +731,8 @@ class IRCBotFactory(ReconnectingClientFactory): self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 def buildProtocol(self, addr): - if self.relay is False: - entry = IRCBot(self.net, self.num) - main.IRCPool[self.name] = entry + entry = IRCBot(self.net, self.num) + main.IRCPool[self.name] = entry self.client = entry return entry @@ -746,21 +745,20 @@ class IRCBotFactory(ReconnectingClientFactory): self.client.authenticated = False self.client.channels = [] error = reason.getErrorMessage() - if not self.relay: - log("%s - %i: connection lost: %s" % (self.net, self.num, error)) - sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) - ctime = str(datetime.now().isoformat()) - sendRelayNotification( - { - "type": "conn", - "net": self.net, - "num": self.num, - "status": "lost", - "message": error, - "ts": ctime, - } - ) - self.retry(connector) + log("%s - %i: connection lost: %s" % (self.net, self.num, error)) + sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) + ctime = str(datetime.now().isoformat()) + sendRelayNotification( + { + "type": "conn", + "net": self.net, + "num": self.num, + "status": "lost", + "message": error, + "ts": ctime, + } + ) + self.retry(connector) # ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): diff --git a/core/factory.py b/core/factory.py deleted file mode 100644 index 0d63e32..0000000 --- a/core/factory.py +++ /dev/null @@ -1,77 +0,0 @@ -class IRCBotFactory(ReconnectingClientFactory): - def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): - if net is None: - self.num = num - self.net = None - self.name = "relay - %i" % num - self.relay = True - else: - self.name = net + str(num) - self.num = num - self.net = net - self.relay = False - self.client = None - self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"] - self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"] - self.factor = main.config["Tweaks"]["Delays"]["Factor"] - self.jitter = main.config["Tweaks"]["Delays"]["Jitter"] - - self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2 - - def buildProtocol(self, addr): - if self.relay is False: - entry = IRCBot(self.net, self.num) - main.IRCPool[self.name] = entry - else: - entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2) - - self.client = entry - return entry - - def clientConnectionLost(self, connector, reason): - if not self.relay: - userinfo.delChannels(self.net, self.client.channels) - if self.client is not None: - self.client.isconnected = False - self.client.authenticated = False - self.client.channels = [] - error = reason.getErrorMessage() - if not self.relay: - log("%s - %i: connection lost: %s" % (self.net, self.num, error)) - sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) - ctime = str(datetime.now().isoformat()) - sendRelayNotification( - { - "type": "conn", - "net": self.net, - "num": self.num, - "status": "lost", - "message": error, - "ts": ctime, - } - ) - self.retry(connector) - # ReconnectingClientFactory.clientConnectionLost(self, connector, reason) - - def clientConnectionFailed(self, connector, reason): - if self.client is not None: - self.client.isconnected = False - self.client.authenticated = False - self.client.channels = [] - error = reason.getErrorMessage() - log("%s - %i: connection failed: %s" % (self.net, self.num, error)) - if not self.relay: - sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) - ctime = str(datetime.now().isoformat()) - sendRelayNotification( - { - "type": "conn", - "net": self.net, - "num": self.num, - "status": "failed", - "message": error, - "ts": ctime, - } - ) - self.retry(connector) - # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) From e5a14b2c91013e080cb6ff0a9d55a4ed6d944398 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:09 +0100 Subject: [PATCH 249/394] Reformat again --- .pre-commit-config.yaml | 16 +++++++++------ commands/alias.py | 3 ++- commands/blacklist.py | 3 ++- commands/dist.py | 3 ++- commands/email.py | 3 ++- commands/network.py | 8 +++++--- commands/relay.py | 3 ++- commands/stats.py | 3 ++- commands/token.py | 6 ++++-- core/bot.py | 36 ++++++++++++++------------------- core/logstash.py | 5 +++-- core/parser.py | 2 +- core/relay.py | 4 +++- core/server.py | 6 +++--- main.py | 5 +++-- modules/alias.py | 3 ++- modules/chankeep.py | 10 +++++---- modules/counters.py | 3 ++- modules/monitor.py | 2 +- modules/network.py | 8 ++++---- modules/provision.py | 5 +++-- modules/regproc.py | 9 +++++---- modules/userinfo.py | 2 +- threshold | 31 +++++++++++----------------- utils/cleanup.py | 3 ++- utils/dedup.py | 6 ++++-- utils/deliver_relay_commands.py | 17 ++++++++-------- utils/loaders/command_loader.py | 3 +-- utils/loaders/single_loader.py | 7 +++---- 29 files changed, 114 insertions(+), 101 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98fa1cb..673c866 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,17 @@ repos: -- repo: https://github.com/ambv/black - rev: 22.3.0 + - repo: https://github.com/psf/black + rev: 22.6.0 hooks: - id: black args: - --line-length=120 -- repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: - - id: flake8 - args: - - "--max-line-length=120" + - id: flake8 + args: [--max-line-length=120] diff --git a/commands/alias.py b/commands/alias.py index ce55037..fd5e644 100644 --- a/commands/alias.py +++ b/commands/alias.py @@ -1,5 +1,6 @@ -import main from yaml import dump + +import main from modules import alias diff --git a/commands/blacklist.py b/commands/blacklist.py index 42684a6..7dc3c19 100644 --- a/commands/blacklist.py +++ b/commands/blacklist.py @@ -1,6 +1,7 @@ -import main from yaml import dump +import main + class BlacklistCommand: def __init__(self, *args): diff --git a/commands/dist.py b/commands/dist.py index 2348e29..a126e07 100644 --- a/commands/dist.py +++ b/commands/dist.py @@ -1,5 +1,6 @@ +from subprocess import PIPE, run + import main -from subprocess import run, PIPE class DistCommand: diff --git a/commands/email.py b/commands/email.py index ea36da0..c388a56 100644 --- a/commands/email.py +++ b/commands/email.py @@ -1,6 +1,7 @@ -import main from yaml import dump +import main + class EmailCommand: def __init__(self, *args): diff --git a/commands/network.py b/commands/network.py index 4a6bdf2..a1da160 100644 --- a/commands/network.py +++ b/commands/network.py @@ -1,8 +1,10 @@ -import main -from yaml import dump -from modules.network import Network from string import digits +from yaml import dump + +import main +from modules.network import Network + class NetworkCommand: def __init__(self, *args): diff --git a/commands/relay.py b/commands/relay.py index 34bbd70..2b80f5e 100644 --- a/commands/relay.py +++ b/commands/relay.py @@ -1,6 +1,7 @@ -import main from yaml import dump +import main + class RelayCommand: def __init__(self, *args): diff --git a/commands/stats.py b/commands/stats.py index 7102dd1..e2e8eb9 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,7 +1,8 @@ +from string import digits + import main import modules.counters as count import modules.userinfo as userinfo -from string import digits class StatsCommand: diff --git a/commands/token.py b/commands/token.py index 7d38eac..b5c9cd5 100644 --- a/commands/token.py +++ b/commands/token.py @@ -1,7 +1,9 @@ -import main -from yaml import dump from uuid import uuid4 +from yaml import dump + +import main + class TokenCommand: def __init__(self, *args): diff --git a/core/bot.py b/core/bot.py index 7edec16..a148350 100644 --- a/core/bot.py +++ b/core/bot.py @@ -1,31 +1,25 @@ -from twisted.internet.protocol import ReconnectingClientFactory -from twisted.words.protocols.irc import IRCClient -from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall -from twisted.internet import reactor -from twisted.words.protocols.irc import ( - numeric_to_symbolic, - lowDequote, - IRCBadMessage, -) - import sys -from random import randint from copy import deepcopy from datetime import datetime +from random import randint -from modules import userinfo -from modules import counters -from modules import monitor -from modules import chankeep -from modules import regproc - -from core.relay import sendRelayNotification -from utils.dedup import dedup +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.internet.protocol import ReconnectingClientFactory +from twisted.internet.task import LoopingCall +from twisted.words.protocols.irc import ( + IRCBadMessage, + IRCClient, + lowDequote, + numeric_to_symbolic, +) import main -from utils.logging.log import log, warn, error +from core.relay import sendRelayNotification +from modules import chankeep, counters, monitor, regproc, userinfo +from utils.dedup import dedup from utils.logging.debug import debug +from utils.logging.log import error, log, warn from utils.logging.send import sendAll from utils.parsing import parsen diff --git a/core/logstash.py b/core/logstash.py index 94429bc..1e3c2c9 100644 --- a/core/logstash.py +++ b/core/logstash.py @@ -1,7 +1,8 @@ -import logstash import logging - from json import dumps + +import logstash + import main logger = None diff --git a/core/parser.py b/core/parser.py index 2f2383d..da0312c 100644 --- a/core/parser.py +++ b/core/parser.py @@ -1,6 +1,6 @@ import main from utils.logging.log import warn -from utils.logging.send import sendSuccess, sendFailure, sendInfo, incorrectUsage +from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess def parseCommand(addr, authed, data): diff --git a/core/relay.py b/core/relay.py index acd64cd..7ae0cba 100644 --- a/core/relay.py +++ b/core/relay.py @@ -1,5 +1,7 @@ -from twisted.internet.protocol import Protocol, Factory from json import dumps, loads + +from twisted.internet.protocol import Factory, Protocol + import main from utils.logging.log import log, warn diff --git a/core/server.py b/core/server.py index eda59e9..581d237 100644 --- a/core/server.py +++ b/core/server.py @@ -1,8 +1,8 @@ -from twisted.internet.protocol import Protocol, Factory -import main -from utils.logging.log import log, warn +from twisted.internet.protocol import Factory, Protocol +import main from core.parser import parseCommand +from utils.logging.log import log, warn class Server(Protocol): diff --git a/main.py b/main.py index be8e446..42950b9 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,9 @@ import json import pickle -from redis import StrictRedis -from string import digits from os import urandom +from string import digits + +from redis import StrictRedis # List of errors ZNC can give us ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] diff --git a/modules/alias.py b/modules/alias.py index 84b0adc..85cebc2 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -1,7 +1,8 @@ -import main import random import re +import main + def generate_password(): return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) diff --git a/modules/chankeep.py b/modules/chankeep.py index 66c1764..d22a755 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -1,11 +1,13 @@ -import main -from utils.logging.log import log, warn, error -from utils.logging.debug import debug from copy import deepcopy from math import ceil -import modules.provision + from twisted.internet.threads import deferToThread +import main +import modules.provision +from utils.logging.debug import debug +from utils.logging.log import error, log, warn + def allRelaysActive(net): relayNum = len(main.network[net].relays.keys()) diff --git a/modules/counters.py b/modules/counters.py index 21ad775..64407e7 100644 --- a/modules/counters.py +++ b/modules/counters.py @@ -1,6 +1,7 @@ -import main from twisted.internet.task import LoopingCall +import main + def event(name, eventType): if "local" not in main.counters.keys(): diff --git a/modules/monitor.py b/modules/monitor.py index 34d6c8c..7bdd662 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,5 +1,5 @@ -from core.relay import sendRelayNotification from core.logstash import sendLogstashNotification +from core.relay import sendRelayNotification from modules import userinfo from utils.dedup import dedup diff --git a/modules/network.py b/modules/network.py index be70f48..58f18ad 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,13 +1,13 @@ +from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory +import main +from core.bot import IRCBotFactory from modules import alias from modules.chankeep import nukeNetwork from modules.regproc import needToRegister -from twisted.internet import reactor -from core.bot import IRCBotFactory -import main -from utils.logging.log import log from utils.get import getRelay +from utils.logging.log import log class Network: diff --git a/modules/provision.py b/modules/provision.py index e4a6c9d..e1e740e 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -1,7 +1,8 @@ -import main -from utils.deliver_relay_commands import deliverRelayCommands from twisted.internet import reactor + +import main import modules.regproc +from utils.deliver_relay_commands import deliverRelayCommands def provisionUserNetworkData( diff --git a/modules/regproc.py b/modules/regproc.py index 6926ad6..3cc35bc 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -1,10 +1,11 @@ -import main -from modules import provision -from utils.logging.log import error -from utils.logging.debug import debug from copy import deepcopy from random import choice +import main +from modules import provision +from utils.logging.debug import debug +from utils.logging.log import error + def needToRegister(net): # Check if the network does not support authentication diff --git a/modules/userinfo.py b/modules/userinfo.py index 9e66463..d6b14e0 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,8 +1,8 @@ from twisted.internet.threads import deferToThread import main -from utils.logging.log import warn from utils.logging.debug import trace +from utils.logging.log import warn from utils.parsing import parsen diff --git a/threshold b/threshold index 893b455..8a9b866 100755 --- a/threshold +++ b/threshold @@ -1,27 +1,20 @@ #!/usr/bin/env python +import sys +from codecs import getwriter +from signal import SIGINT, signal +from sys import stderr, stdout + from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory -import sys -from signal import signal, SIGINT -# from twisted.python import log -# from sys import stdout -# log.startLogging(stdout) -from sys import stdout, stderr # Import again because we want to override -from codecs import getwriter # fix printing odd shit to the terminal - - -import main - -from utils.cleanup import handler - - -from utils.logging.log import log -from utils.loaders.command_loader import loadCommands -from core.server import ServerFactory -from core.relay import RelayFactory -import modules.counters import core.logstash +import main +import modules.counters +from core.relay import RelayFactory +from core.server import ServerFactory +from utils.cleanup import handler +from utils.loaders.command_loader import loadCommands +from utils.logging.log import log main.initMain() diff --git a/utils/cleanup.py b/utils/cleanup.py index aef11bc..057b1bc 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -1,5 +1,6 @@ -import main from twisted.internet import reactor + +import main from utils.logging.debug import debug from utils.logging.log import log diff --git a/utils/dedup.py b/utils/dedup.py index 89654f1..0565a0d 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -1,7 +1,9 @@ -from datetime import datetime -from csiphash import siphash24 from copy import deepcopy +from datetime import datetime from json import dumps + +from csiphash import siphash24 + import main from utils.logging.debug import debug diff --git a/utils/deliver_relay_commands.py b/utils/deliver_relay_commands.py index e02a70d..84f30f0 100644 --- a/utils/deliver_relay_commands.py +++ b/utils/deliver_relay_commands.py @@ -1,16 +1,17 @@ -import main -from twisted.internet.ssl import DefaultOpenSSLContextFactory +from datetime import datetime + from twisted.internet import reactor +from twisted.internet.protocol import ReconnectingClientFactory +from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.words.protocols.irc import IRCClient -from twisted.internet.protocol import ReconnectingClientFactory -from utils.parsing import parsen -from utils.logging.log import log, error -from utils.logging.send import sendAll -from modules import userinfo -from datetime import datetime +import main from core.relay import sendRelayNotification +from modules import userinfo from utils.get import getRelay +from utils.logging.log import error, log +from utils.logging.send import sendAll +from utils.parsing import parsen # TODO: strip out non-relay functionality diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index 1181119..2f0756a 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,10 +1,9 @@ from os import listdir +from main import CommandMap from utils.logging.debug import debug from utils.logging.log import error -from main import CommandMap - def loadCommands(allowDup=False): for filename in listdir("commands"): diff --git a/utils/loaders/single_loader.py b/utils/loaders/single_loader.py index c87a963..65ce147 100644 --- a/utils/loaders/single_loader.py +++ b/utils/loaders/single_loader.py @@ -1,10 +1,9 @@ -from os import listdir -from importlib import reload import sys - -from utils.logging.debug import debug +from importlib import reload +from os import listdir from main import CommandMap +from utils.logging.debug import debug def loadSingle(commandName): From f0acbdbfa3d377efc7b5f93974428a861dc3b1b0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:11 +0100 Subject: [PATCH 250/394] Begin work on API endpoint --- api/views.py | 12 ++++++++++++ commands/token.py | 2 +- conf/help.json | 2 +- core/bot.py | 3 ++- threshold | 12 +++++++++++- 5 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 api/views.py diff --git a/api/views.py b/api/views.py new file mode 100644 index 0000000..af68c64 --- /dev/null +++ b/api/views.py @@ -0,0 +1,12 @@ +from klein import Klein + + +class API(object): + """ + Our API webapp. + """ + + app = Klein() + @app.route("/test", methods=["GET"]) + def hello(self, request): + return "Hello" diff --git a/commands/token.py b/commands/token.py index b5c9cd5..738f537 100644 --- a/commands/token.py +++ b/commands/token.py @@ -34,7 +34,7 @@ class TokenCommand: elif length == 4: if spl[1] == "add": if not spl[2] in main.tokens.keys(): - if spl[3] in ["relay"]: # more to come! + if spl[3] in ["relay", "api"]: # more to come! main.tokens[spl[2]] = { "hello": str(uuid4()), "usage": spl[3], diff --git a/conf/help.json b/conf/help.json index b86ef14..2d40b4b 100644 --- a/conf/help.json +++ b/conf/help.json @@ -20,7 +20,7 @@ "alias": "alias [] []", "auto": "auto []", "cmd": "cmd ", - "token": "token [] []", + "token": "token [] [relay|api]", "all": "all ", "allc": "allc <(network)|(alias)> ", "admall": "admall ", diff --git a/core/bot.py b/core/bot.py index a148350..7c86beb 100644 --- a/core/bot.py +++ b/core/bot.py @@ -452,7 +452,8 @@ class IRCBot(IRCClient): def got_list(self, listinfo): if len(listinfo) == 0: # probably ngircd not supporting LIST >0 return - chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) + if main.config["ChanKeep"]["Enabled"]: + chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) def recheckList(self): allRelays = chankeep.allRelaysActive(self.net) diff --git a/threshold b/threshold index 8a9b866..9d0222a 100755 --- a/threshold +++ b/threshold @@ -5,17 +5,23 @@ from signal import SIGINT, signal from sys import stderr, stdout from twisted.internet import reactor + +# Webapp stuff +from twisted.internet.protocol import Factory from twisted.internet.ssl import DefaultOpenSSLContextFactory import core.logstash import main import modules.counters +from api.views import API from core.relay import RelayFactory from core.server import ServerFactory from utils.cleanup import handler from utils.loaders.command_loader import loadCommands from utils.logging.log import log +Factory.noisy = False + main.initMain() if "--debug" in sys.argv: # yes really @@ -84,4 +90,8 @@ if __name__ == "__main__": for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() - reactor.run() + if main.config["API"]["Enabled"]: + api = API() + api.app.run(main.config["API"]["Address"], main.config["API"]["Port"]) + else: + reactor.run() From f942e94ee56c8d4d16fb3881ff8f83c45c029fd2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:13 +0100 Subject: [PATCH 251/394] Implement API --- api/views.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index af68c64..317a0d5 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,34 @@ +import functools +from json import dumps + from klein import Klein +from twisted.web.server import Request + +import main +from modules import userinfo + + +def login_required(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + if isinstance(args[0], Request): + request = args[0] + apikey = request.getHeader("ApiKey") + token = request.getHeader("Token") + if not apikey: + return "No API key provided" + if not token: + return "No token provided" + if apikey not in main.tokens: + return "No such API key" + config_token = main.tokens[apikey] + if not token == config_token["hello"]: + return "Invalid token" + counter = config_token["counter"] + request.setHeader("Counter", counter) + return func(self, *args, **kwargs) + + return wrapper class API(object): @@ -7,6 +37,29 @@ class API(object): """ app = Klein() - @app.route("/test", methods=["GET"]) + + @app.route("/test/", methods=["GET"]) + @login_required def hello(self, request): return "Hello" + + @app.route("/who//", methods=["GET"]) + @login_required + def who(self, request, query): + result = userinfo.getWho(query) + # Expand the generator + return dumps({k: [x for x in v] for k, v in result.items()}) + + @app.route("/chans//", methods=["GET"]) + @login_required + def chans(self, request, query): + query = query.split(",") + result = userinfo.getChans(query) + return dumps({k: [x for x in v] for k, v in result.items()}) + + @app.route("/users//", methods=["GET"]) + @login_required + def users(self, request, query): + query = [f"#{x}" for x in query.split(",")] + result = userinfo.getUsers(query) + return dumps({k: [x for x in v] for k, v in result.items()}) From 47a3f84c1c262f57215a9442dd46e45f1be42423 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:15 +0100 Subject: [PATCH 252/394] Update config --- conf/example/config.json | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/conf/example/config.json b/conf/example/config.json index 9471a47..9ea4fa5 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -10,25 +10,33 @@ "Address": "127.0.0.1", "UseSSL": true }, + "API": { + "Enabled": true, + "Port": 13869, + "Address": "127.0.0.1" + }, "Key": "key.pem", "Certificate": "cert.pem", - "RedisSocket": "/tmp/redis.sock", - "UsePassword": true, + "RedisSocket": "/var/run/redis/redis.sock", + "RedisDBEphemeral": 2, + "RedisDBPersistent": 3, + "UsePassword": false, "ConnectOnCreate": false, "AutoReg": false, "Debug": false, - "Trace", false, + "Trace": false, "Relay": { "Host": "127.0.0.1", - "Port": "201x", - "User": "sir", - "Password": "sir" + "Port": "2001", + "User": "x", + "Password": "x" }, "Logstash": { - "Host": "10.0.0.2", + "Host": "127.0.0.1", "Port": "4000" }, "ChanKeep": { + "Enabled": false, "MaxRelay": 30, "SigSwitch": 20 }, @@ -38,10 +46,10 @@ "File": "conf/dist.sh" }, "Toggles": { - "Who": false, + "Who": true, "CycleChans": true }, - "Password": "s", + "Password": "x", "Tweaks": { "MaxHash": 10, "DedupPrecision": 9, From f589c7fc16c501203ad2638d7d0cbc0c33b76900 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:17 +0100 Subject: [PATCH 253/394] Implement more API functions --- api/views.py | 84 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/api/views.py b/api/views.py index 317a0d5..98f49bc 100644 --- a/api/views.py +++ b/api/views.py @@ -1,11 +1,12 @@ import functools -from json import dumps +from json import JSONDecodeError, dumps, loads from klein import Klein from twisted.web.server import Request import main from modules import userinfo +from utils.logging.log import warn def login_required(func): @@ -38,28 +39,83 @@ class API(object): app = Klein() - @app.route("/test/", methods=["GET"]) + @app.route("/", methods=["GET"]) @login_required def hello(self, request): return "Hello" - @app.route("/who//", methods=["GET"]) + @app.route("/who//", methods=["POST"]) @login_required def who(self, request, query): - result = userinfo.getWho(query) + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "query" not in data: + return "No query provided" + result = userinfo.getWho(data["query"]) # Expand the generator return dumps({k: [x for x in v] for k, v in result.items()}) - @app.route("/chans//", methods=["GET"]) + @app.route("/chans/", methods=["POST"]) @login_required - def chans(self, request, query): - query = query.split(",") - result = userinfo.getChans(query) - return dumps({k: [x for x in v] for k, v in result.items()}) + def chans(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "net" not in data: + return "No net provided" + if "query" not in data: + return "No query provided" + if not data["query"]: + warn(f"No query provided: for chans {data}") + return dumps({}) + result = userinfo.getChansSingle(data["net"], data["query"]) + if not result: + return dumps({}) + return dumps({"chans": [x for x in result]}) - @app.route("/users//", methods=["GET"]) + @app.route("/users/", methods=["POST"]) @login_required - def users(self, request, query): - query = [f"#{x}" for x in query.split(",")] - result = userinfo.getUsers(query) - return dumps({k: [x for x in v] for k, v in result.items()}) + def users(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "net" not in data: + return "No net provided" + if "query" not in data: + return "No query provided" + if not data["query"]: + warn(f"No query provided for users: {data}") + return dumps({}) + result = userinfo.getUsersSingle(data["net"], data["query"]) + if not result: + return dumps({}) + return dumps({"users": [x for x in result]}) + + @app.route("/online/", methods=["POST"]) + @login_required + def online(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "net" not in data: + return "No net provided" + if "query" not in data: + return "No users provided" + if not data["query"]: + warn(f"No users provided: for online {data}") + return dumps({}) + net = data["net"] + usermap = {} + for user in data["query"]: + channels = userinfo.getChansSingle(net, [user]) + if channels: + usermap[user] = True + else: + usermap[user] = False + + return dumps(usermap) From 4b33559e65dcca78fef1855e5f6f8384a2fa535d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:18 +0100 Subject: [PATCH 254/394] Implement getting number of channels and users --- api/views.py | 38 ++++++++++++++++++++++++++++++++++++++ modules/userinfo.py | 24 +++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 98f49bc..84a5356 100644 --- a/api/views.py +++ b/api/views.py @@ -119,3 +119,41 @@ class API(object): usermap[user] = False return dumps(usermap) + + @app.route("/num_users/", methods=["POST"]) + @login_required + def num_users(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "net" not in data: + return "No net provided" + if "query" not in data: + return "No users provided" + if not data["query"]: + warn(f"No chans provided: for online {data}") + return dumps({}) + net = data["net"] + results = userinfo.getUserNum(net, data["query"]) + + return dumps(results) + + @app.route("/num_chans/", methods=["POST"]) + @login_required + def num_chans(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "net" not in data: + return "No net provided" + if "query" not in data: + return "No users provided" + if not data["query"]: + warn(f"No users provided: for online {data}") + return dumps({}) + net = data["net"] + results = userinfo.getChanNum(net, data["query"]) + + return dumps(results) diff --git a/modules/userinfo.py b/modules/userinfo.py index d6b14e0..38246e3 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -23,7 +23,7 @@ def getWho(query): def getChansSingle(name, nick): - nick = ["live.chan." + name + "." + i for i in nick] + nick = ("live.chan." + name + "." + i for i in nick) result = main.r.sinter(*nick) if len(result) == 0: return None @@ -38,6 +38,28 @@ def getChanList(name, nick): return (i.decode() for i in result) +def getUserNum(name, channels): + """ + Get the number of users on a list of channels. + """ + chanspace = ("live.who." + name + "." + i for i in channels) + results = {} + for channel, space in zip(channels, chanspace): + results[channel] = main.r.scard(space) + return results + + +def getChanNum(name, nicks): + """ + Get the number of channels a list of users are on. + """ + nickspace = ("live.chan." + name + "." + i for i in nicks) + results = {} + for nick, space in zip(nicks, nickspace): + results[nick] = main.r.scard(space) + return results + + def getChans(nick): result = {} for i in main.network.keys(): From b14988612896f75e65b63ea3a21bcac988f8b370 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:40:40 +0100 Subject: [PATCH 255/394] Don't send to Logstash if it's disabled --- modules/monitor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/monitor.py b/modules/monitor.py index 7bdd662..1ada7e5 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -2,6 +2,7 @@ from core.logstash import sendLogstashNotification from core.relay import sendRelayNotification from modules import userinfo from utils.dedup import dedup +import main order = [ "type", @@ -74,5 +75,6 @@ def event(numName, c): # yes I'm using a short variable because otherwise it go del c["muser"] sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order sortedKeys["src"] = "irc" - sendLogstashNotification(sortedKeys) + if main.config["Logstash"]["Enabled"]: + sendLogstashNotification(sortedKeys) sendRelayNotification(sortedKeys) From 5aebf63c2e3dd3926a11d44f3f265e2d8b0f6d09 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 25 Jul 2022 18:05:53 +0100 Subject: [PATCH 256/394] Implement API endpoint for network listing --- api/views.py | 33 +++++++++++++++++++++++++++++++++ modules/monitor.py | 2 +- modules/userinfo.py | 8 ++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 84a5356..7912a62 100644 --- a/api/views.py +++ b/api/views.py @@ -157,3 +157,36 @@ class API(object): results = userinfo.getChanNum(net, data["query"]) return dumps(results) + + @app.route("/irc/stats/", methods=["POST"]) + @login_required + def irc_stats(self, request): + stats = {} + numChannels = 0 + numWhoEntries = 0 + for i in main.IRCPool.keys(): + numChannels += len(main.IRCPool[i].channels) + numWhoEntries += userinfo.getNumTotalWhoEntries() + numRelays = 0 + for net in main.network.keys(): + numRelays += len(main.network[net].relays) + stats["servers_total_total"] = numRelays + stats["servers_total_unique"] = len(main.network.keys()) + stats["servers_online_total"] = len(main.IRCPool.keys()) + stats["servers_online_unique"] = len(main.liveNets()) + stats["channels"] = numChannels + stats["records"] = numWhoEntries + stats["eventrate"] = main.lastMinuteSample + return dumps(stats) + + @app.route("/irc/networks/", methods=["POST"]) + @login_required + def irc_networks(self, request): + networks = {} + for net in main.network.keys(): + networks[net] = { + "relays": len(main.network[net].relays), + "channels": userinfo.getTotalChanNum(net), + "records": userinfo.getNumWhoEntries(net), + } + return dumps(networks) diff --git a/modules/monitor.py b/modules/monitor.py index 1ada7e5..ae4905c 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,8 +1,8 @@ +import main from core.logstash import sendLogstashNotification from core.relay import sendRelayNotification from modules import userinfo from utils.dedup import dedup -import main order = [ "type", diff --git a/modules/userinfo.py b/modules/userinfo.py index 38246e3..d281674 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -38,6 +38,14 @@ def getChanList(name, nick): return (i.decode() for i in result) +def getTotalChanNum(net): + """ + Get the number of channels a network has. + """ + chans = main.r.keys(f"live.who.{net}.*") + return len(chans) + + def getUserNum(name, channels): """ Get the number of users on a list of channels. From b9c1470410f5c16997763959e7d7758b962b37d6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 26 Jul 2022 22:16:35 +0100 Subject: [PATCH 257/394] Implement network and channels view --- api/views.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 7912a62..38566e0 100644 --- a/api/views.py +++ b/api/views.py @@ -132,7 +132,7 @@ class API(object): if "query" not in data: return "No users provided" if not data["query"]: - warn(f"No chans provided: for online {data}") + warn(f"No chans provided: for num_users {data}") return dumps({}) net = data["net"] results = userinfo.getUserNum(net, data["query"]) @@ -151,7 +151,7 @@ class API(object): if "query" not in data: return "No users provided" if not data["query"]: - warn(f"No users provided: for online {data}") + warn(f"No users provided: for num_chans {data}") return dumps({}) net = data["net"] results = userinfo.getChanNum(net, data["query"]) @@ -190,3 +190,58 @@ class API(object): "records": userinfo.getNumWhoEntries(net), } return dumps(networks) + + @app.route("/irc/network//", methods=["POST"]) + @login_required + def irc_network(self, request, net): + if net not in main.network.keys(): + return dumps(False) + inst = main.network[net] + network = {} + network["net"] = inst.net + network["auth"] = inst.auth + network["host"] = inst.host + network["last"] = inst.last + network["port"] = inst.port + network["security"] = inst.security + network["relays"] = len(inst.relays) + network["channels"] = userinfo.getTotalChanNum(net) + network["records"] = userinfo.getNumWhoEntries(net) + return dumps(network) + + @app.route("/irc/network//relays/", methods=["POST"]) + @login_required + def irc_network_relays(self, request, net): + if net not in main.network.keys(): + return dumps(False) + relays_inst = main.network[net].relays + relays = [] + for num in relays_inst.keys(): + to_append = relays_inst[num] + name = f"{net}{num}" + if name in main.IRCPool.keys(): + to_append["chans"] = len(main.IRCPool[name].channels) + to_append["nick"] = main.IRCPool[name].nickname + else: + to_append["chans"] = 0 + to_append["nick"] = None + relays.append(to_append) + + return dumps({"relays": relays}) + + @app.route("/irc/network//channels/", methods=["POST"]) + @login_required + def irc_network_channels(self, request, net): + if net not in main.network.keys(): + return dumps(False) + relays_inst = main.network[net].relays + channels = {} + for num in relays_inst.keys(): + name = f"{net}{num}" + if name in main.IRCPool.keys(): + net_chans = main.IRCPool[name].channels + channels_annotated = userinfo.getUserNum(net, net_chans) + for channel in net_chans: + channels[channel] = channels_annotated[channel] + + return dumps({"channels": channels}) From b30a3a535d57430923ba80dc3c09caa65d6a6989 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 27 Jul 2022 08:59:17 +0100 Subject: [PATCH 258/394] Implement editing networks via the API --- api/views.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/api/views.py b/api/views.py index 38566e0..8b56948 100644 --- a/api/views.py +++ b/api/views.py @@ -209,6 +209,45 @@ class API(object): network["records"] = userinfo.getNumWhoEntries(net) return dumps(network) + @app.route("/irc/network//edit/", methods=["POST"]) + @login_required + def irc_network_edit(self, request, net): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if net not in main.network.keys(): + return dumps(False) + inst = main.network[net] + for item in data: + if item == "auth": + auth = data[item][0] + if auth not in ["sasl", "ns", "none"]: + return dumps({"success": False, "reason": "invalid auth."}) + elif item == "host": + host = data[item][0] + elif item == "last": + last = data[item][0] + if not last.isdigit(): + return dumps({"success": False, "reason": "invalid last: not a number."}) + elif item == "port": + port = data[item][0] + if not port.isdigit(): + return dumps({"success": False, "reason": "invalid port: not a number."}) + port = int(port) + elif item == "security": + security = data[item][0] + if security not in ["ssl", "plain"]: + return dumps({"success": False, "reason": "invalid security."}) + inst.auth = auth + inst.host = host + inst.last = last + inst.port = port + inst.security = security + main.saveConf("network") + + return dumps({"success": True}) + @app.route("/irc/network//relays/", methods=["POST"]) @login_required def irc_network_relays(self, request, net): From 8409a39e57717a5f070b0173bc44c1ad43eed1be Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Wed, 27 Jul 2022 22:03:42 +0100 Subject: [PATCH 259/394] Implement relay, channel and alias management --- api/views.py | 67 +++++++++++++++++++++++++++++++++++++++++---- modules/chankeep.py | 41 +++++++++++++++++++++++---- modules/network.py | 37 +++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 10 deletions(-) diff --git a/api/views.py b/api/views.py index 8b56948..94e046c 100644 --- a/api/views.py +++ b/api/views.py @@ -5,7 +5,7 @@ from klein import Klein from twisted.web.server import Request import main -from modules import userinfo +from modules import chankeep, userinfo from utils.logging.log import warn @@ -195,7 +195,7 @@ class API(object): @login_required def irc_network(self, request, net): if net not in main.network.keys(): - return dumps(False) + return dumps({"success": False, "reason": "no such net."}) inst = main.network[net] network = {} network["net"] = inst.net @@ -217,7 +217,7 @@ class API(object): except JSONDecodeError: return "Invalid JSON" if net not in main.network.keys(): - return dumps(False) + return dumps({"success": False, "reason": "no such net."}) inst = main.network[net] for item in data: if item == "auth": @@ -252,7 +252,7 @@ class API(object): @login_required def irc_network_relays(self, request, net): if net not in main.network.keys(): - return dumps(False) + return dumps({"success": False, "reason": "no such net."}) relays_inst = main.network[net].relays relays = [] for num in relays_inst.keys(): @@ -268,11 +268,37 @@ class API(object): return dumps({"relays": relays}) + @app.route("/irc/network///", methods=["POST"]) + @login_required + def irc_network_relay(self, request, net, num): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + net_inst = main.network[net] + if num not in net_inst.relays: + return dumps({"success": False, "reason": "network has no such relay."}) + if "status" in data: + if not type(data["status"]) == int: + return dumps({"success": False, "reason": "invalid type for enabled."}) + enabled = data["status"] + if enabled: + net_inst.enable_relay(num) + else: + net_inst.disable_relay(num) + main.saveConf("network") + return dumps({"success": True}) + @app.route("/irc/network//channels/", methods=["POST"]) @login_required def irc_network_channels(self, request, net): if net not in main.network.keys(): - return dumps(False) + return dumps({"success": False, "reason": "no such net."}) relays_inst = main.network[net].relays channels = {} for num in relays_inst.keys(): @@ -284,3 +310,34 @@ class API(object): channels[channel] = channels_annotated[channel] return dumps({"channels": channels}) + + @app.route("/irc/network//channel//", methods=["DELETE"]) + @login_required + def irc_network_channel_part(self, request, net, channel): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + parted = chankeep.partSingle(net, channel) + if not parted: + dumps({"success": False, "reason": "no channels matched."}) + return dumps({"success": True, "relays": parted}) + + @app.route("/irc/network//channel//", methods=["PUT"]) + @login_required + def irc_network_channel_join(self, request, net, channel): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + joined = chankeep.joinSingle(net, channel) + if not joined: + dumps({"success": False, "reason": "no channels joined."}) + return dumps({"success": True, "relays": joined}) + + @app.route("/aliases/", methods=["GET"]) + @login_required + def aliases(self, request): + alias_list = [] + for num, alias in main.alias.items(): + alias_dup = dict(alias) + alias_dup["num"] = num + alias_list.append(alias_dup) + + return dumps({"aliases": alias_list}) diff --git a/modules/chankeep.py b/modules/chankeep.py index d22a755..59c0e0d 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -23,6 +23,15 @@ def allRelaysActive(net): def getChanFree(net, new): + """ + Get a dictionary with the free channel spaces for + each relay, and a channel limit. + Example return: + ({1: 99}, 100) + :param net: network + :param new: list of newly provisioned relays to skip + :return: ({relay: channel spaces}, channel limit) + """ chanfree = {} chanlimits = set() for i in main.network[net].relays.keys(): @@ -123,16 +132,38 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): def joinSingle(net, channel): if allRelaysActive(net): - chanfree = getChanFree(net, []) - print("chanfree", chanfree) - for i in chanfree[0]: - if chanfree[0][i] < 0: - print("JOIN CHAN") + # Use the algorithm to allocate our channel to a relay + eca = emptyChanAllocate(net, [channel], None, []) + if not len(eca.keys()) == 1: + return False + num = list(eca.keys())[0] + name = f"{net}{num}" + if name not in main.IRCPool: + return False + main.IRCPool[name].join(channel) + return num else: error("All relays for %s are not active" % net) return False +def partSingle(net, channel): + """ + Iterate over all the relays of net and part channels matching channel. + :param net: + :param channel: + :return: + """ + parted = [] + for i in main.network[net].relays.keys(): + name = f"{net}{i}" + if name in main.IRCPool.keys(): + if channel in main.IRCPool[name].channels: + main.IRCPool[name].part(channel) + parted.append(str(i)) + return parted + + def nukeNetwork(net): # purgeRecords(net) # p = main.g.pipeline() diff --git a/modules/network.py b/modules/network.py index 58f18ad..ccfbb32 100644 --- a/modules/network.py +++ b/modules/network.py @@ -6,6 +6,7 @@ from core.bot import IRCBotFactory from modules import alias from modules.chankeep import nukeNetwork from modules.regproc import needToRegister +from utils.deliver_relay_commands import deliverRelayCommands from utils.get import getRelay from utils.logging.log import log @@ -48,6 +49,42 @@ class Network: # self.start_bot(num) return num, main.alias[num]["nick"] + def enable_relay(self, num): + """ + Enable a relay for this network. + Send a command to ZNC to connect. + """ + self.relays[num]["enabled"] = True + user = main.alias[num]["nick"] + commands = {"status": ["Connect"]} + name = f"{self.net}{num}" + deliverRelayCommands(num, commands, user=user + "/" + self.net) + main.saveConf("network") + if name not in main.IRCPool.keys(): + self.start_bot(num) + + def disable_relay(self, num): + """ + Disable a relay for this network. + Send a command to ZNC to disconnect. + Stop trying to connect to the relay. + """ + self.relays[num]["enabled"] = False + user = main.alias[num]["nick"] + # relay = main.network[spl[1]].relays[relayNum] + commands = {"status": ["Disconnect"]} + name = f"{self.net}{num}" + deliverRelayCommands(num, commands, user=user + "/" + self.net) + main.saveConf("network") + if name in main.ReactorPool.keys(): + if name in main.FactoryPool.keys(): + main.FactoryPool[name].stopTrying() + main.ReactorPool[name].disconnect() + if name in main.IRCPool.keys(): + del main.IRCPool[name] + del main.ReactorPool[name] + del main.FactoryPool[name] + def killAliases(self, aliasList): for i in aliasList: name = self.net + str(i) From 4e195b2954e2d734ac8114de4f44620ca08c2585 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 19:21:08 +0100 Subject: [PATCH 260/394] Add docker definitions --- Dockerfile | 18 ++++++++++++++++++ docker-compose.yml | 34 ++++++++++++++++++++++++++++++++++ docker/redis.conf | 2 ++ requirements.txt | 1 + 4 files changed, 55 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker/redis.conf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6c63843 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 +FROM python:3 + +RUN useradd -d /code pathogen +RUN mkdir /code +RUN chown pathogen:pathogen /code + +RUN mkdir /venv +RUN chown pathogen:pathogen /venv + +USER pathogen +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +WORKDIR /code +COPY requirements.txt /code/ +RUN python -m venv /venv +RUN . /venv/bin/activate && pip install -r requirements.txt +CMD . /venv/bin/activate && exec python /code/threshold \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ec514e4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +version: "3.9" + +services: + app: + image: pathogen/threshold + build: . + volumes: + - .:/code + ports: + - "13867:13867" + - "13868:13868" + - "13869:13869" + # for development + extra_hosts: + - "host.docker.internal:host-gateway" + + tmp: + image: busybox + command: chmod -R 777 /var/run/redis + volumes: + - /var/run/redis + + redis: + image: redis + command: redis-server /etc/redis.conf + volumes: + - ./docker/redis.conf:/etc/redis.conf + volumes_from: + - tmp + +networks: + default: + external: + name: pathogen \ No newline at end of file diff --git a/docker/redis.conf b/docker/redis.conf new file mode 100644 index 0000000..46366bf --- /dev/null +++ b/docker/redis.conf @@ -0,0 +1,2 @@ +unixsocket /var/run/redis/redis.sock +unixsocketperm 777 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8d6b25a..375a8eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pyYaML python-logstash service_identity csiphash +Klein From e3700e309dfc6549f026210c6bfade1d191df1b9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 19:25:15 +0100 Subject: [PATCH 261/394] Lower compose version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ec514e4..ea35296 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.9" +version: "2" services: app: From f66f998f54325451deafbcf35dab7332b3526a21 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 19:38:37 +0100 Subject: [PATCH 262/394] Make some addresses and hosts configurable with environment variables --- .gitignore | 1 + threshold | 49 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 34d919c..2a4c599 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ conf/irc.json conf/dist.sh conf/blacklist.json env/ +.idea/ diff --git a/threshold b/threshold index 9d0222a..5c9e032 100755 --- a/threshold +++ b/threshold @@ -19,6 +19,7 @@ from core.server import ServerFactory from utils.cleanup import handler from utils.loaders.command_loader import loadCommands from utils.logging.log import log +from os import getenv Factory.noisy = False @@ -38,9 +39,25 @@ stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using # unicode quit messages for no reason +# Main listener +listener_address = getenv("THRESHOLD_LISTENER_HOST") or main.config["Listener"]["Address"] +listener_port = int(getenv("THRESHOLD_LISTENER_PORT")) or main.config["Listener"]["Port"] +listener_ssl = int(getenv("THRESHOLD_LISTENER_SSL")) or main.config["Listener"]["UseSSL"] + +# RelayAPI +relay_enabled = int(getenv("THRESHOLD_RELAY_ENABLED")) or main.config["Relay"]["Enabled"] +relay_address = getenv("THRESHOLD_RELAY_HOST") or main.config["RelayAPI"]["Address"] +relay_port = int(getenv("THRESHOLD_RELAY_PORT")) or main.config["RelayAPI"]["Port"] +relay_ssl = int(getenv("THRESHOLD_RELAY_SSL")) or main.config["RelayAPI"]["UseSSL"] + +# Web API +api_enabled = int(getenv("THRESHOLD_API_ENABLED")) or main.config["API"]["Enabled"] +api_address = getenv("THRESHOLD_API_HOST") or main.config["API"]["Address"] +api_port = int(getenv("THRESHOLD_API_PORT")) or main.config["API"]["Port"] + if __name__ == "__main__": listener = ServerFactory() - if main.config["Listener"]["UseSSL"] is True: + if listener_ssl is True: reactor.listenSSL( main.config["Listener"]["Port"], listener, @@ -48,50 +65,50 @@ if __name__ == "__main__": main.certPath + main.config["Key"], main.certPath + main.config["Certificate"], ), - interface=main.config["Listener"]["Address"], + interface=listener_address, ) log( "Threshold running with SSL on %s:%s" - % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]) + % (listener_address, listener_port) ) else: reactor.listenTCP( - main.config["Listener"]["Port"], + listener_port, listener, - interface=main.config["Listener"]["Address"], + interface=listener_address, ) - log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) - if main.config["RelayAPI"]["Enabled"]: + log("Threshold running on %s:%s" % (listener_address, listener_port)) + if relay_enabled: relay = RelayFactory() - if main.config["RelayAPI"]["UseSSL"] is True: + if relay_ssl is True: reactor.listenSSL( - main.config["RelayAPI"]["Port"], + relay_port, relay, DefaultOpenSSLContextFactory( main.certPath + main.config["Key"], main.certPath + main.config["Certificate"], ), - interface=main.config["RelayAPI"]["Address"], + interface=relay_address, ) log( "Threshold relay running with SSL on %s:%s" - % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]) + % (relay_address, relay_port) ) else: reactor.listenTCP( - main.config["RelayAPI"]["Port"], + relay_port, relay, - interface=main.config["RelayAPI"]["Address"], + interface=relay_address, ) log( "Threshold relay running on %s:%s" - % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]) + % (relay_address, relay_port) ) for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() - if main.config["API"]["Enabled"]: + if api_enabled: api = API() - api.app.run(main.config["API"]["Address"], main.config["API"]["Port"]) + api.app.run(api_address, api_port) else: reactor.run() From a258ec8ad15d2c11d5beed0f64f0ab41db1a5c62 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 19:50:07 +0100 Subject: [PATCH 263/394] Properly pass environment variables to the process --- .env | 3 +++ docker-compose.yml | 8 +++++--- threshold | 24 ++++++++++++++---------- 3 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..f46978d --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +THRESHOLD_LISTENER_PORT=13867 +THRESHOLD_RELAY_PORT=13868 +THRESHOLD_API_PORT=13869 diff --git a/docker-compose.yml b/docker-compose.yml index ea35296..9cb401e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,11 @@ services: volumes: - .:/code ports: - - "13867:13867" - - "13868:13868" - - "13869:13869" + - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" + - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" + - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" + env_file: + - .env # for development extra_hosts: - "host.docker.internal:host-gateway" diff --git a/threshold b/threshold index 5c9e032..edf71da 100755 --- a/threshold +++ b/threshold @@ -39,21 +39,25 @@ stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using # unicode quit messages for no reason +trues = ('true', '1', 't', True) + # Main listener -listener_address = getenv("THRESHOLD_LISTENER_HOST") or main.config["Listener"]["Address"] -listener_port = int(getenv("THRESHOLD_LISTENER_PORT")) or main.config["Listener"]["Port"] -listener_ssl = int(getenv("THRESHOLD_LISTENER_SSL")) or main.config["Listener"]["UseSSL"] +listener_address = getenv("THRESHOLD_LISTENER_HOST", main.config["Listener"]["Address"]) +listener_port = int(getenv("THRESHOLD_LISTENER_PORT", main.config["Listener"]["Port"])) +listener_ssl = getenv("THRESHOLD_LISTENER_SSL", main.config["Listener"]["UseSSL"]) in trues # RelayAPI -relay_enabled = int(getenv("THRESHOLD_RELAY_ENABLED")) or main.config["Relay"]["Enabled"] -relay_address = getenv("THRESHOLD_RELAY_HOST") or main.config["RelayAPI"]["Address"] -relay_port = int(getenv("THRESHOLD_RELAY_PORT")) or main.config["RelayAPI"]["Port"] -relay_ssl = int(getenv("THRESHOLD_RELAY_SSL")) or main.config["RelayAPI"]["UseSSL"] +relay_enabled = getenv("THRESHOLD_RELAY_ENABLED", main.config["RelayAPI"]["Enabled"]) in trues +relay_address = getenv("THRESHOLD_RELAY_HOST", main.config["RelayAPI"]["Address"]) +relay_port = int(getenv("THRESHOLD_RELAY_PORT", main.config["RelayAPI"]["Port"])) +relay_ssl = getenv("THRESHOLD_RELAY_SSL", main.config["RelayAPI"]["UseSSL"]) in trues # Web API -api_enabled = int(getenv("THRESHOLD_API_ENABLED")) or main.config["API"]["Enabled"] -api_address = getenv("THRESHOLD_API_HOST") or main.config["API"]["Address"] -api_port = int(getenv("THRESHOLD_API_PORT")) or main.config["API"]["Port"] +api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in trues +api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"]) +api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"])) +print("API PORT", api_port) +print("ENV PORT", getenv("THRESHOLD_API_PORT")) if __name__ == "__main__": listener = ServerFactory() From 9715b28f4732465bc2be0a0d9385ba28b2ae9b76 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 19:50:48 +0100 Subject: [PATCH 264/394] Move env file to example --- .env => .env.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .env => .env.example (100%) diff --git a/.env b/.env.example similarity index 100% rename from .env rename to .env.example From 3d67578179b449ae6786c6880e808aeaaee1c57d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 19:57:26 +0100 Subject: [PATCH 265/394] Add stack.env file --- .env.example | 3 --- .gitignore | 1 + docker-compose.yml | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 .env.example diff --git a/.env.example b/.env.example deleted file mode 100644 index f46978d..0000000 --- a/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -THRESHOLD_LISTENER_PORT=13867 -THRESHOLD_RELAY_PORT=13868 -THRESHOLD_API_PORT=13869 diff --git a/.gitignore b/.gitignore index 2a4c599..3c36401 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ conf/dist.sh conf/blacklist.json env/ .idea/ +.env diff --git a/docker-compose.yml b/docker-compose.yml index 9cb401e..7bce405 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" env_file: - - .env + - stack.env # for development extra_hosts: - "host.docker.internal:host-gateway" From 2f74d79bc49cbe948bf8882909d5ef904e608bea Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 21:11:01 +0100 Subject: [PATCH 266/394] Seamlessly handle nonexistent configurations --- main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 42950b9..2854525 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ import json import pickle from os import urandom +from os.path import exists from string import digits from redis import StrictRedis @@ -9,6 +10,7 @@ from redis import StrictRedis ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] configPath = "conf/" +exampleConfigPath = "conf/example" certPath = "cert/" filemap = { @@ -80,8 +82,16 @@ def saveConf(var): def loadConf(var): if filemap[var][2] == "json": - with open(configPath + filemap[var][0], "r") as f: + filename = configPath + filemap[var][0] + if not exists(filename): + if var == "config": + filename = exampleConfigPath + filemap[var][0] + else: + globals()[var] = {} + return + with open(filename, "r") as f: globals()[var] = json.load(f) + return if var == "alias": # This is a workaround to convert all the keys into integers since JSON # turns them into strings... From 3818308b75d708de1e678936257c29fb16732e44 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 21:27:26 +0100 Subject: [PATCH 267/394] Add Portainer Git directory to env file --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7bce405..1982dee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: pathogen/threshold build: . volumes: - - .:/code + - ${PORTAINER_GIT_DIR}:/code ports: - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" From ef611456718134bd87a60a4e55e678c371c0c584 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 21:29:08 +0100 Subject: [PATCH 268/394] Add trailing slash to example directory --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 2854525..972a897 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ from redis import StrictRedis ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] configPath = "conf/" -exampleConfigPath = "conf/example" +exampleConfigPath = "conf/example/" certPath = "cert/" filemap = { From a9f499ec67e8cb8225ec414a72b958f8fa492b3c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 28 Jul 2022 21:30:23 +0100 Subject: [PATCH 269/394] Remove print statements --- threshold | 2 -- 1 file changed, 2 deletions(-) diff --git a/threshold b/threshold index edf71da..32c4a39 100755 --- a/threshold +++ b/threshold @@ -56,8 +56,6 @@ relay_ssl = getenv("THRESHOLD_RELAY_SSL", main.config["RelayAPI"]["UseSSL"]) in api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in trues api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"]) api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"])) -print("API PORT", api_port) -print("ENV PORT", getenv("THRESHOLD_API_PORT")) if __name__ == "__main__": listener = ServerFactory() From 479e5072d2600d58bf03934e964c30ed3493ef80 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:01:48 +0100 Subject: [PATCH 270/394] Create separate production configuration --- .env.example | 13 +++++++++++ docker-compose.yml | 6 ++++-- Dockerfile => docker/Dockerfile | 4 ++-- docker/docker-compose.prod.yml | 38 +++++++++++++++++++++++++++++++++ docker/requirements.prod.txt | 9 ++++++++ requirements.txt | 1 + 6 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 .env.example rename Dockerfile => docker/Dockerfile (77%) create mode 100644 docker/docker-compose.prod.yml create mode 100644 docker/requirements.prod.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cdc5bd8 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +THRESHOLD_LISTENER_HOST=0.0.0.0 +THRESHOLD_LISTENER_PORT=13867 +THRESHOLD_LISTENER_SSL=1 + +THRESHOLD_RELAY_ENABLED=1 +THRESHOLD_RELAY_HOST=0.0.0.0 +THRESHOLD_RELAY_PORT=13868 +THRESHOLD_RELAY_SSL=1 + +THRESHOLD_API_ENABLED=1 +THRESHOLD_API_HOST=0.0.0.0 +THRESHOLD_API_PORT=13869 +PORTAINER_GIT_DIR=. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1982dee..b7df7de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: "2" services: app: image: pathogen/threshold - build: . + build: ./docker volumes: - ${PORTAINER_GIT_DIR}:/code ports: @@ -11,10 +11,12 @@ services: - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" env_file: - - stack.env + - .env # for development extra_hosts: - "host.docker.internal:host-gateway" + volumes_from: + - tmp tmp: image: busybox diff --git a/Dockerfile b/docker/Dockerfile similarity index 77% rename from Dockerfile rename to docker/Dockerfile index 6c63843..f89ac37 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,7 @@ USER pathogen ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 WORKDIR /code -COPY requirements.txt /code/ +COPY requirements.prod.txt /code/ RUN python -m venv /venv -RUN . /venv/bin/activate && pip install -r requirements.txt +RUN . /venv/bin/activate && pip install -r requirements.prod.txt CMD . /venv/bin/activate && exec python /code/threshold \ No newline at end of file diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 0000000..b7df7de --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,38 @@ +version: "2" + +services: + app: + image: pathogen/threshold + build: ./docker + volumes: + - ${PORTAINER_GIT_DIR}:/code + ports: + - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" + - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" + - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" + env_file: + - .env + # for development + extra_hosts: + - "host.docker.internal:host-gateway" + volumes_from: + - tmp + + tmp: + image: busybox + command: chmod -R 777 /var/run/redis + volumes: + - /var/run/redis + + redis: + image: redis + command: redis-server /etc/redis.conf + volumes: + - ./docker/redis.conf:/etc/redis.conf + volumes_from: + - tmp + +networks: + default: + external: + name: pathogen \ No newline at end of file diff --git a/docker/requirements.prod.txt b/docker/requirements.prod.txt new file mode 100644 index 0000000..5f94628 --- /dev/null +++ b/docker/requirements.prod.txt @@ -0,0 +1,9 @@ +wheel +twisted +pyOpenSSL +redis +pyYaML +python-logstash +service_identity +csiphash +Klein diff --git a/requirements.txt b/requirements.txt index 375a8eb..f6a2dcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +wheel pre-commit twisted pyOpenSSL From 248273648daed6c9b7678576932476890df4d86f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:02:10 +0100 Subject: [PATCH 271/394] Properly configure production compose file --- docker/docker-compose.prod.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index b7df7de..671fae5 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -11,10 +11,7 @@ services: - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" env_file: - - .env - # for development - extra_hosts: - - "host.docker.internal:host-gateway" + - stack.env volumes_from: - tmp From 6e9960570104fc5d2fd99c2683e3eda073b09e3b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:11:37 +0100 Subject: [PATCH 272/394] Fix environment variable path on production compose --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 671fae5..c4f91c1 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -11,7 +11,7 @@ services: - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" - "${THRESHOLD_API_PORT}:${THRESHOLD_API_PORT}" env_file: - - stack.env + - ../stack.env volumes_from: - tmp From cd38aab3186a0baa8096a3f7ccc499c427f77412 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:31:01 +0100 Subject: [PATCH 273/394] Pass through configuration directories to compose --- .gitignore | 11 ++--------- conf/example/alias.json | 1 - conf/example/blacklist.json | 1 - conf/example/counters.json | 1 - conf/example/tokens.json | 1 - conf/{ => templates}/aliasdata.json | 0 conf/{example => templates}/config.json | 0 conf/{ => templates}/help.json | 0 conf/{example => templates}/irc.json | 0 docker-compose.yml | 3 +++ docker/docker-compose.prod.yml | 3 +++ main.py | 18 +++++++++++++----- 12 files changed, 21 insertions(+), 18 deletions(-) delete mode 100644 conf/example/alias.json delete mode 100644 conf/example/blacklist.json delete mode 100644 conf/example/counters.json delete mode 100644 conf/example/tokens.json rename conf/{ => templates}/aliasdata.json (100%) rename conf/{example => templates}/config.json (100%) rename conf/{ => templates}/help.json (100%) rename conf/{example => templates}/irc.json (100%) diff --git a/.gitignore b/.gitignore index 3c36401..67403c5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,15 +2,8 @@ *.pem *.swp __pycache__/ -conf/config.json -conf/wholist.json -conf/counters.json -conf/tokens.json -conf/network.dat -conf/alias.json -conf/irc.json -conf/dist.sh -conf/blacklist.json +conf/live/ +conf/cert/ env/ .idea/ .env diff --git a/conf/example/alias.json b/conf/example/alias.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/alias.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/example/blacklist.json b/conf/example/blacklist.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/blacklist.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/example/counters.json b/conf/example/counters.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/counters.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/example/tokens.json b/conf/example/tokens.json deleted file mode 100644 index 0967ef4..0000000 --- a/conf/example/tokens.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/conf/aliasdata.json b/conf/templates/aliasdata.json similarity index 100% rename from conf/aliasdata.json rename to conf/templates/aliasdata.json diff --git a/conf/example/config.json b/conf/templates/config.json similarity index 100% rename from conf/example/config.json rename to conf/templates/config.json diff --git a/conf/help.json b/conf/templates/help.json similarity index 100% rename from conf/help.json rename to conf/templates/help.json diff --git a/conf/example/irc.json b/conf/templates/irc.json similarity index 100% rename from conf/example/irc.json rename to conf/templates/irc.json diff --git a/docker-compose.yml b/docker-compose.yml index b7df7de..fb036b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,9 @@ services: build: ./docker volumes: - ${PORTAINER_GIT_DIR}:/code + - ${THRESHOLD_CONFIG_DIR}:/code/conf/live + - ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates + - ${THRESHOLD_CERT_DIR}:/code/conf/cert ports: - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index c4f91c1..7ac7813 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -6,6 +6,9 @@ services: build: ./docker volumes: - ${PORTAINER_GIT_DIR}:/code + - ${THRESHOLD_CONFIG_DIR}:/code/conf/live + - ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates + - ${THRESHOLD_CERT_DIR}:/code/conf/cert ports: - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" diff --git a/main.py b/main.py index 972a897..4e16522 100644 --- a/main.py +++ b/main.py @@ -5,13 +5,14 @@ from os.path import exists from string import digits from redis import StrictRedis +from os import getenv # List of errors ZNC can give us ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] -configPath = "conf/" -exampleConfigPath = "conf/example/" -certPath = "cert/" +configPath = getenv("THRESHOLD_CONFIG_DIR", "conf/live/") +templateConfigPath = getenv("THRESHOLD_TEMPLATE_DIR", "conf/templates/") +certPath = getenv("THRESHOLD_CERT_DIR", "conf/cert/") filemap = { # JSON configs @@ -70,6 +71,8 @@ def liveNets(): def saveConf(var): + if var in ("help", "aliasdata"): + return # no need to save this if filemap[var][2] == "json": with open(configPath + filemap[var][0], "w") as f: json.dump(globals()[var], f, indent=4) @@ -83,21 +86,26 @@ def saveConf(var): def loadConf(var): if filemap[var][2] == "json": filename = configPath + filemap[var][0] + # Only take the help from the templates + if var in ("help", "aliasdata"): + filename = templateConfigPath + filemap[var][0] if not exists(filename): + # Load the template config if var == "config": - filename = exampleConfigPath + filemap[var][0] + filename = templateConfigPath + filemap[var][0] else: + # Everything else should be blank globals()[var] = {} return with open(filename, "r") as f: globals()[var] = json.load(f) - return if var == "alias": # This is a workaround to convert all the keys into integers since JSON # turns them into strings... # Dammit Jason! global alias alias = {int(x): y for x, y in alias.items()} + elif filemap[var][2] == "pickle": try: with open(configPath + filemap[var][0], "rb") as f: From e662d365420a9e53234d4e23e88740ea2a96e29b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:32:39 +0100 Subject: [PATCH 274/394] Fix path issue --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 4e16522..99b9dcc 100644 --- a/main.py +++ b/main.py @@ -10,9 +10,9 @@ from os import getenv # List of errors ZNC can give us ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] -configPath = getenv("THRESHOLD_CONFIG_DIR", "conf/live/") -templateConfigPath = getenv("THRESHOLD_TEMPLATE_DIR", "conf/templates/") -certPath = getenv("THRESHOLD_CERT_DIR", "conf/cert/") +configPath = "conf/live/" +templateConfigPath = "conf/templates/" +certPath = "conf/cert/" filemap = { # JSON configs From e1fc59f636660063dc9faa218a854c237ba67354 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:35:56 +0100 Subject: [PATCH 275/394] Don't pass template directory --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 7ac7813..4dfa1ad 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -7,7 +7,7 @@ services: volumes: - ${PORTAINER_GIT_DIR}:/code - ${THRESHOLD_CONFIG_DIR}:/code/conf/live - - ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates + #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates - ${THRESHOLD_CERT_DIR}:/code/conf/cert ports: - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" From bf79c013d5c2fe5695c0ff39edb96ae731bcc423 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:48:30 +0100 Subject: [PATCH 276/394] Fix redis.conf location in prod compose --- docker-compose.yml | 2 +- docker/docker-compose.prod.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fb036b6..bb538c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: volumes: - ${PORTAINER_GIT_DIR}:/code - ${THRESHOLD_CONFIG_DIR}:/code/conf/live - - ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates + #- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates - ${THRESHOLD_CERT_DIR}:/code/conf/cert ports: - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 4dfa1ad..f003ef1 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -28,7 +28,7 @@ services: image: redis command: redis-server /etc/redis.conf volumes: - - ./docker/redis.conf:/etc/redis.conf + - ./redis.conf:/etc/redis.conf volumes_from: - tmp From 9de0b0919de59b36d2e95bf6441bb57c081ae292 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 08:59:02 +0100 Subject: [PATCH 277/394] Use relative paths --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index f003ef1..85084ca 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -28,7 +28,7 @@ services: image: redis command: redis-server /etc/redis.conf volumes: - - ./redis.conf:/etc/redis.conf + - redis.conf:/etc/redis.conf volumes_from: - tmp From 1d8bb73645ed6ead5c942a8a56ff785dde561c2e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 09:00:08 +0100 Subject: [PATCH 278/394] Switch paths --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 85084ca..f003ef1 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -28,7 +28,7 @@ services: image: redis command: redis-server /etc/redis.conf volumes: - - redis.conf:/etc/redis.conf + - ./redis.conf:/etc/redis.conf volumes_from: - tmp From dc6dcd79dbca72bfc601a89cc4e975e9702b38d1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 09:04:18 +0100 Subject: [PATCH 279/394] Use paths relative to root in production compose --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index f003ef1..727f4a1 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -28,7 +28,7 @@ services: image: redis command: redis-server /etc/redis.conf volumes: - - ./redis.conf:/etc/redis.conf + - ${PWD}/docker/redis.conf:/etc/redis.conf volumes_from: - tmp From f88551f926947fe7c5c1984bcdfc372638da41e6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 09:06:13 +0100 Subject: [PATCH 280/394] Use Git dir to make redis config absolute path --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 727f4a1..6542927 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -28,7 +28,7 @@ services: image: redis command: redis-server /etc/redis.conf volumes: - - ${PWD}/docker/redis.conf:/etc/redis.conf + - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf volumes_from: - tmp From deb89e9202d3e61771820788b45b4caac3483d2d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 22:22:22 +0100 Subject: [PATCH 281/394] Use proper port for SSL listener --- .gitignore | 1 + docker/docker-compose.prod.yml | 2 +- threshold | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 67403c5..2db9b87 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ __pycache__/ conf/live/ conf/cert/ env/ +venv/ .idea/ .env diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 6542927..e8e011d 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -2,7 +2,7 @@ version: "2" services: app: - image: pathogen/threshold + image: pathogen/threshold:latest build: ./docker volumes: - ${PORTAINER_GIT_DIR}:/code diff --git a/threshold b/threshold index 32c4a39..b18f93d 100755 --- a/threshold +++ b/threshold @@ -57,11 +57,13 @@ api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"]) api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"])) +print("KEY", main.certPath + main.config["Key"]) +print("CERT", main.certPath + main.config["Certificate"]) if __name__ == "__main__": listener = ServerFactory() if listener_ssl is True: reactor.listenSSL( - main.config["Listener"]["Port"], + listener_port, listener, DefaultOpenSSLContextFactory( main.certPath + main.config["Key"], From 78f3f4520d5e60650e86e3edb09dd27a82486337 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 22:22:22 +0100 Subject: [PATCH 282/394] Fix Redis config path --- .gitignore | 1 + docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2db9b87..0396584 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ env/ venv/ .idea/ .env +.bash_history diff --git a/docker-compose.yml b/docker-compose.yml index bb538c0..28d0c60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "2" services: app: - image: pathogen/threshold + image: pathogen/threshold:latest build: ./docker volumes: - ${PORTAINER_GIT_DIR}:/code @@ -31,7 +31,7 @@ services: image: redis command: redis-server /etc/redis.conf volumes: - - ./docker/redis.conf:/etc/redis.conf + - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf volumes_from: - tmp From ba1f8407d1c8f85a93ab965b46b1d4abfe7bb2d0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 17:27:40 +0100 Subject: [PATCH 283/394] Implement creating relays via the API --- api/views.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 94e046c..8934d1a 100644 --- a/api/views.py +++ b/api/views.py @@ -294,6 +294,21 @@ class API(object): main.saveConf("network") return dumps({"success": True}) + @app.route("/irc/network///", methods=["PUT"]) + @login_required + def irc_network_relay_add(self, request, net, num): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + net_inst = main.network[net] + if num in net_inst.relays: + return dumps({"success": False, "reason": "network already has this relay."}) + id, alias = net_inst.add_relay() + main.saveConf("network") + return dumps({"success": True, "id": id, "alias": alias}) + @app.route("/irc/network//channels/", methods=["POST"]) @login_required def irc_network_channels(self, request, net): @@ -328,7 +343,7 @@ class API(object): return dumps({"success": False, "reason": "no such net."}) joined = chankeep.joinSingle(net, channel) if not joined: - dumps({"success": False, "reason": "no channels joined."}) + return dumps({"success": False, "reason": "could not allocate channel to relay."}) return dumps({"success": True, "relays": joined}) @app.route("/aliases/", methods=["GET"]) From 6359918639a662f75ea006c869b7c9f9fdfe00e7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 17:28:09 +0100 Subject: [PATCH 284/394] Fix joining channels with inactive relays --- modules/chankeep.py | 25 ++++++++++++------------- modules/network.py | 2 ++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 59c0e0d..dfd334d 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -38,6 +38,10 @@ def getChanFree(net, new): if i in new: continue name = net + str(i) + if name not in main.IRCPool.keys(): + continue + if not main.IRCPool[name].isconnected: + continue chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels) chanlimits.add(main.IRCPool[name].chanlimit) if not len(chanlimits) == 1: @@ -131,20 +135,15 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): def joinSingle(net, channel): - if allRelaysActive(net): - # Use the algorithm to allocate our channel to a relay - eca = emptyChanAllocate(net, [channel], None, []) - if not len(eca.keys()) == 1: - return False - num = list(eca.keys())[0] - name = f"{net}{num}" - if name not in main.IRCPool: - return False - main.IRCPool[name].join(channel) - return num - else: - error("All relays for %s are not active" % net) + eca = emptyChanAllocate(net, [channel], None, []) + if not len(eca.keys()) == 1: return False + num = list(eca.keys())[0] + name = f"{net}{num}" + if name not in main.IRCPool: + return False + main.IRCPool[name].join(channel) + return num def partSingle(net, channel): diff --git a/modules/network.py b/modules/network.py index ccfbb32..7aea0b1 100644 --- a/modules/network.py +++ b/modules/network.py @@ -24,6 +24,8 @@ class Network: self.aliases = {} def add_relay(self, num=None): + # Grrrrrrrrrr + self.last = int(self.last) if not num: num = self.last self.last += 1 From d51e87b09f55bdc2d84deae8872eb5f241d1faa9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 17:28:19 +0100 Subject: [PATCH 285/394] Reformat code --- main.py | 3 +-- threshold | 19 +++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index 99b9dcc..0c7e24c 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,6 @@ from os.path import exists from string import digits from redis import StrictRedis -from os import getenv # List of errors ZNC can give us ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] @@ -72,7 +71,7 @@ def liveNets(): def saveConf(var): if var in ("help", "aliasdata"): - return # no need to save this + return # no need to save this if filemap[var][2] == "json": with open(configPath + filemap[var][0], "w") as f: json.dump(globals()[var], f, indent=4) diff --git a/threshold b/threshold index b18f93d..8f0b7e6 100755 --- a/threshold +++ b/threshold @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys from codecs import getwriter +from os import getenv from signal import SIGINT, signal from sys import stderr, stdout @@ -19,7 +20,6 @@ from core.server import ServerFactory from utils.cleanup import handler from utils.loaders.command_loader import loadCommands from utils.logging.log import log -from os import getenv Factory.noisy = False @@ -39,7 +39,7 @@ stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using # unicode quit messages for no reason -trues = ('true', '1', 't', True) +trues = ("true", "1", "t", True) # Main listener listener_address = getenv("THRESHOLD_LISTENER_HOST", main.config["Listener"]["Address"]) @@ -71,10 +71,7 @@ if __name__ == "__main__": ), interface=listener_address, ) - log( - "Threshold running with SSL on %s:%s" - % (listener_address, listener_port) - ) + log("Threshold running with SSL on %s:%s" % (listener_address, listener_port)) else: reactor.listenTCP( listener_port, @@ -94,20 +91,14 @@ if __name__ == "__main__": ), interface=relay_address, ) - log( - "Threshold relay running with SSL on %s:%s" - % (relay_address, relay_port) - ) + log("Threshold relay running with SSL on %s:%s" % (relay_address, relay_port)) else: reactor.listenTCP( relay_port, relay, interface=relay_address, ) - log( - "Threshold relay running on %s:%s" - % (relay_address, relay_port) - ) + log("Threshold relay running on %s:%s" % (relay_address, relay_port)) for net in main.network.keys(): main.network[net].start_bots() modules.counters.setupCounterLoop() From 54b5561a75f164e301c08843525a4b676e2304da Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 22:11:43 +0100 Subject: [PATCH 286/394] Implement deleeting relays and fix adding --- api/views.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 8934d1a..48e34f9 100644 --- a/api/views.py +++ b/api/views.py @@ -305,10 +305,25 @@ class API(object): net_inst = main.network[net] if num in net_inst.relays: return dumps({"success": False, "reason": "network already has this relay."}) - id, alias = net_inst.add_relay() + id, alias = net_inst.add_relay(num) main.saveConf("network") return dumps({"success": True, "id": id, "alias": alias}) + @app.route("/irc/network///", methods=["DELETE"]) + @login_required + def irc_network_relay_del(self, request, net, num): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + net_inst = main.network[net] + if num not in net_inst.relays: + return dumps({"success": False, "reason": "network does not have this relay."}) + net_inst.delete_relay(num) + main.saveConf("network") + return dumps({"success": True}) + @app.route("/irc/network//channels/", methods=["POST"]) @login_required def irc_network_channels(self, request, net): From 6909fb68f7e8f22055bd3f84240ae311a7857d09 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 29 Jul 2022 22:39:08 +0100 Subject: [PATCH 287/394] Implement API endpoint to add next relay --- api/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/views.py b/api/views.py index 48e34f9..754fd07 100644 --- a/api/views.py +++ b/api/views.py @@ -309,6 +309,16 @@ class API(object): main.saveConf("network") return dumps({"success": True, "id": id, "alias": alias}) + @app.route("/irc/network//", methods=["PUT"]) + @login_required + def irc_network_relay_add_next(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + net_inst = main.network[net] + id, alias = net_inst.add_relay() + main.saveConf("network") + return dumps({"success": True, "id": id, "alias": alias}) + @app.route("/irc/network///", methods=["DELETE"]) @login_required def irc_network_relay_del(self, request, net, num): From 0dedb545f0d7767d22bb2a028b0c48302ee8b07d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 1 Aug 2022 19:05:12 +0100 Subject: [PATCH 288/394] Implement updating aliases --- api/views.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/api/views.py b/api/views.py index 754fd07..ba45d7a 100644 --- a/api/views.py +++ b/api/views.py @@ -381,3 +381,25 @@ class API(object): alias_list.append(alias_dup) return dumps({"aliases": alias_list}) + + @app.route("/aliases/", methods=["POST"]) + @login_required + def aliases_update(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + for alias, fields in data.items(): + if not alias.isdigit(): + return dumps({"success": False, "reason": "alias not a number."}) + alias = int(alias) + if alias not in main.alias.keys(): + return dumps({"success": False, "reason": "alias does not exist."}) + if fields: + main.alias[alias] = fields + if "emails" in fields: + if not fields["emails"]: + main.alias[alias]["emails"] = [] + main.saveConf("alias") + + return dumps({"success": True}) From e8870e95e7a153b423d153fd4920cf47d507939b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 1 Aug 2022 19:34:35 +0100 Subject: [PATCH 289/394] Implement automatic provisioning --- api/views.py | 20 ++++++++++++++++++-- modules/provision.py | 1 - 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index ba45d7a..5b6b5c2 100644 --- a/api/views.py +++ b/api/views.py @@ -5,7 +5,7 @@ from klein import Klein from twisted.web.server import Request import main -from modules import chankeep, userinfo +from modules import chankeep, provision, userinfo from utils.logging.log import warn @@ -396,10 +396,26 @@ class API(object): if alias not in main.alias.keys(): return dumps({"success": False, "reason": "alias does not exist."}) if fields: - main.alias[alias] = fields + for field in fields: + if field in main.alias[alias]: + main.alias[alias][field] = fields[field] if "emails" in fields: if not fields["emails"]: main.alias[alias]["emails"] = [] main.saveConf("alias") return dumps({"success": True}) + + @app.route("/irc/auto//", methods=["POST"]) + @login_required + def irc_auto_network(self, request, net): + print(repr(main.network[net].relays.keys())) + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if 1 in main.network[net].relays.keys(): + return dumps({"success": False, "reason": f"First relay exists on {net}"}) + num, alias = main.network[net].add_relay(1) + provision.provisionRelay(num, net) + main.saveConf("network") + + return dumps({"success": True, f"message": "Created relay {num} with alias {alias} on {net}"}) diff --git a/modules/provision.py b/modules/provision.py index e1e740e..6681a78 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -63,7 +63,6 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): def provisionRelay(num, network): # provision user and network data aliasObj = main.alias[num] - print("ALIASOBJ FALUES", aliasObj.values()) # alias = aliasObj["nick"] provisionUserNetworkData( num, From dae62ea544571ed3ff54e3993cca6c62ef7edd62 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 1 Aug 2022 21:31:48 +0100 Subject: [PATCH 290/394] Remove debugging code --- api/views.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index 5b6b5c2..34c1adf 100644 --- a/api/views.py +++ b/api/views.py @@ -409,13 +409,25 @@ class API(object): @app.route("/irc/auto//", methods=["POST"]) @login_required def irc_auto_network(self, request, net): - print(repr(main.network[net].relays.keys())) if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) if 1 in main.network[net].relays.keys(): - return dumps({"success": False, "reason": f"First relay exists on {net}"}) + return dumps({"success": False, "reason": f"first relay exists on {net}"}) num, alias = main.network[net].add_relay(1) provision.provisionRelay(num, net) main.saveConf("network") - return dumps({"success": True, f"message": "Created relay {num} with alias {alias} on {net}"}) + return dumps({"success": True, f"message": "created relay {num} with alias {alias} on {net}"}) + + @app.route("/irc/list//", methods=["POST"]) + @login_required + def irc_list_network(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if 1 not in main.network[net].relays.keys(): + return dumps({"success": False, "reason": f"no first relay on {net}"}) + num, alias = main.network[net].add_relay(1) + provision.provisionRelay(num, net) + main.saveConf("network") + + return dumps({"success": True, f"message": "created relay {num} with alias {alias} on {net}"}) \ No newline at end of file From db4b6cc6f99a4dbb77e1eab46691fbf152ecffc1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 1 Aug 2022 21:38:46 +0100 Subject: [PATCH 291/394] Implement requesting channel list for network --- api/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/views.py b/api/views.py index 34c1adf..b35463c 100644 --- a/api/views.py +++ b/api/views.py @@ -417,7 +417,7 @@ class API(object): provision.provisionRelay(num, net) main.saveConf("network") - return dumps({"success": True, f"message": "created relay {num} with alias {alias} on {net}"}) + return dumps({"success": True, "message": f"created relay {num} with alias {alias} on {net}"}) @app.route("/irc/list//", methods=["POST"]) @login_required @@ -426,8 +426,8 @@ class API(object): return dumps({"success": False, "reason": "no such net."}) if 1 not in main.network[net].relays.keys(): return dumps({"success": False, "reason": f"no first relay on {net}"}) - num, alias = main.network[net].add_relay(1) - provision.provisionRelay(num, net) - main.saveConf("network") - - return dumps({"success": True, f"message": "created relay {num} with alias {alias} on {net}"}) \ No newline at end of file + name = f"{net}1" + if name not in main.IRCPool.keys(): + return dumps({"success": False, "reason": f"first relay not active for {net}"}) + main.IRCPool[name].list() + return dumps({"success": True, "message": f"requested list with first instance of {net}"}) From 4c9ac3ec42bcda272e86fcd4367a8974e4c10f46 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 1 Aug 2022 23:02:20 +0100 Subject: [PATCH 292/394] Implement adding networks --- api/views.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/api/views.py b/api/views.py index b35463c..76abaa4 100644 --- a/api/views.py +++ b/api/views.py @@ -1,11 +1,13 @@ import functools from json import JSONDecodeError, dumps, loads +from string import digits from klein import Klein from twisted.web.server import Request import main from modules import chankeep, provision, userinfo +from modules.network import Network from utils.logging.log import warn @@ -248,6 +250,43 @@ class API(object): return dumps({"success": True}) + @app.route("/irc/network/create/", methods=["PUT"]) + @login_required + def irc_network_create(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + fields = ["net", "auth", "host", "port", "security"] + if not set(fields).issubset(set(data)): + return dumps({"success": False, "reason": "not enough fields."}) + for item in data: + if item == "net": + net = data[item] + if net in main.network.keys(): + return dumps({"success": False, "reason": "network already exists."}) + if set(net).intersection(set(digits)): + return dumps({"success": False, "reason": "network name cannot contain numbers."}) + elif item == "auth": + auth = data[item] + if auth not in ["sasl", "ns", "none"]: + return dumps({"success": False, "reason": "invalid auth."}) + elif item == "host": + host = data[item] + elif item == "port": + port = data[item] + if not port.isdigit(): + return dumps({"success": False, "reason": "invalid port: not a number."}) + port = int(port) + elif item == "security": + security = data[item] + if security not in ["ssl", "plain"]: + return dumps({"success": False, "reason": "invalid security."}) + main.network[net] = Network(net, host, int(port), security, auth) + main.saveConf("network") + + return dumps({"success": True}) + @app.route("/irc/network//relays/", methods=["POST"]) @login_required def irc_network_relays(self, request, net): From b42c82eac211ae5d0a371e59b4d2b539ab0db768 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 2 Aug 2022 09:01:24 +0100 Subject: [PATCH 293/394] More error handling when joining channels with ChanKeep --- modules/chankeep.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index dfd334d..a1d13cd 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -136,6 +136,8 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): def joinSingle(net, channel): eca = emptyChanAllocate(net, [channel], None, []) + if not eca: + return False if not len(eca.keys()) == 1: return False num = list(eca.keys())[0] From 4c8b584ef43466efca2b703db256684d1c4f8885 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 2 Aug 2022 09:01:34 +0100 Subject: [PATCH 294/394] Implement deleting networks --- api/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/views.py b/api/views.py index 76abaa4..22bd866 100644 --- a/api/views.py +++ b/api/views.py @@ -211,6 +211,16 @@ class API(object): network["records"] = userinfo.getNumWhoEntries(net) return dumps(network) + @app.route("/irc/network//", methods=["DELETE"]) + @login_required + def irc_network_delete(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + main.network[net].seppuku() + del main.network[net] + main.saveConf("network") + return dumps({"success": True}) + @app.route("/irc/network//edit/", methods=["POST"]) @login_required def irc_network_edit(self, request, net): From 502b45cda5ec49985b70fe5f34affedfcd15550f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 19:22:09 +0100 Subject: [PATCH 295/394] Allow gaps in relay numbering --- api/views.py | 14 ++++++-------- commands/list.py | 24 +++++++++++------------- core/bot.py | 17 +++++++++++------ modules/__init__.py | 0 modules/chankeep.py | 28 ++++++++++++++++++---------- modules/helpers.py | 35 +++++++++++++++++++++++++++++++++++ modules/provision.py | 6 +++++- threshold | 2 -- 8 files changed, 86 insertions(+), 40 deletions(-) create mode 100644 modules/__init__.py create mode 100644 modules/helpers.py diff --git a/api/views.py b/api/views.py index 22bd866..d899e5b 100644 --- a/api/views.py +++ b/api/views.py @@ -6,7 +6,7 @@ from klein import Klein from twisted.web.server import Request import main -from modules import chankeep, provision, userinfo +from modules import chankeep, helpers, provision, userinfo from modules.network import Network from utils.logging.log import warn @@ -473,10 +473,8 @@ class API(object): def irc_list_network(self, request, net): if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) - if 1 not in main.network[net].relays.keys(): - return dumps({"success": False, "reason": f"no first relay on {net}"}) - name = f"{net}1" - if name not in main.IRCPool.keys(): - return dumps({"success": False, "reason": f"first relay not active for {net}"}) - main.IRCPool[name].list() - return dumps({"success": True, "message": f"requested list with first instance of {net}"}) + first_relay = helpers.get_first_relay(net) + if not first_relay: + return dumps({"success": False, "reason": f"could not get first relay for {net}"}) + first_relay.list() + return dumps({"success": True, "message": f"requested list with instance {first_relay.num} of {net}"}) diff --git a/commands/list.py b/commands/list.py index e72cc2e..9dccb8b 100644 --- a/commands/list.py +++ b/commands/list.py @@ -1,4 +1,5 @@ import main +from modules import helpers class ListCommand: @@ -9,27 +10,24 @@ class ListCommand: if authed: if length == 1: for i in main.network.keys(): - if 1 not in main.network[i].relays.keys(): + first_relay = helpers.get_first_relay(i) + #### + if not first_relay: info("Network has no first instance: %s" % i) continue - if not i + "1" in main.IRCPool.keys(): - info("No IRC instance: %s - 1" % i) - continue - main.IRCPool[i + "1"].list() - success("Requested list with first instance of %s" % i) + first_relay.list() + success(f"Requested list with instance {first_relay.num} of {i}") return elif length == 2: if not spl[1] in main.network.keys(): failure("No such network: %s" % spl[1]) return - if 1 not in main.network[spl[1]].relays.keys(): - failure("Network has no first instance") + first_relay = helpers.get_first_relay(spl[1]) + if not first_relay: + failure("Could not get first instance") return - if spl[1] + "1" not in main.IRCPool.keys(): - failure("No IRC instance: %s - 1" % spl[1]) - return - main.IRCPool[spl[1] + "1"].list() - success("Requested list with first instance of %s" % spl[1]) + first_relay.list() + success(f"Requested list with instance {first_relay.num} of {spl[1]}") return else: incUsage("list") diff --git a/core/bot.py b/core/bot.py index 7c86beb..b8f664f 100644 --- a/core/bot.py +++ b/core/bot.py @@ -16,7 +16,7 @@ from twisted.words.protocols.irc import ( import main from core.relay import sendRelayNotification -from modules import chankeep, counters, monitor, regproc, userinfo +from modules import chankeep, counters, helpers, monitor, regproc, userinfo from utils.dedup import dedup from utils.logging.debug import debug from utils.logging.log import error, log, warn @@ -458,11 +458,16 @@ class IRCBot(IRCClient): def recheckList(self): allRelays = chankeep.allRelaysActive(self.net) if allRelays: - name = self.net + "1" - if main.IRCPool[name].wantList is True: - main.IRCPool[name].list(nocheck=True) - debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) - if self.num == 1: # Only one instance should do a list + debug(f"All relays active for {self.net}") + first_relay = helpers.get_first_relay(self.net) + if first_relay: + if first_relay.wantList is True: + first_relay.list(nocheck=True) + debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) + # name = self.net + "1" + + # if self.num == 1: # Only one instance should do a list + if helpers.is_first_relay(self.net, self.num): if self.chanlimit: if allRelays: self.list() diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/chankeep.py b/modules/chankeep.py index a1d13cd..b0bb39d 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -9,10 +9,19 @@ from utils.logging.debug import debug from utils.logging.log import error, log, warn +def getActiveRelays(net): + activeRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] + return activeRelays + + def allRelaysActive(net): - relayNum = len(main.network[net].relays.keys()) + """ + Check if all enabled relays are active and authenticated. + """ + activeRelays = getActiveRelays(net) + relayNum = len(activeRelays) + 1 existNum = 0 - for i in main.network[net].relays.keys(): + for i in activeRelays: name = net + str(i) if name in main.IRCPool.keys(): if main.IRCPool[name].authenticated: @@ -34,7 +43,7 @@ def getChanFree(net, new): """ chanfree = {} chanlimits = set() - for i in main.network[net].relays.keys(): + for i in getActiveRelays(net): if i in new: continue name = net + str(i) @@ -60,9 +69,8 @@ def emptyChanAllocate(net, flist, relay, new): toalloc = len(flist) if toalloc > sum(chanfree[0].values()): correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) - # print("correction", correction) warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) - # newNums = modules.provision.provisionMultipleRelays(net, correction) + modules.provision.provisionMultipleRelays(net, correction) return False for i in chanfree[0].keys(): for x in range(chanfree[0][i]): @@ -88,7 +96,7 @@ def populateChans(net, clist, relay, new): def notifyJoin(net): - for i in main.network[net].relays.keys(): + for i in getActiveRelays(net): name = net + str(i) if name in main.IRCPool.keys(): main.IRCPool[name].checkChannels() @@ -98,7 +106,7 @@ def minifyChans(net, listinfo): if not allRelaysActive(net): error("All relays for %s are not active, cannot minify list" % net) return False - for i in main.network[net].relays.keys(): + for i in getActiveRelays(net): name = net + str(i) for x in main.IRCPool[name].channels: for y in listinfo: @@ -122,12 +130,12 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) return if coverAll: - needed = relay - len(main.network[net].relays.keys()) + needed = relay - len(getActiveRelays(net)) newNums = modules.provision.provisionMultipleRelays(net, needed) flist = [i[0] for i in listinfo] populateChans(net, flist, relay, newNums) else: - needed = sigrelay - len(main.network[net].relays.keys()) + needed = sigrelay - len(getActiveRelays(net)) newNums = modules.provision.provisionMultipleRelays(net, needed) siglist = [i[0] for i in listinfo if int(i[1]) > mean] populateChans(net, siglist, sigrelay, newNums) @@ -156,7 +164,7 @@ def partSingle(net, channel): :return: """ parted = [] - for i in main.network[net].relays.keys(): + for i in getActiveRelays(net): name = f"{net}{i}" if name in main.IRCPool.keys(): if channel in main.IRCPool[name].channels: diff --git a/modules/helpers.py b/modules/helpers.py new file mode 100644 index 0000000..58ad4a3 --- /dev/null +++ b/modules/helpers.py @@ -0,0 +1,35 @@ +import main +from modules import chankeep + +def get_first_relay(net): + """ + Get the first relay in the network. + :param net: the network + :param num: number or relay + :return: IRCPool instance for the IRC bot + """ + cur_relay = 0 + max_relay = len(main.network[net].relays.keys())+1 + activeRelays = chankeep.getActiveRelays(net) + while cur_relay != max_relay: + cur_relay += 1 + if cur_relay not in activeRelays: + continue + name = net + str(cur_relay) + if name in main.IRCPool.keys(): + return main.IRCPool[name] + return None + +def is_first_relay(net, num): + """ + Determine if we are the first relay for the network. + :param net: the network + :param num: number or relay + :return: True if we are the first relay, False otherwise + """ + cur_relay = 0 + max_relay = len(main.network[net].relays.keys()) + while cur_relay > max_relay: + name = net + str(cur_relay) + if name in main.IRCPool.keys(): + return cur_relay == num \ No newline at end of file diff --git a/modules/provision.py b/modules/provision.py index 6681a78..b146699 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -3,6 +3,7 @@ from twisted.internet import reactor import main import modules.regproc from utils.deliver_relay_commands import deliverRelayCommands +from utils.logging.log import warn def provisionUserNetworkData( @@ -72,13 +73,16 @@ def provisionRelay(num, network): # provision user and network data main.network[network].port, main.network[network].security, main.network[network].auth, - main.network[network].aliases[num]["password"] + main.network[network].aliases[num]["password"], ) if main.config["ConnectOnCreate"]: reactor.callLater(10, main.network[network].start_bot, num) def provisionMultipleRelays(net, relaysNeeded): + if not main.config["ChanKeep"]["Provision"]: + warn(f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled") + return 0 numsProvisioned = [] for i in range(relaysNeeded): num, alias = main.network[net].add_relay() diff --git a/threshold b/threshold index 8f0b7e6..5706025 100755 --- a/threshold +++ b/threshold @@ -57,8 +57,6 @@ api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"]) api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"])) -print("KEY", main.certPath + main.config["Key"]) -print("CERT", main.certPath + main.config["Certificate"]) if __name__ == "__main__": listener = ServerFactory() if listener_ssl is True: From b16289cdedeb2d74f5b905327186b8cecd3dec1b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 19:49:58 +0100 Subject: [PATCH 296/394] Update IRC template --- conf/templates/irc.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/conf/templates/irc.json b/conf/templates/irc.json index 8270e2f..e37a788 100644 --- a/conf/templates/irc.json +++ b/conf/templates/irc.json @@ -2,7 +2,7 @@ "_": { "register": true, "entity": "NickServ", - "email": "{nickname}@domain.com", + "domains": [], "registermsg": "REGISTER {password} {email}", "confirm": "CONFIRM {token}", "check": true, @@ -11,7 +11,7 @@ "pingmsg": "STATUS", "negativemsg": "INFO {nickname}", "checktype": "mode", - "checkmode": "r", + "checkmode": "R", "checkmsg": "Password accepted - you are now recognized.", "checkmsg2": "You are logged in as", "checknegativemsg": "has \u0002NOT COMPLETED\u0002 registration verification", @@ -19,5 +19,15 @@ }, "freenode": { "confirm": "VERIFY REGISTER {nickname} {token}" + }, + "libera": { + "confirm": "VERIFY REGISTER {nickname} {token}" + }, + "cyberia": { + "setmode": "I" + }, + "ircnow": { + "negative": false, + "ping": false } } From 6193502f2e2d543417c94a4ac6fa00ee5416335d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:09:01 +0100 Subject: [PATCH 297/394] Enable debug mode with env vars --- threshold | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/threshold b/threshold index 5706025..c728709 100755 --- a/threshold +++ b/threshold @@ -57,6 +57,15 @@ api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"]) api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"])) +# Debugging +debug_enabled = getenv("THRESHOLD_DEBUG", main.config["Debug"]) in trues +trace_enabled = getenv("THRESHOLD_TRACE", main.config["Trace"]) in trues +if debug_enabled: + main.config["Debug"] = True +if trace_enabled: + main.config["Trace"] = True + + if __name__ == "__main__": listener = ServerFactory() if listener_ssl is True: From 16d268ca9051b94d7b0a207fd5ffdbd302010ff3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:09:14 +0100 Subject: [PATCH 298/394] Reformat helpers --- modules/helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/helpers.py b/modules/helpers.py index 58ad4a3..63534ed 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -1,6 +1,7 @@ import main from modules import chankeep + def get_first_relay(net): """ Get the first relay in the network. @@ -9,7 +10,7 @@ def get_first_relay(net): :return: IRCPool instance for the IRC bot """ cur_relay = 0 - max_relay = len(main.network[net].relays.keys())+1 + max_relay = len(main.network[net].relays.keys()) + 1 activeRelays = chankeep.getActiveRelays(net) while cur_relay != max_relay: cur_relay += 1 @@ -20,6 +21,7 @@ def get_first_relay(net): return main.IRCPool[name] return None + def is_first_relay(net, num): """ Determine if we are the first relay for the network. @@ -32,4 +34,4 @@ def is_first_relay(net, num): while cur_relay > max_relay: name = net + str(cur_relay) if name in main.IRCPool.keys(): - return cur_relay == num \ No newline at end of file + return cur_relay == num From cc0e3b872bbf588149cc90e91cb92649983d65f9 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:12:38 +0100 Subject: [PATCH 299/394] Add extra debug call for allRelaysActive --- modules/chankeep.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index b0bb39d..6cc501c 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -19,6 +19,7 @@ def allRelaysActive(net): Check if all enabled relays are active and authenticated. """ activeRelays = getActiveRelays(net) + debug(f"allRelaysActive() active relays for {net}: {activeRelays}") relayNum = len(activeRelays) + 1 existNum = 0 for i in activeRelays: @@ -26,6 +27,7 @@ def allRelaysActive(net): if name in main.IRCPool.keys(): if main.IRCPool[name].authenticated: existNum += 1 + debug(f"allRelaysActive() finished, {existNum}/{relayNum} relays active for {net}") if existNum == relayNum: return True return False From 87ee96dd26894e88914393bb8d355e4395bbda5e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:13:30 +0100 Subject: [PATCH 300/394] Don't add 1 to current relays when iterating --- modules/chankeep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 6cc501c..ffb2eb7 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -20,7 +20,7 @@ def allRelaysActive(net): """ activeRelays = getActiveRelays(net) debug(f"allRelaysActive() active relays for {net}: {activeRelays}") - relayNum = len(activeRelays) + 1 + relayNum = len(activeRelays) existNum = 0 for i in activeRelays: name = net + str(i) From 604bee1b78bef44d7ef6995c6460d75e7d681d9f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:18:49 +0100 Subject: [PATCH 301/394] Add more LIST handling debugging --- core/bot.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/core/bot.py b/core/bot.py index b8f664f..c63e705 100644 --- a/core/bot.py +++ b/core/bot.py @@ -127,8 +127,10 @@ class IRCBot(IRCClient): error("%s - Cannot join channel we are already on: %s - %i" % (i, self.net, self.num)) def checkChannels(self): - if not chankeep.allRelaysActive(self.net): - debug("Skipping channel check as we have inactive relays: %s - %i" % (self.net, self.num)) + if chankeep.allRelaysActive(self.net): + debug(f"checkChannels() all relays active for {self.net}") + else: + debug("checkChannels() skipping channel check as we have inactive relays: %s - %i" % (self.net, self.num)) return if self.net in main.TempChan.keys(): if self.num in main.TempChan[self.net].keys(): @@ -457,24 +459,29 @@ class IRCBot(IRCClient): def recheckList(self): allRelays = chankeep.allRelaysActive(self.net) + debug(f"recheckList() all relays for {self.net} {allRelays}") if allRelays: - debug(f"All relays active for {self.net}") + debug(f"recheckList() all relays active for {self.net}") first_relay = helpers.get_first_relay(self.net) + debug(f"recheckList() first relay for {self.net}: {first_relay.num}") if first_relay: if first_relay.wantList is True: first_relay.list(nocheck=True) - debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) + debug("recheckList() asking for a list for {self.net} after final relay {self.num} connected") # name = self.net + "1" # if self.num == 1: # Only one instance should do a list if helpers.is_first_relay(self.net, self.num): + debug(f"recheckList() we are the first relay for {self.net} ({self.num})") if self.chanlimit: if allRelays: self.list() + debug(f"recheckList() requested a list for {self.net} from {self.num}") + else: self.wantList = True else: - debug("Aborting LIST due to bad chanlimit") + debug("recheckList() aborting LIST due to bad chanlimit") self.checkChannels() def seed_chanlimit(self, chanlimit): From b5326e92a1cc5d87a153d9bfe8877ef60d7ab613 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:21:39 +0100 Subject: [PATCH 302/394] Add even more debugging --- core/bot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index c63e705..579bab8 100644 --- a/core/bot.py +++ b/core/bot.py @@ -468,6 +468,8 @@ class IRCBot(IRCClient): if first_relay.wantList is True: first_relay.list(nocheck=True) debug("recheckList() asking for a list for {self.net} after final relay {self.num} connected") + else: + debug(f"recheckList() first relay wantList is False for {self.net} ({first_relay.num})") # name = self.net + "1" # if self.num == 1: # Only one instance should do a list @@ -477,8 +479,8 @@ class IRCBot(IRCClient): if allRelays: self.list() debug(f"recheckList() requested a list for {self.net} from {self.num}") - else: + debug(f"recheckList() not all relays active for {self.net}") self.wantList = True else: debug("recheckList() aborting LIST due to bad chanlimit") From 22e853a3f76a9823fbb55eb84944bd945b23544b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:26:19 +0100 Subject: [PATCH 303/394] Simplify is_first_relay --- modules/helpers.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/helpers.py b/modules/helpers.py index 63534ed..cd0c510 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -29,9 +29,7 @@ def is_first_relay(net, num): :param num: number or relay :return: True if we are the first relay, False otherwise """ - cur_relay = 0 - max_relay = len(main.network[net].relays.keys()) - while cur_relay > max_relay: - name = net + str(cur_relay) - if name in main.IRCPool.keys(): - return cur_relay == num + first_relay = get_first_relay(net) + if not first_relay: + return False + return first_relay.num == num From 7c9903bca2d07f3d7a788cfdd99604a66eaaffaa Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:29:01 +0100 Subject: [PATCH 304/394] Return correct data type for provisioning relays --- modules/provision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/provision.py b/modules/provision.py index b146699..01d371f 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -82,7 +82,7 @@ def provisionRelay(num, network): # provision user and network data def provisionMultipleRelays(net, relaysNeeded): if not main.config["ChanKeep"]["Provision"]: warn(f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled") - return 0 + return [] numsProvisioned = [] for i in range(relaysNeeded): num, alias = main.network[net].add_relay() From d38f7ba1bae486f0651c1d774cf5cc297e786607 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:32:49 +0100 Subject: [PATCH 305/394] Print information about received LIST --- modules/chankeep.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index ffb2eb7..740482f 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -225,6 +225,13 @@ def _initialList(net, num, listinfo, chanlimit): p.hset(abase, "relay", relay) p.hset(abase, "sigrelay", sigrelay) p.hset(abase, "insigrelay", ceil(insiglength / chanlimit)) + debug( + ( + f"_initialList() net:{net} num:{num} listlength:{listlength} " + f"mean:{mean} siglength:{siglength} insiglength:{insiglength} " + f"sigrelay:{sigrelay} relay:{relay} chanlimit:{chanlimit}" + ) + ) # Purge existing records before writing # purgeRecords(net) From dc13515aa87fc0f395f371525ec7cf419c869518 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:36:24 +0100 Subject: [PATCH 306/394] Adding more debug statements in ECA system --- modules/chankeep.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index 740482f..d46e7fd 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -71,6 +71,7 @@ def emptyChanAllocate(net, flist, relay, new): toalloc = len(flist) if toalloc > sum(chanfree[0].values()): correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) + debug(f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}") warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) modules.provision.provisionMultipleRelays(net, correction) return False @@ -133,11 +134,13 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): return if coverAll: needed = relay - len(getActiveRelays(net)) + debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") newNums = modules.provision.provisionMultipleRelays(net, needed) flist = [i[0] for i in listinfo] populateChans(net, flist, relay, newNums) else: needed = sigrelay - len(getActiveRelays(net)) + debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}") newNums = modules.provision.provisionMultipleRelays(net, needed) siglist = [i[0] for i in listinfo if int(i[1]) > mean] populateChans(net, siglist, sigrelay, newNums) From 8c3a75b3c85a8912e865a5973cde1ffe9662256b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:43:34 +0100 Subject: [PATCH 307/394] Expand ECA secondary allocation algorithm --- modules/chankeep.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index d46e7fd..d672010 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -69,8 +69,15 @@ def emptyChanAllocate(net, flist, relay, new): chanfree[0][i] = chanfree[1] allocated = {} toalloc = len(flist) + # toalloc:2148 free:{1: 250} chanlimit:250 correction:2147 if toalloc > sum(chanfree[0].values()): - correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) + sum_free = sum(chanfree[0].values()) # 250 + chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 + relays_needed = chans_not_covered / chanfree[1] # 1898 / 250 = 7.592 + relays_needed_rounded = round(relays_needed) + debug(f"emptyChanAllocate() secondary allocation sum_free:{sum_free} chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} relays_needed_rounded:{relays_needed_rounded}") + #correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) + correction = relays_needed_rounded debug(f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}") warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) modules.provision.provisionMultipleRelays(net, correction) From 5a4ae2153e1901f30f6157499b3780da6ef35c43 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:46:44 +0100 Subject: [PATCH 308/394] Use ceil instead of round for relay number rounding --- modules/chankeep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index d672010..3133a98 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -74,7 +74,7 @@ def emptyChanAllocate(net, flist, relay, new): sum_free = sum(chanfree[0].values()) # 250 chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 relays_needed = chans_not_covered / chanfree[1] # 1898 / 250 = 7.592 - relays_needed_rounded = round(relays_needed) + relays_needed_rounded = ceil(relays_needed) debug(f"emptyChanAllocate() secondary allocation sum_free:{sum_free} chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} relays_needed_rounded:{relays_needed_rounded}") #correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) correction = relays_needed_rounded From 4c040bbf782735fc550d3b4e67b3eeca037325d2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 20:51:41 +0100 Subject: [PATCH 309/394] Simplify variable names and reformat --- modules/chankeep.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 3133a98..7f7ce14 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -69,16 +69,27 @@ def emptyChanAllocate(net, flist, relay, new): chanfree[0][i] = chanfree[1] allocated = {} toalloc = len(flist) + # Used to correct allocations and provision additional relays + # if the math since the last LIST is a bit wrong # toalloc:2148 free:{1: 250} chanlimit:250 correction:2147 if toalloc > sum(chanfree[0].values()): - sum_free = sum(chanfree[0].values()) # 250 - chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 - relays_needed = chans_not_covered / chanfree[1] # 1898 / 250 = 7.592 - relays_needed_rounded = ceil(relays_needed) - debug(f"emptyChanAllocate() secondary allocation sum_free:{sum_free} chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} relays_needed_rounded:{relays_needed_rounded}") - #correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1]) - correction = relays_needed_rounded - debug(f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}") + sum_free = sum(chanfree[0].values()) # 250 + chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 + relays_needed = chans_not_covered / chanfree[1] # 1898 / 250 = 7.592 + correction = ceil(relays_needed) + debug( + ( + f"emptyChanAllocate() secondary allocation sum_free:{sum_free} " + f"chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} " + f"correction:{correction}" + ) + ) + debug( + ( + f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} " + f"free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}" + ) + ) warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) modules.provision.provisionMultipleRelays(net, correction) return False From 8ba4831d9c72d632929ecb691ec59416137cafc3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 21:44:19 +0100 Subject: [PATCH 310/394] Implement best effort allocation --- modules/chankeep.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 7f7ce14..89b8e60 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -5,7 +5,7 @@ from twisted.internet.threads import deferToThread import main import modules.provision -from utils.logging.debug import debug +from utils.logging.debug import debug, trace from utils.logging.log import error, log, warn @@ -77,22 +77,31 @@ def emptyChanAllocate(net, flist, relay, new): chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 relays_needed = chans_not_covered / chanfree[1] # 1898 / 250 = 7.592 correction = ceil(relays_needed) - debug( - ( - f"emptyChanAllocate() secondary allocation sum_free:{sum_free} " - f"chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} " - f"correction:{correction}" + if main.config["ChanKeep"]["Provision"]: + debug( + ( + f"emptyChanAllocate() secondary allocation sum_free:{sum_free} " + f"chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} " + f"correction:{correction}" + ) ) - ) - debug( - ( - f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} " - f"free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}" + debug( + ( + f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} " + f"free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}" + ) ) - ) - warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) - modules.provision.provisionMultipleRelays(net, correction) - return False + warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) + modules.provision.provisionMultipleRelays(net, correction) + return False + else: + # We don't have enough spaces and we can't add any. + # Let's do the best we can in the circumstances. + debug(f"emptyChanAllocate() cannot create additional relays for {net}") + debug(f"emptyChanAllocate() {chans_not_covered} channels cannot be covered") + flist = flist[:sum_free] + debug(f"emptyChanAllocate() flist truncated to {sum_free}, length nis now {len(flist)}") + trace(f"emptyChanAllocate() best effort allocation: {flist}") for i in chanfree[0].keys(): for x in range(chanfree[0][i]): if not len(flist): From 8dc176aa5413523dd82e49a2a8db001afe94b93c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 9 Aug 2022 07:20:30 +0100 Subject: [PATCH 311/394] Fire a fake event when we send a message --- api/views.py | 24 ++++++++++++++++++++++++ core/bot.py | 4 ++++ 2 files changed, 28 insertions(+) diff --git a/api/views.py b/api/views.py index d899e5b..602bc50 100644 --- a/api/views.py +++ b/api/views.py @@ -478,3 +478,27 @@ class API(object): return dumps({"success": False, "reason": f"could not get first relay for {net}"}) first_relay.list() return dumps({"success": True, "message": f"requested list with instance {first_relay.num} of {net}"}) + + @app.route("/irc/msg////", methods=["PUT"]) + @login_required + def irc_send_message(self, request, net, num, channel): + print("IRC SEND", net, num, channel) + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "msg" not in data: + return dumps({"success": False, "reason": "no msg."}) + msg = data["msg"] + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + if num not in main.network[net].relays.keys(): + return dumps({"success": False, "reason": f"no relay {num} on {net}"}) + name = f"{net}{num}" + if name not in main.IRCPool.keys(): + return dumps({"success": False, "reason": f"relay {num} not on {net}"}) + main.IRCPool[name].sendmsg(channel, msg) + return dumps({"success": True, "message": f"sent message to {channel} on {name}"}) diff --git a/core/bot.py b/core/bot.py index 579bab8..81a74b0 100644 --- a/core/bot.py +++ b/core/bot.py @@ -252,6 +252,10 @@ class IRCBot(IRCClient): def action(self, user, channel, msg): self.event(type="action", muser=user, channel=channel, msg=msg) + def sendmsg(self, channel, msg): + self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, msg=msg) + self.msg(channel, msg) + def get(self, var): try: result = getattr(self, var) From ec943203d0ead17a30721bb1b7e034aebdcc1c2e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 9 Aug 2022 07:20:30 +0100 Subject: [PATCH 312/394] Get our hostname from WHO when we create fake events --- api/views.py | 1 - core/bot.py | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index 602bc50..e9b2ef6 100644 --- a/api/views.py +++ b/api/views.py @@ -482,7 +482,6 @@ class API(object): @app.route("/irc/msg////", methods=["PUT"]) @login_required def irc_send_message(self, request, net, num, channel): - print("IRC SEND", net, num, channel) try: data = loads(request.content.read()) except JSONDecodeError: diff --git a/core/bot.py b/core/bot.py index 81a74b0..47348e4 100644 --- a/core/bot.py +++ b/core/bot.py @@ -16,7 +16,7 @@ from twisted.words.protocols.irc import ( import main from core.relay import sendRelayNotification -from modules import chankeep, counters, helpers, monitor, regproc, userinfo +from modules import chankeep, counters, helpers, monitor, regproc, userinfo, userinfo from utils.dedup import dedup from utils.logging.debug import debug from utils.logging.log import error, log, warn @@ -253,7 +253,17 @@ class IRCBot(IRCClient): self.event(type="action", muser=user, channel=channel, msg=msg) def sendmsg(self, channel, msg): - self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, msg=msg) + query = f"{self.nickname}!*@*" + us = list(userinfo.getWhoSingle(self.net, query)) + if len(us) > 0: + hostmask = us[0] + else: + # Close enough... + hostmask = f"{self.nickname}!*@{self.servername}" + warn(f"Could not get a hostname, using {hostmask}") + nick, ident, host = parsen(hostmask) + self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) + self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) self.msg(channel, msg) def get(self, var): From a2b6ebd9122b46958fc2ce218be6489d3075f346 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 9 Aug 2022 07:20:30 +0100 Subject: [PATCH 313/394] Properly implement querying with API --- api/views.py | 9 ++++++++- core/bot.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/api/views.py b/api/views.py index e9b2ef6..9d2348d 100644 --- a/api/views.py +++ b/api/views.py @@ -499,5 +499,12 @@ class API(object): name = f"{net}{num}" if name not in main.IRCPool.keys(): return dumps({"success": False, "reason": f"relay {num} not on {net}"}) - main.IRCPool[name].sendmsg(channel, msg) + # We are in a query + if channel == main.IRCPool[name].nickname: + if "nick" not in data: + return dumps({"success": False, "reason": "no nick specified to query"}) + else: + main.IRCPool[name].sendmsg(data["nick"], msg, in_query=True) + else: + main.IRCPool[name].sendmsg(channel, msg) return dumps({"success": True, "message": f"sent message to {channel} on {name}"}) diff --git a/core/bot.py b/core/bot.py index 47348e4..8014f4b 100644 --- a/core/bot.py +++ b/core/bot.py @@ -16,7 +16,7 @@ from twisted.words.protocols.irc import ( import main from core.relay import sendRelayNotification -from modules import chankeep, counters, helpers, monitor, regproc, userinfo, userinfo +from modules import chankeep, counters, helpers, monitor, regproc, userinfo from utils.dedup import dedup from utils.logging.debug import debug from utils.logging.log import error, log, warn @@ -252,7 +252,7 @@ class IRCBot(IRCClient): def action(self, user, channel, msg): self.event(type="action", muser=user, channel=channel, msg=msg) - def sendmsg(self, channel, msg): + def sendmsg(self, channel, msg, in_query=False): query = f"{self.nickname}!*@*" us = list(userinfo.getWhoSingle(self.net, query)) if len(us) > 0: @@ -262,8 +262,12 @@ class IRCBot(IRCClient): hostmask = f"{self.nickname}!*@{self.servername}" warn(f"Could not get a hostname, using {hostmask}") nick, ident, host = parsen(hostmask) - self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) - self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) + # We sent someone a query reply + if in_query: + self.event(type="self", mtype="msg", channel=self.nickname, nick=channel, ident=ident, host=host, msg=msg) + else: + self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) + self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) self.msg(channel, msg) def get(self, var): From 7e51178a10dd31d1302ec35bf80cffc59ab5f6be Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 9 Aug 2022 07:20:30 +0100 Subject: [PATCH 314/394] Add endpoint to get the bot's nickname --- api/views.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/views.py b/api/views.py index 9d2348d..2af0241 100644 --- a/api/views.py +++ b/api/views.py @@ -508,3 +508,18 @@ class API(object): else: main.IRCPool[name].sendmsg(channel, msg) return dumps({"success": True, "message": f"sent message to {channel} on {name}"}) + + @app.route("/irc/nick///", methods=["GET"]) + @login_required + def irc_get_nick(self, request, net, num): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + if num not in main.network[net].relays.keys(): + return dumps({"success": False, "reason": f"no relay {num} on {net}"}) + name = f"{net}{num}" + if name not in main.IRCPool.keys(): + return dumps({"success": False, "reason": f"relay {num} not on {net}"}) + return dumps({"nickname": main.IRCPool[name].nickname}) \ No newline at end of file From 5c2ef740e612dd6142682d0afca808aa96c0704f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 21:47:27 +0100 Subject: [PATCH 315/394] Fix email command --- commands/email.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/commands/email.py b/commands/email.py index c388a56..fa99fa0 100644 --- a/commands/email.py +++ b/commands/email.py @@ -11,8 +11,21 @@ class EmailCommand: if authed: if length == 4: if spl[1] == "add": - if not spl[2].isdigit(): - # failure("Must be a number, not %s" % spl[2]) + if spl[2].isdigit(): + num = int(spl[2]) + if num not in main.alias.keys(): + failure("No such alias: %i" % num) + return + if not spl[3] in main.alias[num]["emails"]: + main.alias[num]["emails"].append(spl[3]) + main.saveConf("alias") + success("Successfully added email %s to alias %i" % (spl[3], num)) + return + else: + failure("Email already exists in alias %i: %s" % (num, spl[3])) + return + else: + failure("Must be a number, not %s" % spl[2]) if spl[2] == "domain": domain = spl[3] if "@" in domain: @@ -25,6 +38,7 @@ class EmailCommand: else: failure("Domain already exists in default config: %s" % domain) return + elif spl[1] == "del": if not spl[2].isdigit(): # failure("Must be a number, not %s" % spl[2]) @@ -39,20 +53,6 @@ class EmailCommand: failure("Domain does not exist in default config: %s" % domain) return - else: - num = int(spl[2]) - if spl[1] == "add": - if num not in main.alias.keys(): - failure("No such alias: %i" % num) - return - if not spl[3] in main.alias[num]["emails"]: - main.alias[num]["emails"].append(spl[3]) - main.saveConf("alias") - success("Successfully added email %s to alias %i" % (spl[3], num)) - return - else: - failure("Email already exists in alias %i: %s" % (num, spl[3])) - return elif spl[1] == "del": if num not in main.alias.keys(): failure("No such alias: %i" % num) From 6306231098faf92b04b0ac383c33fde269844151 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 11 Aug 2022 21:47:44 +0100 Subject: [PATCH 316/394] Make channel join notification a TRACE --- core/bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index 8014f4b..0808ee9 100644 --- a/core/bot.py +++ b/core/bot.py @@ -18,7 +18,7 @@ import main from core.relay import sendRelayNotification from modules import chankeep, counters, helpers, monitor, regproc, userinfo from utils.dedup import dedup -from utils.logging.debug import debug +from utils.logging.debug import debug, trace from utils.logging.log import error, log, warn from utils.logging.send import sendAll from utils.parsing import parsen @@ -116,7 +116,7 @@ class IRCBot(IRCClient): if i in main.blacklist[self.net]: debug("Not joining blacklisted channel %s on %s - %i" % (i, self.net, self.num)) continue - debug(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds") + trace(self.net, "-", self.num, ": joining", i, "in", sleeptime, "seconds") reactor.callLater(sleeptime, self.join, i) sleeptime += increment if sleeptime == 10: From 065fe94cbd913896b12e67307f9ad3f7957d4d6a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 12 Aug 2022 22:27:49 +0100 Subject: [PATCH 317/394] Improve channel allocation and write basic tests for it --- api/views.py | 2 +- modules/chankeep.py | 41 +++++++++++----- runtest.sh | 3 ++ tests/test_chankeep.py | 109 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 12 deletions(-) create mode 100755 runtest.sh create mode 100644 tests/test_chankeep.py diff --git a/api/views.py b/api/views.py index 2af0241..d5795a2 100644 --- a/api/views.py +++ b/api/views.py @@ -522,4 +522,4 @@ class API(object): name = f"{net}{num}" if name not in main.IRCPool.keys(): return dumps({"success": False, "reason": f"relay {num} not on {net}"}) - return dumps({"nickname": main.IRCPool[name].nickname}) \ No newline at end of file + return dumps({"nickname": main.IRCPool[name].nickname}) diff --git a/modules/chankeep.py b/modules/chankeep.py index 89b8e60..6865f28 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -61,7 +61,7 @@ def getChanFree(net, new): return (chanfree, chanlimits.pop()) -def emptyChanAllocate(net, flist, relay, new): +def emptyChanAllocate(net, flist, new): chanfree = getChanFree(net, new) if not chanfree: return @@ -102,20 +102,21 @@ def emptyChanAllocate(net, flist, relay, new): flist = flist[:sum_free] debug(f"emptyChanAllocate() flist truncated to {sum_free}, length nis now {len(flist)}") trace(f"emptyChanAllocate() best effort allocation: {flist}") + newlist = list(flist) for i in chanfree[0].keys(): for x in range(chanfree[0][i]): - if not len(flist): + if not len(newlist): break if i in allocated.keys(): - allocated[i].append(flist.pop()) + allocated[i].append(newlist.pop()) else: - allocated[i] = [flist.pop()] + allocated[i] = [newlist.pop()] return allocated -def populateChans(net, clist, relay, new): +def populateChans(net, clist, new): # divided = array_split(clist, relay) - allocated = emptyChanAllocate(net, clist, relay, new) + allocated = emptyChanAllocate(net, clist, new) if not allocated: return for i in allocated.keys(): @@ -148,7 +149,7 @@ def minifyChans(net, listinfo): return listinfo -def keepChannels(net, listinfo, mean, sigrelay, relay): +def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): listinfo = minifyChans(net, listinfo) if not listinfo: return @@ -159,23 +160,27 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) return + num_instances = len(getActiveRelays(net)) + max_chans = chanlimit * num_instances if coverAll: needed = relay - len(getActiveRelays(net)) debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") newNums = modules.provision.provisionMultipleRelays(net, needed) flist = [i[0] for i in listinfo] - populateChans(net, flist, relay, newNums) + chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + populateChans(net, chosen, newNums) else: needed = sigrelay - len(getActiveRelays(net)) debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}") newNums = modules.provision.provisionMultipleRelays(net, needed) siglist = [i[0] for i in listinfo if int(i[1]) > mean] - populateChans(net, siglist, sigrelay, newNums) + chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] + populateChans(net, chosen, newNums) notifyJoin(net) def joinSingle(net, channel): - eca = emptyChanAllocate(net, [channel], None, []) + eca = emptyChanAllocate(net, [channel], []) if not eca: return False if not len(eca.keys()) == 1: @@ -272,8 +277,22 @@ def _initialList(net, num, listinfo, chanlimit): p.execute() debug("List parsing completed on %s" % net) - keepChannels(net, listinfo, mean, sigrelay, relay) + keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit) + + # return (listinfo, mean, sigrelay, relay) def initialList(net, num, listinfo, chanlimit): deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) + + +def chankeep_handler(net, num, listinfo, chanlimit): + """ + Handle a channel keep request. + :param net: + :param num: + :param listinfo: + :param chanlimit: + :return: + """ + listinfo, mean, sigrelay, relay = _initialList(net, num, listinfo, chanlimit) \ No newline at end of file diff --git a/runtest.sh b/runtest.sh new file mode 100755 index 0000000..1ee7505 --- /dev/null +++ b/runtest.sh @@ -0,0 +1,3 @@ +#!/bin/sh +#pre-commit run -a +python -m unittest discover -s tests -p 'test_*.py' diff --git a/tests/test_chankeep.py b/tests/test_chankeep.py new file mode 100644 index 0000000..f35c653 --- /dev/null +++ b/tests/test_chankeep.py @@ -0,0 +1,109 @@ +import logging +from unittest import TestCase +from unittest.mock import MagicMock, patch +from random import randint +from modules import chankeep +from math import ceil +import heapq + +class TestChanKeep(TestCase): + def setUp(self): + self.net = "testnet" + self.num = 1 + self.chanlimit = 100 + chankeep.main.initConf() + chankeep.main.r = MagicMock() + chankeep.main.g = MagicMock() + chankeep.main.g.pipeline = MagicMock() + chankeep.main.config["ChanKeep"]["Provision"] = False + + self.listinfo = self.generate_listinfo() + self.chan_name_list = [x[0] for x in self.listinfo] + self.chan_member_list = [x[1] for x in self.listinfo] + + def generate_listinfo(self, ranges=None): + """ + Create a fake listinfo. + Where #channel has 192 users, and #channel2 has 188 users. + listinfo = [["#channel", 192], ["#channel2", 188]] + """ + if not ranges: + ranges = [[100, 5, 10], [400, 100, 200], [2, 500, 1000]] + listinfo = [] + for channum, min, max in ranges: + for i in range(channum): + chan_name = f"#num-{channum}-{i}" + chan_users = randint(min, max) + listinfo.append([chan_name, chan_users]) + return listinfo + + def percent_diff(self, a, b): + return (abs(b - a) / a) * 100.0 + + def test_alt_listinfo(self): + # We're looking for a perc of 1000-1100 + # And a sigrelay of 2 + # We only want those 10 big channels + instances = 1 + chanlimit = 5 + max_chans = instances * chanlimit + listinfo = self.generate_listinfo(ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]]) + listinfo_num = [x[1] for x in listinfo] + + listlength = len(listinfo) + cumul = 0 + try: + cumul += sum(int(i[1]) for i in listinfo) + except TypeError: + return + mean = round(cumul / listlength, 2) + siglength = 0 + insiglength = 0 + sigcumul = 0 + insigcumul = 0 + for i in listinfo: + if int(i[1]) > mean: + siglength += 1 + sigcumul += int(i[1]) + elif int(i[1]) < mean: + insiglength += 1 + insigcumul += int(i[1]) + + sigrelay = ceil(siglength / chanlimit) + relay = ceil(listlength / chanlimit) + print(f"len:{listlength} cumul:{cumul} mean:{mean} siglength:{siglength} insiglength:{insiglength} sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} insigcumul:{insigcumul}") + # We want a return between 1000 and 1100 + + list_insig = [x for x in listinfo_num if x < mean] + list_sig = [x for x in listinfo if x[1] > mean] + chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans] + print("CHOSEN", chosen) + self.assertEqual(len(chosen), 5) + + @patch("modules.chankeep.keepChannels") + def test__initialList(self, keepchannels): + chankeep._initialList(self.net, self.num, self.listinfo, self.chanlimit) + net, passed_list, mean, sigrelay, relay, chanlimit = keepchannels.call_args_list[0][0] + self.assertEqual(net, self.net) + self.assertEqual(passed_list, self.listinfo) + self.assertEqual(chanlimit, self.chanlimit) + # print(net, mean, sigrelay, relay) + + @patch("modules.chankeep.getChanFree") + def test_empty_chan_allocate(self, getchanfree): + getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free + eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, []) + self.assertEqual(len(eca), 1) + num = list(eca.keys())[0] + chans = eca[list(eca.keys())[0]] + self.assertEqual(num, self.num) + self.assertCountEqual(chans, self.chan_name_list) + + getchanfree.return_value = ({1: 100}, 10) + eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, []) + self.assertEqual(len(eca), 1) + num = list(eca.keys())[0] + chans = eca[list(eca.keys())[0]] + self.assertEqual(num, self.num) + self.assertEqual(len(chans), 100) + #self.assertCountEqual(chans, self.chan_name_list) \ No newline at end of file From 20f59362ffc99b3556320eb969be3b3c5e855160 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 12 Aug 2022 22:31:12 +0100 Subject: [PATCH 318/394] Subtract allocated channel slots from total --- modules/chankeep.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 6865f28..f91bfcb 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -60,6 +60,14 @@ def getChanFree(net, new): return False return (chanfree, chanlimits.pop()) +def getTotalChans(net): + total = 0 + for i in getActiveRelays(net): + name = net + str(i) + if name in main.IRCPool.keys(): + total += len(main.IRCPool[name].channels) + return total + def emptyChanAllocate(net, flist, new): chanfree = getChanFree(net, new) @@ -161,7 +169,11 @@ def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) return num_instances = len(getActiveRelays(net)) - max_chans = chanlimit * num_instances + debug(f"keepChannels() {net} instances:{num_instances} chanlimit:{chanlimit}") + chan_slots_used = getTotalChans(net) + debug(f"keepChannels() slots_used:{chan_slots_used}") + max_chans = (chanlimit * num_instances) - chan_slots_used + debug(f"keepChannels() max_chans:{max_chans}") if coverAll: needed = relay - len(getActiveRelays(net)) debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") From 49214644ffdfed15f24a0cee5e233dcbd8e22a43 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 12 Aug 2022 23:32:00 +0100 Subject: [PATCH 319/394] Implement migrating networks --- api/views.py | 4 ++++ modules/chankeep.py | 3 ++- modules/network.py | 28 ++++++++++++++++++++++++++++ modules/provision.py | 5 ----- tests/test_chankeep.py | 27 +++++++++++++++++---------- threshold | 4 ++++ 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/api/views.py b/api/views.py index d5795a2..ce236a4 100644 --- a/api/views.py +++ b/api/views.py @@ -198,6 +198,7 @@ class API(object): def irc_network(self, request, net): if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) + first_relay = helpers.get_first_relay(net) inst = main.network[net] network = {} network["net"] = inst.net @@ -209,6 +210,9 @@ class API(object): network["relays"] = len(inst.relays) network["channels"] = userinfo.getTotalChanNum(net) network["records"] = userinfo.getNumWhoEntries(net) + if first_relay: + network["chanlimit_live"] = first_relay.chanlimit + network["chanlimit_conf"] = inst.chanlimit return dumps(network) @app.route("/irc/network//", methods=["DELETE"]) diff --git a/modules/chankeep.py b/modules/chankeep.py index f91bfcb..5abcb8f 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -60,6 +60,7 @@ def getChanFree(net, new): return False return (chanfree, chanlimits.pop()) + def getTotalChans(net): total = 0 for i in getActiveRelays(net): @@ -307,4 +308,4 @@ def chankeep_handler(net, num, listinfo, chanlimit): :param chanlimit: :return: """ - listinfo, mean, sigrelay, relay = _initialList(net, num, listinfo, chanlimit) \ No newline at end of file + listinfo, mean, sigrelay, relay = _initialList(net, num, listinfo, chanlimit) diff --git a/modules/network.py b/modules/network.py index 7aea0b1..758391e 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,3 +1,5 @@ +from copy import deepcopy + from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory @@ -11,6 +13,31 @@ from utils.get import getRelay from utils.logging.log import log +def migrate(): + existing = deepcopy(main.network) + log("Migrating network configuration") + log(f"Existing network configuration: {existing}") + for net, net_inst in existing.items(): + log(f"Migrating network {net}") + net = net_inst.net + host = net_inst.host + port = net_inst.port + security = net_inst.security + auth = net_inst.auth + last = net_inst.last + relays = net_inst.relays + aliases = net_inst.aliases + + new_net = Network(net, host, port, security, auth) + log(f"New network for {net}: {new_net}") + new_net.last = last + new_net.relays = relays + new_net.aliases = aliases + main.network[net] = new_net + main.saveConf("network") + log("Finished migrating network configuration") + + class Network: def __init__(self, net, host, port, security, auth): self.net = net @@ -18,6 +45,7 @@ class Network: self.port = port self.security = security self.auth = auth + self.chanlimit = None self.last = 1 self.relays = {} diff --git a/modules/provision.py b/modules/provision.py index 01d371f..d5a0417 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -9,11 +9,6 @@ from utils.logging.log import warn def provisionUserNetworkData( num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password ): - print("nick", nick) - print("altnick", altnick) - print("emails", emails) - print("ident", ident) - print("realname", realname) commands = {} stage2commands = {} stage2commands["status"] = [] diff --git a/tests/test_chankeep.py b/tests/test_chankeep.py index f35c653..13c8918 100644 --- a/tests/test_chankeep.py +++ b/tests/test_chankeep.py @@ -1,10 +1,10 @@ -import logging +from math import ceil +from random import randint from unittest import TestCase from unittest.mock import MagicMock, patch -from random import randint + from modules import chankeep -from math import ceil -import heapq + class TestChanKeep(TestCase): def setUp(self): @@ -48,7 +48,7 @@ class TestChanKeep(TestCase): chanlimit = 5 max_chans = instances * chanlimit listinfo = self.generate_listinfo(ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]]) - listinfo_num = [x[1] for x in listinfo] + # listinfo_num = [x[1] for x in listinfo] listlength = len(listinfo) cumul = 0 @@ -71,15 +71,22 @@ class TestChanKeep(TestCase): sigrelay = ceil(siglength / chanlimit) relay = ceil(listlength / chanlimit) - print(f"len:{listlength} cumul:{cumul} mean:{mean} siglength:{siglength} insiglength:{insiglength} sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} insigcumul:{insigcumul}") + print( + ( + f"len:{listlength} cumul:{cumul} mean:{mean} " + f"siglength:{siglength} insiglength:{insiglength} " + f"sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} " + f"insigcumul:{insigcumul}" + ) + ) # We want a return between 1000 and 1100 - list_insig = [x for x in listinfo_num if x < mean] + # list_insig = [x for x in listinfo_num if x < mean] list_sig = [x for x in listinfo if x[1] > mean] chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans] print("CHOSEN", chosen) self.assertEqual(len(chosen), 5) - + @patch("modules.chankeep.keepChannels") def test__initialList(self, keepchannels): chankeep._initialList(self.net, self.num, self.listinfo, self.chanlimit) @@ -91,7 +98,7 @@ class TestChanKeep(TestCase): @patch("modules.chankeep.getChanFree") def test_empty_chan_allocate(self, getchanfree): - getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free + getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, []) self.assertEqual(len(eca), 1) num = list(eca.keys())[0] @@ -106,4 +113,4 @@ class TestChanKeep(TestCase): chans = eca[list(eca.keys())[0]] self.assertEqual(num, self.num) self.assertEqual(len(chans), 100) - #self.assertCountEqual(chans, self.chan_name_list) \ No newline at end of file + # self.assertCountEqual(chans, self.chan_name_list) diff --git a/threshold b/threshold index c728709..3b4bbbf 100755 --- a/threshold +++ b/threshold @@ -29,7 +29,11 @@ if "--debug" in sys.argv: # yes really main.config["Debug"] = True if "--trace" in sys.argv: main.config["Trace"] = True +if "--migrate" in sys.argv: + from modules.network import migrate + migrate() + exit() loadCommands() From acc363d207f3f0fc5ef0415c258e9f33178ceb75 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 12 Aug 2022 23:53:02 +0100 Subject: [PATCH 320/394] Add docstrings to chankeep --- modules/chankeep.py | 116 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 17 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 5abcb8f..85865f2 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -10,6 +10,11 @@ from utils.logging.log import error, log, warn def getActiveRelays(net): + """ + Get a list of active relays for a network. + :param net: network + :rtype: list of int + :return: list of active relay numbers""" activeRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] return activeRelays @@ -17,6 +22,9 @@ def getActiveRelays(net): def allRelaysActive(net): """ Check if all enabled relays are active and authenticated. + :param net: network + :rtype: bool + :return: True if all relays are active and authenticated, False otherwise """ activeRelays = getActiveRelays(net) debug(f"allRelaysActive() active relays for {net}: {activeRelays}") @@ -62,6 +70,12 @@ def getChanFree(net, new): def getTotalChans(net): + """ + Get the total number of channels on all relays for a network. + :param net: network + :rtype: int + :return: total number of channels + """ total = 0 for i in getActiveRelays(net): name = net + str(i) @@ -71,16 +85,31 @@ def getTotalChans(net): def emptyChanAllocate(net, flist, new): + """ + Allocate channels to relays. + :param net: network + :param flist: list of channels to allocate + :param new: list of newly provisioned relays to account for + :rtype: dict + :return: dictionary of {relay: list of channels}""" + + # Get the free channel spaces for each relay chanfree = getChanFree(net, new) if not chanfree: return + + # Pretend the newly provisioned relays are already on the network for i in new: chanfree[0][i] = chanfree[1] allocated = {} + + # Copy the list since we're going to mutate it toalloc = len(flist) + # Used to correct allocations and provision additional relays # if the math since the last LIST is a bit wrong # toalloc:2148 free:{1: 250} chanlimit:250 correction:2147 + newlist = list(flist) if toalloc > sum(chanfree[0].values()): sum_free = sum(chanfree[0].values()) # 250 chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 @@ -108,10 +137,9 @@ def emptyChanAllocate(net, flist, new): # Let's do the best we can in the circumstances. debug(f"emptyChanAllocate() cannot create additional relays for {net}") debug(f"emptyChanAllocate() {chans_not_covered} channels cannot be covered") - flist = flist[:sum_free] + newlist = newlist[:sum_free] debug(f"emptyChanAllocate() flist truncated to {sum_free}, length nis now {len(flist)}") trace(f"emptyChanAllocate() best effort allocation: {flist}") - newlist = list(flist) for i in chanfree[0].keys(): for x in range(chanfree[0][i]): if not len(newlist): @@ -124,6 +152,12 @@ def emptyChanAllocate(net, flist, new): def populateChans(net, clist, new): + """ + Populate channels on relays. + Stores channels to join in a list in main.TempChan[net][num] + :param net: network + :param clist: list of channels to join + :param new: list of newly provisioned relays to account for""" # divided = array_split(clist, relay) allocated = emptyChanAllocate(net, clist, new) if not allocated: @@ -136,6 +170,11 @@ def populateChans(net, clist, new): def notifyJoin(net): + """ + Notify relays to join channels. + They will pull from main.TempChan and remove channels they join. + :param net: network + """ for i in getActiveRelays(net): name = net + str(i) if name in main.IRCPool.keys(): @@ -143,6 +182,14 @@ def notifyJoin(net): def minifyChans(net, listinfo): + """ + Remove channels from listinfo that are already covered by a relay. + :param net: network + :param listinfo: list of channels to check + :type listinfo: list of [channel, num_users] + :return: list of channels with joined channels removed + :rtype: list of [channel, num_users] + """ if not allRelaysActive(net): error("All relays for %s are not active, cannot minify list" % net) return False @@ -159,6 +206,19 @@ def minifyChans(net, listinfo): def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): + """ + Minify channels, determine whether we can cover all the channels + on the network, or need to use 'significant' mode. + Truncate the channel list to available channel spaces. + Allocate these channels to relays. + Notify relays that they should pull from TempChan to join. + :param net: network + :param listinfo: list of [channel, num_users] lists + :param mean: mean of channel population + :param sigrelay: number of relays needed to cover significant channels + :param relay: number of relays needed to cover all channels + :param chanlimit: maximum number of channels to allocate to a relay + """ listinfo = minifyChans(net, listinfo) if not listinfo: return @@ -193,6 +253,14 @@ def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): def joinSingle(net, channel): + """ + Join a channel on a relay. + Use ECA to determine which relay to join on. + :param net: network + :param channel: channel to join + :return: relay number that joined the channel + :rtype: int + """ eca = emptyChanAllocate(net, [channel], []) if not eca: return False @@ -209,9 +277,10 @@ def joinSingle(net, channel): def partSingle(net, channel): """ Iterate over all the relays of net and part channels matching channel. - :param net: - :param channel: - :return: + :param net: network + :param channel: channel to part + :return: list of relays that parted the channel + :rtype: list of str """ parted = [] for i in getActiveRelays(net): @@ -224,6 +293,9 @@ def partSingle(net, channel): def nukeNetwork(net): + """ + Remove network records. + :param net: network""" # purgeRecords(net) # p = main.g.pipeline() main.g.delete("analytics.list." + net) @@ -236,6 +308,21 @@ def nukeNetwork(net): def _initialList(net, num, listinfo, chanlimit): + """ + Called when a relay receives a full LIST response. + Run statistics to determine how many channels are significant. + This is done by adding all the numbers of users on the channels together, + then dividing by the number of channels. + * cumul - cumulative sum of all channel membership + * siglength - number of significant channels + * listlength - number of channels in the list + * sigrelay - number of relays needed to cover siglength + * relay - number of relays needed to cover all channels + :param net: network + :param num: relay number + :param listinfo: list of [channel, num_users] lists + :param chanlimit: maximum number of channels the relay can join + """ listlength = len(listinfo) cumul = 0 try: @@ -258,9 +345,11 @@ def _initialList(net, num, listinfo, chanlimit): sigrelay = ceil(siglength / chanlimit) relay = ceil(listlength / chanlimit) - # netbase = "list.%s" % net + abase = "analytics.list.%s" % net p = main.g.pipeline() + + # See docstring for meanings p.hset(abase, "mean", mean) p.hset(abase, "total", listlength) p.hset(abase, "sigtotal", siglength) @@ -296,16 +385,9 @@ def _initialList(net, num, listinfo, chanlimit): def initialList(net, num, listinfo, chanlimit): + """ + Run _initialList in a thread. + See above docstring. + """ deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) - -def chankeep_handler(net, num, listinfo, chanlimit): - """ - Handle a channel keep request. - :param net: - :param num: - :param listinfo: - :param chanlimit: - :return: - """ - listinfo, mean, sigrelay, relay = _initialList(net, num, listinfo, chanlimit) From c3fd8a97f73e71b3ac1ea3142e0a21d74cbb8491 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 00:18:06 +0100 Subject: [PATCH 321/394] Provision relay on creation --- modules/network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/network.py b/modules/network.py index 758391e..233b76b 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,4 +1,5 @@ from copy import deepcopy +from modules import provision from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory @@ -8,6 +9,7 @@ from core.bot import IRCBotFactory from modules import alias from modules.chankeep import nukeNetwork from modules.regproc import needToRegister +from modules.provision import provisionRelay from utils.deliver_relay_commands import deliverRelayCommands from utils.get import getRelay from utils.logging.log import log @@ -77,6 +79,7 @@ class Network: self.aliases[num] = {"password": password} # if main.config["ConnectOnCreate"]: -- Done in provision # self.start_bot(num) + provisionRelay(num, self.net) return num, main.alias[num]["nick"] def enable_relay(self, num): From 5c63fb5048986741dab774dc6a1df8575a666788 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 13:27:20 +0100 Subject: [PATCH 322/394] Implement getting LIST information from API --- api/views.py | 14 ++++++++++---- modules/chankeep.py | 43 ++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/api/views.py b/api/views.py index ce236a4..60765c0 100644 --- a/api/views.py +++ b/api/views.py @@ -198,7 +198,6 @@ class API(object): def irc_network(self, request, net): if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) - first_relay = helpers.get_first_relay(net) inst = main.network[net] network = {} network["net"] = inst.net @@ -210,9 +209,6 @@ class API(object): network["relays"] = len(inst.relays) network["channels"] = userinfo.getTotalChanNum(net) network["records"] = userinfo.getNumWhoEntries(net) - if first_relay: - network["chanlimit_live"] = first_relay.chanlimit - network["chanlimit_conf"] = inst.chanlimit return dumps(network) @app.route("/irc/network//", methods=["DELETE"]) @@ -483,6 +479,15 @@ class API(object): first_relay.list() return dumps({"success": True, "message": f"requested list with instance {first_relay.num} of {net}"}) + @app.route("/irc/list//", methods=["GET"]) + @login_required + def get_irc_list_info(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + listinfo = chankeep.getListInfo(net) + return dumps({"success": True, "listinfo": listinfo}) + + @app.route("/irc/msg////", methods=["PUT"]) @login_required def irc_send_message(self, request, net, num, channel): @@ -527,3 +532,4 @@ class API(object): if name not in main.IRCPool.keys(): return dumps({"success": False, "reason": f"relay {num} not on {net}"}) return dumps({"nickname": main.IRCPool[name].nickname}) + diff --git a/modules/chankeep.py b/modules/chankeep.py index 85865f2..0de8baf 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -347,21 +347,22 @@ def _initialList(net, num, listinfo, chanlimit): relay = ceil(listlength / chanlimit) abase = "analytics.list.%s" % net + main.g.delete(abase) p = main.g.pipeline() # See docstring for meanings p.hset(abase, "mean", mean) - p.hset(abase, "total", listlength) - p.hset(abase, "sigtotal", siglength) - p.hset(abase, "insigtotal", insiglength) - p.hset(abase, "sigperc", round(siglength / listlength * 100, 2)) - p.hset(abase, "insigperc", round(insiglength / listlength * 100, 2)) - p.hset(abase, "cumul", cumul) - p.hset(abase, "sigcumul", sigcumul) - p.hset(abase, "insigcumul", insigcumul) - p.hset(abase, "relay", relay) - p.hset(abase, "sigrelay", sigrelay) - p.hset(abase, "insigrelay", ceil(insiglength / chanlimit)) + p.hset(abase, "total_chans", listlength) + p.hset(abase, "big_chans", siglength) + p.hset(abase, "small_chans", insiglength) + p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2)) + p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2)) + p.hset(abase, "total_cumul_mem", cumul) + p.hset(abase, "big_chan_cumul_mem", sigcumul) + p.hset(abase, "small_chan_cumul_mem", insigcumul) + p.hset(abase, "relays_for_all_chans", relay) + p.hset(abase, "relays_for_big_chans", sigrelay) + p.hset(abase, "relays_for_small_chans", ceil(insiglength / chanlimit)) debug( ( f"_initialList() net:{net} num:{num} listlength:{listlength} " @@ -383,6 +384,26 @@ def _initialList(net, num, listinfo, chanlimit): # return (listinfo, mean, sigrelay, relay) +def convert(data): + """ + Recursively convert a dictionary. + """ + if isinstance(data, bytes): + return data.decode("ascii") + if isinstance(data, dict): + return dict(map(convert, data.items())) + if isinstance(data, tuple): + return map(convert, data) + if isinstance(data, list): + return list(map(convert, data)) + return data + + +def getListInfo(net): + abase = f"analytics.list.{net}" + info = main.g.hgetall(abase) + return convert(info) + def initialList(net, num, listinfo, chanlimit): """ From 21ed66bc0095c045e9124db2a7931ad2c9ff1f9e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 13:32:22 +0100 Subject: [PATCH 323/394] Reformat code --- api/views.py | 2 -- modules/chankeep.py | 2 +- modules/network.py | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/api/views.py b/api/views.py index 60765c0..9bb2e97 100644 --- a/api/views.py +++ b/api/views.py @@ -487,7 +487,6 @@ class API(object): listinfo = chankeep.getListInfo(net) return dumps({"success": True, "listinfo": listinfo}) - @app.route("/irc/msg////", methods=["PUT"]) @login_required def irc_send_message(self, request, net, num, channel): @@ -532,4 +531,3 @@ class API(object): if name not in main.IRCPool.keys(): return dumps({"success": False, "reason": f"relay {num} not on {net}"}) return dumps({"nickname": main.IRCPool[name].nickname}) - diff --git a/modules/chankeep.py b/modules/chankeep.py index 0de8baf..7279e97 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -384,6 +384,7 @@ def _initialList(net, num, listinfo, chanlimit): # return (listinfo, mean, sigrelay, relay) + def convert(data): """ Recursively convert a dictionary. @@ -411,4 +412,3 @@ def initialList(net, num, listinfo, chanlimit): See above docstring. """ deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) - diff --git a/modules/network.py b/modules/network.py index 233b76b..b07f831 100644 --- a/modules/network.py +++ b/modules/network.py @@ -1,5 +1,4 @@ from copy import deepcopy -from modules import provision from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory @@ -8,8 +7,8 @@ import main from core.bot import IRCBotFactory from modules import alias from modules.chankeep import nukeNetwork -from modules.regproc import needToRegister from modules.provision import provisionRelay +from modules.regproc import needToRegister from utils.deliver_relay_commands import deliverRelayCommands from utils.get import getRelay from utils.logging.log import log From facf58ec2c2417b5334d12c7ee01a2bda4abb5fc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 13:40:33 +0100 Subject: [PATCH 324/394] Add connected status to IRC info return and check when getting active relays --- api/views.py | 2 ++ modules/chankeep.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 9bb2e97..57c1238 100644 --- a/api/views.py +++ b/api/views.py @@ -310,9 +310,11 @@ class API(object): if name in main.IRCPool.keys(): to_append["chans"] = len(main.IRCPool[name].channels) to_append["nick"] = main.IRCPool[name].nickname + to_append["conn"] = main.IRCPool[name].isconnected else: to_append["chans"] = 0 to_append["nick"] = None + to_append["conn"] = False relays.append(to_append) return dumps({"relays": relays}) diff --git a/modules/chankeep.py b/modules/chankeep.py index 7279e97..1a497e0 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -33,7 +33,7 @@ def allRelaysActive(net): for i in activeRelays: name = net + str(i) if name in main.IRCPool.keys(): - if main.IRCPool[name].authenticated: + if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: existNum += 1 debug(f"allRelaysActive() finished, {existNum}/{relayNum} relays active for {net}") if existNum == relayNum: From df6b9e34a3f1b2ffeef94be0c602f5b70554b2b6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 13:47:42 +0100 Subject: [PATCH 325/394] Return relay numbers with channel list --- api/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 57c1238..a19a4a1 100644 --- a/api/views.py +++ b/api/views.py @@ -391,14 +391,16 @@ class API(object): if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) relays_inst = main.network[net].relays - channels = {} + channels = [] for num in relays_inst.keys(): name = f"{net}{num}" if name in main.IRCPool.keys(): net_chans = main.IRCPool[name].channels channels_annotated = userinfo.getUserNum(net, net_chans) for channel in net_chans: - channels[channel] = channels_annotated[channel] + channel_inst = {"name": channel, "users": channels_annotated[channel], "num": num} + channels.append(channel_inst) + # channels[channel] = channels_annotated[channel] return dumps({"channels": channels}) From 1e9dd1b2232112e4d4edf494b887fcfd40374e23 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 14:06:34 +0100 Subject: [PATCH 326/394] Print message if relay is unauthenticated/disconnected --- modules/chankeep.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index 1a497e0..71f655c 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -35,6 +35,14 @@ def allRelaysActive(net): if name in main.IRCPool.keys(): if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: existNum += 1 + else: + debug(f"allRelaysActive() {name} is not authenticated or connected") + debug( + ( + f"allRelaysActive() {name} auth:{main.IRCPool[name].authenticated} " + f"connected:{main.IRCPool[name].isconnected}" + ) + ) debug(f"allRelaysActive() finished, {existNum}/{relayNum} relays active for {net}") if existNum == relayNum: return True From 75965497be4efeb475bc4ef4c1891088ac1d5679 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 18:40:13 +0100 Subject: [PATCH 327/394] Add some debug statements and statistics for chanlimits --- modules/chankeep.py | 89 +++++++++++++++++++++++++++++++------------- modules/provision.py | 2 + 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 71f655c..ea09274 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -8,14 +8,30 @@ import modules.provision from utils.logging.debug import debug, trace from utils.logging.log import error, log, warn +def getEnabledRelays(net): + """ + Get a list of enabled relays for a network. + :param net: network + :rtype: list of int + :return: list of enabled relay numbers + """ + enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] + return enabledRelays def getActiveRelays(net): """ Get a list of active relays for a network. :param net: network :rtype: list of int - :return: list of active relay numbers""" - activeRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] + :return: list of getEnabledRelays relay numbers + """ + enabledRelays = getEnabledRelays(net) + activeRelays = [] + for i in enabledRelays: + name = net + str(i) + if name in main.IRCPool.keys(): + if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: + activeRelays.append(i) return activeRelays @@ -27,27 +43,40 @@ def allRelaysActive(net): :return: True if all relays are active and authenticated, False otherwise """ activeRelays = getActiveRelays(net) - debug(f"allRelaysActive() active relays for {net}: {activeRelays}") - relayNum = len(activeRelays) - existNum = 0 - for i in activeRelays: + enabledRelays = getEnabledRelays(net) + relaysActive = len(activeRelays) == len(enabledRelays) + debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})") + return relaysActive + +def getAverageChanlimit(net): + """ + Get the average channel limit for a network. + :param net: network + :rtype: int + :return: average channel limit + """ + total = 0 + for i in getActiveRelays(net): name = net + str(i) if name in main.IRCPool.keys(): - if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: - existNum += 1 - else: - debug(f"allRelaysActive() {name} is not authenticated or connected") - debug( - ( - f"allRelaysActive() {name} auth:{main.IRCPool[name].authenticated} " - f"connected:{main.IRCPool[name].isconnected}" - ) - ) - debug(f"allRelaysActive() finished, {existNum}/{relayNum} relays active for {net}") - if existNum == relayNum: - return True - return False + total += main.IRCPool[name].chanlimit + avg_chanlimit = total / len(getActiveRelays(net)) + debug(f"getAverageChanlimit() {net}: {avg_chanlimit}") + return avg_chanlimit +def getSumChanlimit(net): + """ + Get the sum of all channel limits for a network. + :param net: network + :rtype: int + :return: sum of channel limits + """ + total = 0 + for i in getActiveRelays(net): + name = net + str(i) + if name in main.IRCPool.keys(): + total += main.IRCPool[name].chanlimit + return total def getChanFree(net, new): """ @@ -245,17 +274,27 @@ def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): debug(f"keepChannels() max_chans:{max_chans}") if coverAll: needed = relay - len(getActiveRelays(net)) - debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") - newNums = modules.provision.provisionMultipleRelays(net, needed) + if needed: + debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") + newNums = modules.provision.provisionMultipleRelays(net, needed) + else: + newNums = [] flist = [i[0] for i in listinfo] chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") + trace(f"keepChannels() {net}: joining:{chosen}") populateChans(net, chosen, newNums) else: needed = sigrelay - len(getActiveRelays(net)) - debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}") - newNums = modules.provision.provisionMultipleRelays(net, needed) + if needed: + debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}") + newNums = modules.provision.provisionMultipleRelays(net, needed) + else: + newNums = [] siglist = [i[0] for i in listinfo if int(i[1]) > mean] - chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] + chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") + trace(f"keepChannels() {net}: joining:{chosen}") populateChans(net, chosen, newNums) notifyJoin(net) diff --git a/modules/provision.py b/modules/provision.py index d5a0417..ca62585 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -75,6 +75,8 @@ def provisionRelay(num, network): # provision user and network data def provisionMultipleRelays(net, relaysNeeded): + if not relaysNeeded: + return [] if not main.config["ChanKeep"]["Provision"]: warn(f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled") return [] From 496a3d03745b6c64bd1e1103f1ec6390583f9fdf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 19:20:29 +0100 Subject: [PATCH 328/394] Implement ChanKeep without requiring persistent chanlimits on all networks --- core/bot.py | 2 +- modules/chankeep.py | 143 ++++++++++++++++------------------------- modules/provision.py | 1 + tests/test_chankeep.py | 12 ++-- 4 files changed, 64 insertions(+), 94 deletions(-) diff --git a/core/bot.py b/core/bot.py index 0808ee9..e6b1009 100644 --- a/core/bot.py +++ b/core/bot.py @@ -473,7 +473,7 @@ class IRCBot(IRCClient): if len(listinfo) == 0: # probably ngircd not supporting LIST >0 return if main.config["ChanKeep"]["Enabled"]: - chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) + chankeep.initialList(self.net, self.num, listinfo) def recheckList(self): allRelays = chankeep.allRelaysActive(self.net) diff --git a/modules/chankeep.py b/modules/chankeep.py index ea09274..402de69 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -4,10 +4,10 @@ from math import ceil from twisted.internet.threads import deferToThread import main -import modules.provision from utils.logging.debug import debug, trace from utils.logging.log import error, log, warn + def getEnabledRelays(net): """ Get a list of enabled relays for a network. @@ -18,6 +18,7 @@ def getEnabledRelays(net): enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] return enabledRelays + def getActiveRelays(net): """ Get a list of active relays for a network. @@ -48,6 +49,7 @@ def allRelaysActive(net): debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})") return relaysActive + def getAverageChanlimit(net): """ Get the average channel limit for a network. @@ -64,6 +66,7 @@ def getAverageChanlimit(net): debug(f"getAverageChanlimit() {net}: {avg_chanlimit}") return avg_chanlimit + def getSumChanlimit(net): """ Get the sum of all channel limits for a network. @@ -78,32 +81,26 @@ def getSumChanlimit(net): total += main.IRCPool[name].chanlimit return total -def getChanFree(net, new): + +def getChanFree(net): """ Get a dictionary with the free channel spaces for each relay, and a channel limit. Example return: ({1: 99}, 100) :param net: network - :param new: list of newly provisioned relays to skip :return: ({relay: channel spaces}, channel limit) """ chanfree = {} - chanlimits = set() for i in getActiveRelays(net): - if i in new: - continue name = net + str(i) if name not in main.IRCPool.keys(): continue if not main.IRCPool[name].isconnected: continue chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels) - chanlimits.add(main.IRCPool[name].chanlimit) - if not len(chanlimits) == 1: - error("Network %s has servers with different CHANLIMIT values" % net) - return False - return (chanfree, chanlimits.pop()) + + return chanfree def getTotalChans(net): @@ -121,7 +118,7 @@ def getTotalChans(net): return total -def emptyChanAllocate(net, flist, new): +def emptyChanAllocate(net, flist): """ Allocate channels to relays. :param net: network @@ -131,64 +128,32 @@ def emptyChanAllocate(net, flist, new): :return: dictionary of {relay: list of channels}""" # Get the free channel spaces for each relay - chanfree = getChanFree(net, new) + chanfree = getChanFree(net) if not chanfree: return # Pretend the newly provisioned relays are already on the network - for i in new: - chanfree[0][i] = chanfree[1] + # for i in new: + # chanfree[0][i] = chanfree[1] allocated = {} - # Copy the list since we're going to mutate it - toalloc = len(flist) - - # Used to correct allocations and provision additional relays - # if the math since the last LIST is a bit wrong - # toalloc:2148 free:{1: 250} chanlimit:250 correction:2147 newlist = list(flist) - if toalloc > sum(chanfree[0].values()): - sum_free = sum(chanfree[0].values()) # 250 - chans_not_covered = toalloc - sum_free # 2148 - 250 = 1898 - relays_needed = chans_not_covered / chanfree[1] # 1898 / 250 = 7.592 - correction = ceil(relays_needed) - if main.config["ChanKeep"]["Provision"]: - debug( - ( - f"emptyChanAllocate() secondary allocation sum_free:{sum_free} " - f"chans_not_covered:{chans_not_covered} relays_needed:{relays_needed} " - f"correction:{correction}" - ) - ) - debug( - ( - f"emptyChanAllocate() not enough free channels: toalloc:{toalloc} " - f"free:{chanfree[0]} chanlimit:{chanfree[1]} correction:{correction}" - ) - ) - warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) - modules.provision.provisionMultipleRelays(net, correction) - return False - else: - # We don't have enough spaces and we can't add any. - # Let's do the best we can in the circumstances. - debug(f"emptyChanAllocate() cannot create additional relays for {net}") - debug(f"emptyChanAllocate() {chans_not_covered} channels cannot be covered") - newlist = newlist[:sum_free] - debug(f"emptyChanAllocate() flist truncated to {sum_free}, length nis now {len(flist)}") - trace(f"emptyChanAllocate() best effort allocation: {flist}") - for i in chanfree[0].keys(): - for x in range(chanfree[0][i]): - if not len(newlist): + sum_free = sum(chanfree.values()) # 250 + trunc_list = newlist[:sum_free] + debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}") + + for i in chanfree.keys(): + for x in range(chanfree[i]): + if not len(trunc_list): break if i in allocated.keys(): - allocated[i].append(newlist.pop()) + allocated[i].append(trunc_list.pop()) else: - allocated[i] = [newlist.pop()] + allocated[i] = [trunc_list.pop()] return allocated -def populateChans(net, clist, new): +def populateChans(net, clist): """ Populate channels on relays. Stores channels to join in a list in main.TempChan[net][num] @@ -196,7 +161,7 @@ def populateChans(net, clist, new): :param clist: list of channels to join :param new: list of newly provisioned relays to account for""" # divided = array_split(clist, relay) - allocated = emptyChanAllocate(net, clist, new) + allocated = emptyChanAllocate(net, clist) if not allocated: return for i in allocated.keys(): @@ -242,7 +207,7 @@ def minifyChans(net, listinfo): return listinfo -def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): +def keepChannels(net, listinfo, mean, sigrelay, relay): """ Minify channels, determine whether we can cover all the channels on the network, or need to use 'significant' mode. @@ -263,39 +228,40 @@ def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit): coverAll = True elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels coverAll = False - if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: - error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) - return + # if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: + # error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) + # return num_instances = len(getActiveRelays(net)) - debug(f"keepChannels() {net} instances:{num_instances} chanlimit:{chanlimit}") + debug(f"keepChannels() {net} instances:{num_instances}") chan_slots_used = getTotalChans(net) debug(f"keepChannels() slots_used:{chan_slots_used}") - max_chans = (chanlimit * num_instances) - chan_slots_used + # max_chans = (chanlimit * num_instances) - chan_slots_used + max_chans = getSumChanlimit(net) - chan_slots_used debug(f"keepChannels() max_chans:{max_chans}") if coverAll: - needed = relay - len(getActiveRelays(net)) - if needed: - debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") - newNums = modules.provision.provisionMultipleRelays(net, needed) - else: - newNums = [] + # needed = relay - len(getActiveRelays(net)) + # if needed: + # debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}") + # newNums = modules.provision.provisionMultipleRelays(net, needed) + # else: + # newNums = [] flist = [i[0] for i in listinfo] chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") - populateChans(net, chosen, newNums) + populateChans(net, chosen) else: - needed = sigrelay - len(getActiveRelays(net)) - if needed: - debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}") - newNums = modules.provision.provisionMultipleRelays(net, needed) - else: - newNums = [] + # needed = sigrelay - len(getActiveRelays(net)) + # if needed: + # debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}") + # newNums = modules.provision.provisionMultipleRelays(net, needed) + # else: + # newNums = [] siglist = [i[0] for i in listinfo if int(i[1]) > mean] - chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") - populateChans(net, chosen, newNums) + populateChans(net, chosen) notifyJoin(net) @@ -308,7 +274,7 @@ def joinSingle(net, channel): :return: relay number that joined the channel :rtype: int """ - eca = emptyChanAllocate(net, [channel], []) + eca = emptyChanAllocate(net, [channel]) if not eca: return False if not len(eca.keys()) == 1: @@ -354,7 +320,7 @@ def nukeNetwork(net): # deferToThread(_nukeNetwork, net) -def _initialList(net, num, listinfo, chanlimit): +def _initialList(net, num, listinfo): """ Called when a relay receives a full LIST response. Run statistics to determine how many channels are significant. @@ -390,8 +356,9 @@ def _initialList(net, num, listinfo, chanlimit): insiglength += 1 insigcumul += int(i[1]) - sigrelay = ceil(siglength / chanlimit) - relay = ceil(listlength / chanlimit) + avg_chanlimit = getAverageChanlimit(net) + sigrelay = ceil(siglength / avg_chanlimit) + relay = ceil(listlength / avg_chanlimit) abase = "analytics.list.%s" % net main.g.delete(abase) @@ -409,12 +376,12 @@ def _initialList(net, num, listinfo, chanlimit): p.hset(abase, "small_chan_cumul_mem", insigcumul) p.hset(abase, "relays_for_all_chans", relay) p.hset(abase, "relays_for_big_chans", sigrelay) - p.hset(abase, "relays_for_small_chans", ceil(insiglength / chanlimit)) + p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit)) debug( ( f"_initialList() net:{net} num:{num} listlength:{listlength} " f"mean:{mean} siglength:{siglength} insiglength:{insiglength} " - f"sigrelay:{sigrelay} relay:{relay} chanlimit:{chanlimit}" + f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}" ) ) @@ -427,7 +394,7 @@ def _initialList(net, num, listinfo, chanlimit): p.execute() debug("List parsing completed on %s" % net) - keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit) + keepChannels(net, listinfo, mean, sigrelay, relay) # return (listinfo, mean, sigrelay, relay) @@ -453,9 +420,9 @@ def getListInfo(net): return convert(info) -def initialList(net, num, listinfo, chanlimit): +def initialList(net, num, listinfo): """ Run _initialList in a thread. See above docstring. """ - deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) + deferToThread(_initialList, net, num, deepcopy(listinfo)) diff --git a/modules/provision.py b/modules/provision.py index ca62585..f517236 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -52,6 +52,7 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): commands["nickserv"].append("Set %s" % password) inst = modules.regproc.selectInst(network) if "setmode" in inst.keys(): + # perform is loaded above # commands["status"].append("LoadMod perform") commands["perform"] = ["add mode %nick% +" + inst["setmode"]] deliverRelayCommands(num, commands, user=user + "/" + network) diff --git a/tests/test_chankeep.py b/tests/test_chankeep.py index 13c8918..38a0d04 100644 --- a/tests/test_chankeep.py +++ b/tests/test_chankeep.py @@ -16,6 +16,8 @@ class TestChanKeep(TestCase): chankeep.main.g = MagicMock() chankeep.main.g.pipeline = MagicMock() chankeep.main.config["ChanKeep"]["Provision"] = False + chankeep.getAverageChanlimit = MagicMock() + chankeep.getAverageChanlimit.return_value = self.chanlimit self.listinfo = self.generate_listinfo() self.chan_name_list = [x[0] for x in self.listinfo] @@ -89,17 +91,17 @@ class TestChanKeep(TestCase): @patch("modules.chankeep.keepChannels") def test__initialList(self, keepchannels): - chankeep._initialList(self.net, self.num, self.listinfo, self.chanlimit) - net, passed_list, mean, sigrelay, relay, chanlimit = keepchannels.call_args_list[0][0] + chankeep._initialList(self.net, self.num, self.listinfo) + net, passed_list, mean, sigrelay, relay = keepchannels.call_args_list[0][0] self.assertEqual(net, self.net) self.assertEqual(passed_list, self.listinfo) - self.assertEqual(chanlimit, self.chanlimit) + # self.assertEqual(chanlimit, self.chanlimit) # print(net, mean, sigrelay, relay) @patch("modules.chankeep.getChanFree") def test_empty_chan_allocate(self, getchanfree): getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free - eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, []) + eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list) self.assertEqual(len(eca), 1) num = list(eca.keys())[0] chans = eca[list(eca.keys())[0]] @@ -107,7 +109,7 @@ class TestChanKeep(TestCase): self.assertCountEqual(chans, self.chan_name_list) getchanfree.return_value = ({1: 100}, 10) - eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, []) + eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list) self.assertEqual(len(eca), 1) num = list(eca.keys())[0] chans = eca[list(eca.keys())[0]] From 9470f0d0d951d3d12c7eeca51834f860ee60153e Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 20:36:51 +0100 Subject: [PATCH 329/394] Implement updating registration via API --- api/views.py | 43 ++++++++++++++++++++++++++++++++++++++++++- modules/regproc.py | 23 +++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index a19a4a1..2137f52 100644 --- a/api/views.py +++ b/api/views.py @@ -6,7 +6,7 @@ from klein import Klein from twisted.web.server import Request import main -from modules import chankeep, helpers, provision, userinfo +from modules import chankeep, helpers, provision, regproc, userinfo from modules.network import Network from utils.logging.log import warn @@ -207,6 +207,7 @@ class API(object): network["port"] = inst.port network["security"] = inst.security network["relays"] = len(inst.relays) + network["chanlimit"] = inst.chanlimit network["channels"] = userinfo.getTotalChanNum(net) network["records"] = userinfo.getNumWhoEntries(net) return dumps(network) @@ -247,6 +248,14 @@ class API(object): if not port.isdigit(): return dumps({"success": False, "reason": "invalid port: not a number."}) port = int(port) + elif item == "chanlimit": + chanlimit = data[item][0] + if chanlimit == "None": + chanlimit = None + elif not chanlimit.isdigit(): + return dumps({"success": False, "reason": "invalid chanlimit: not a number."}) + else: + chanlimit = int(chanlimit) elif item == "security": security = data[item][0] if security not in ["ssl", "plain"]: @@ -256,6 +265,7 @@ class API(object): inst.last = last inst.port = port inst.security = security + inst.chanlimit = chanlimit main.saveConf("network") return dumps({"success": True}) @@ -535,3 +545,34 @@ class API(object): if name not in main.IRCPool.keys(): return dumps({"success": False, "reason": f"relay {num} not on {net}"}) return dumps({"nickname": main.IRCPool[name].nickname}) + + @app.route("/irc/reg//", methods=["GET"]) + @login_required + def irc_get_unreg_net(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + unreg = regproc.get_unregistered_relays(net) + return dumps({"success": True, "unreg": unreg}) + + @app.route("/irc/reg/", methods=["GET"]) + @login_required + def irc_get_unreg(self, request): + unreg = regproc.get_unregistered_relays() + return dumps({"success": True, "unreg": unreg}) + + @app.route("/irc/reg/", methods=["PUT"]) + @login_required + def irc_confirm_accounts(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + for item, token in data.items(): + if "|" not in item: + return dumps({"success": False, "reason": f"malformed item: {item}"}) + spl = item.split("|") + if not len(spl) == 2: + return dumps({"success": False, "reason": f"malformed item: {item}"}) + net, num = spl + regproc.confirmAccount(net, num, token) + return dumps({"success": True}) diff --git a/modules/regproc.py b/modules/regproc.py index 3cc35bc..facf120 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -120,6 +120,29 @@ def enableAuthentication(net, num): confirmRegistration(net, num) +def get_unregistered_relays(net=None): + """ + Get a dict of unregistereed relays, either globally or + for a network. + Returns: + {"net": [["nick1", 1], ["nick2", 2], ...]} + """ + unreg = {} + if net: + nets = [net] + else: + nets = main.network.keys() + for i in nets: + for num in main.network[i].relays.keys(): + if not main.network[i].relays[num]["registered"]: + nick = main.alias[num]["nick"] + if net in unreg: + unreg[i].append([nick, num]) + else: + unreg[i] = [[nick, num]] + return unreg + + def registerTest(c): sinst = substitute(c["net"], c["num"]) name = c["net"] + str(c["num"]) From 28c1a33615d54008847c19f07b42e606163316c7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 20:37:21 +0100 Subject: [PATCH 330/394] Implement configurable chanlimit and add more fields about LIST output to Redis --- core/bot.py | 7 +++++++ modules/chankeep.py | 10 +++++++++- tests/test_chankeep.py | 1 - 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/core/bot.py b/core/bot.py index e6b1009..c10f0ac 100644 --- a/core/bot.py +++ b/core/bot.py @@ -61,6 +61,7 @@ class IRCBot(IRCClient): self.name = net + str(num) alias = main.alias[num] relay = main.network[self.net].relays[num] + self.netinst = main.network[self.net] self.nickname = alias["nick"] self.realname = alias["realname"] self.username = alias["nick"].lower() + "/" + relay["net"] @@ -518,6 +519,12 @@ class IRCBot(IRCClient): warn("Invalid chanlimit: %s" % chanlimit) if self.chanlimit == 0: self.chanlimit = 200 # don't take the piss if it's not limited + net_inst_chanlimit = self.netinst.chanlimit + if net_inst_chanlimit: + if self.chanlimit > net_inst_chanlimit: + self.chanlimit = net_inst_chanlimit + warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}") + if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation self.recheckList() diff --git a/modules/chankeep.py b/modules/chankeep.py index 402de69..edbdf96 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -237,6 +237,8 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): debug(f"keepChannels() slots_used:{chan_slots_used}") # max_chans = (chanlimit * num_instances) - chan_slots_used max_chans = getSumChanlimit(net) - chan_slots_used + if max_chans < 0: + max_chans = 0 debug(f"keepChannels() max_chans:{max_chans}") if coverAll: # needed = relay - len(getActiveRelays(net)) @@ -259,7 +261,7 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = [] siglist = [i[0] for i in listinfo if int(i[1]) > mean] chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] - debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") + debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") populateChans(net, chosen) notifyJoin(net) @@ -360,6 +362,10 @@ def _initialList(net, num, listinfo): sigrelay = ceil(siglength / avg_chanlimit) relay = ceil(listlength / avg_chanlimit) + cur_relays = len(getActiveRelays(net)) + sig_relays_missing = sigrelay - cur_relays + all_relays_missing = relay - cur_relays + abase = "analytics.list.%s" % net main.g.delete(abase) p = main.g.pipeline() @@ -377,6 +383,8 @@ def _initialList(net, num, listinfo): p.hset(abase, "relays_for_all_chans", relay) p.hset(abase, "relays_for_big_chans", sigrelay) p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit)) + p.hset(abase, "sig_relays_missing", sig_relays_missing) + p.hset(abase, "all_relays_missing", all_relays_missing) debug( ( f"_initialList() net:{net} num:{num} listlength:{listlength} " diff --git a/tests/test_chankeep.py b/tests/test_chankeep.py index 38a0d04..491a8ce 100644 --- a/tests/test_chankeep.py +++ b/tests/test_chankeep.py @@ -86,7 +86,6 @@ class TestChanKeep(TestCase): # list_insig = [x for x in listinfo_num if x < mean] list_sig = [x for x in listinfo if x[1] > mean] chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans] - print("CHOSEN", chosen) self.assertEqual(len(chosen), 5) @patch("modules.chankeep.keepChannels") From 92df4fb9a316a198f20ccbac16b8132daf899d41 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 20:51:31 +0100 Subject: [PATCH 331/394] Implement API endpoint for provisioning relays --- api/views.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/views.py b/api/views.py index 2137f52..2e38a2a 100644 --- a/api/views.py +++ b/api/views.py @@ -395,6 +395,20 @@ class API(object): main.saveConf("network") return dumps({"success": True}) + @app.route("/irc/network///provision/", methods=["POST"]) + @login_required + def irc_network_relay_provision(self, request, net, num): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + net_inst = main.network[net] + if num not in net_inst.relays: + return dumps({"success": False, "reason": "network does not have this relay."}) + provision.provisionRelay(num, net) + return dumps({"success": True}) + @app.route("/irc/network//channels/", methods=["POST"]) @login_required def irc_network_channels(self, request, net): From ad7a5cfe49956839168df6ff4cb8df48e2940757 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 20:55:36 +0100 Subject: [PATCH 332/394] Check token before attempting to confirm --- api/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 2e38a2a..e8fc7a1 100644 --- a/api/views.py +++ b/api/views.py @@ -588,5 +588,6 @@ class API(object): if not len(spl) == 2: return dumps({"success": False, "reason": f"malformed item: {item}"}) net, num = spl - regproc.confirmAccount(net, num, token) + if token: + regproc.confirmAccount(net, num, token) return dumps({"success": True}) From 5c95f35c61fc7bf68b243ec3852f5129c6933df7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 21:22:43 +0100 Subject: [PATCH 333/394] Return chanlimit for each relay --- api/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/views.py b/api/views.py index e8fc7a1..35b17b6 100644 --- a/api/views.py +++ b/api/views.py @@ -321,10 +321,12 @@ class API(object): to_append["chans"] = len(main.IRCPool[name].channels) to_append["nick"] = main.IRCPool[name].nickname to_append["conn"] = main.IRCPool[name].isconnected + to_append["limit"] = main.IRCPool[name].chanlimit else: to_append["chans"] = 0 to_append["nick"] = None to_append["conn"] = False + to_append["limit"] = None relays.append(to_append) return dumps({"relays": relays}) From 16133fb7b4f126dbee177bfa7aebb9e13e7d4ef8 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 21:40:53 +0100 Subject: [PATCH 334/394] Fully make use of ECA for multiple channels --- modules/chankeep.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index edbdf96..263c3b5 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -276,7 +276,11 @@ def joinSingle(net, channel): :return: relay number that joined the channel :rtype: int """ - eca = emptyChanAllocate(net, [channel]) + if "," in channel: + channels = channel.split(",") + eca = emptyChanAllocate(net, channels) + else: + eca = emptyChanAllocate(net, [channel]) if not eca: return False if not len(eca.keys()) == 1: From fced2b7d751e64a1dedc1acf212a4f5b903a8b11 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 21:54:14 +0100 Subject: [PATCH 335/394] Use ChanKeep system for joining channels with joinSingle --- modules/chankeep.py | 55 +++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 263c3b5..0ac9bf8 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -138,8 +138,9 @@ def emptyChanAllocate(net, flist): allocated = {} newlist = list(flist) - sum_free = sum(chanfree.values()) # 250 - trunc_list = newlist[:sum_free] + chan_slots_used = getTotalChans(net) + max_chans = getSumChanlimit(net) - chan_slots_used + trunc_list = newlist[:max_chans] debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}") for i in chanfree.keys(): @@ -162,6 +163,7 @@ def populateChans(net, clist): :param new: list of newly provisioned relays to account for""" # divided = array_split(clist, relay) allocated = emptyChanAllocate(net, clist) + trace(f"populateChans() allocated:{allocated}") if not allocated: return for i in allocated.keys(): @@ -169,6 +171,7 @@ def populateChans(net, clist): main.TempChan[net][i] = allocated[i] else: main.TempChan[net] = {i: allocated[i]} + trace(f"populateChans() TempChan {net}{i}: {allocated[i]}") def notifyJoin(net): @@ -180,10 +183,11 @@ def notifyJoin(net): for i in getActiveRelays(net): name = net + str(i) if name in main.IRCPool.keys(): + trace(f"notifyJoin() {name}") main.IRCPool[name].checkChannels() -def minifyChans(net, listinfo): +def minifyChans(net, listinfo, as_list=False): """ Remove channels from listinfo that are already covered by a relay. :param net: network @@ -192,19 +196,32 @@ def minifyChans(net, listinfo): :return: list of channels with joined channels removed :rtype: list of [channel, num_users] """ + # We want to make this reusable for joining a bunch of channels. + if as_list: + channel_list = listinfo + if not allRelaysActive(net): error("All relays for %s are not active, cannot minify list" % net) return False for i in getActiveRelays(net): name = net + str(i) for x in main.IRCPool[name].channels: - for y in listinfo: - if y[0] == x: - listinfo.remove(y) - if not listinfo: - log("We're on all the channels we want to be on, dropping LIST") - return False - return listinfo + if as_list: + for y in channel_list: + if y == x: + channel_list.remove(y) + else: + for y in listinfo: + if y[0] == x: + listinfo.remove(y) + if not as_list: + if not listinfo: + log("We're on all the channels we want to be on, dropping LIST") + return False + if as_list: + return channel_list + else: + return listinfo def keepChannels(net, listinfo, mean, sigrelay, relay): @@ -278,19 +295,13 @@ def joinSingle(net, channel): """ if "," in channel: channels = channel.split(",") - eca = emptyChanAllocate(net, channels) + channels = minifyChans(net, channels, as_list=True) + else: - eca = emptyChanAllocate(net, [channel]) - if not eca: - return False - if not len(eca.keys()) == 1: - return False - num = list(eca.keys())[0] - name = f"{net}{num}" - if name not in main.IRCPool: - return False - main.IRCPool[name].join(channel) - return num + channels = [channel] + populateChans(net, channels) + notifyJoin(net) + return True def partSingle(net, channel): From 5db659b9afe8457c8a28e69a70cbb0bfcc2f3c0a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 22:15:50 +0100 Subject: [PATCH 336/394] Filter AUTH channel (OFTC fix) --- core/bot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/bot.py b/core/bot.py index c10f0ac..5d40d6e 100644 --- a/core/bot.py +++ b/core/bot.py @@ -207,6 +207,9 @@ class IRCBot(IRCClient): # self.event(**castDup) # Don't call self.event for this one because queries are not events on a # channel, but we still want to see them + if cast["channel"] == "AUTH": + cast["type"] = "conn" + cast["mtype"] = cast["type"] # TODO: better way to do this # as we changed the types above, check again From 047e9148aaba99f50e344915b74f9f1e1425c364 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 22:25:29 +0100 Subject: [PATCH 337/394] Implement API endpoint to enable authentication --- api/views.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/views.py b/api/views.py index 35b17b6..c0a734c 100644 --- a/api/views.py +++ b/api/views.py @@ -593,3 +593,16 @@ class API(object): if token: regproc.confirmAccount(net, num, token) return dumps({"success": True}) + + @app.route("/irc/network///auth/", methods=["POST"]) + @login_required + def irc_enable_auth(self, request, net, num): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) + if num not in main.network[net].relays.keys(): + return dumps({"success": False, "reason": f"no relay {num} on {net}"}) + regproc.enableAuthentication(net, num) + return dumps({"success": True}) From 406b3d77f43fce0e2d2ff0bad18f23a630328be2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 22:36:18 +0100 Subject: [PATCH 338/394] Add helper to get all active relays --- modules/helpers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/helpers.py b/modules/helpers.py index cd0c510..3c4efdf 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -33,3 +33,19 @@ def is_first_relay(net, num): if not first_relay: return False return first_relay.num == num + + +def get_active_relays(net): + """ + Get all active instances for the network. + :param net: the network + :return: list of active instances + :rtype: list of IRCPool instances + """ + active_nums = chankeep.getActiveRelays(net) + active_insts = [] + for num in active_nums: + name = net + str(num) + if name in main.IRCPool.keys(): + active_insts.append(main.IRCPool[name]) + return active_insts From abeba6bc0606b22d1f09c44a545588cdb60eb050 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 22:36:52 +0100 Subject: [PATCH 339/394] Update CHANLIMIT on all instances when set via API --- api/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/views.py b/api/views.py index c0a734c..d7b5805 100644 --- a/api/views.py +++ b/api/views.py @@ -256,6 +256,9 @@ class API(object): return dumps({"success": False, "reason": "invalid chanlimit: not a number."}) else: chanlimit = int(chanlimit) + online_relays = helpers.get_active_relays(net) + for inst in online_relays: + inst.chanlimit = chanlimit elif item == "security": security = data[item][0] if security not in ["ssl", "plain"]: From 4c91b6ad2ce0698e96cc799ae0b99e01323ad418 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 22:46:10 +0100 Subject: [PATCH 340/394] Filter queries more carefully --- core/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index 5d40d6e..df3345a 100644 --- a/core/bot.py +++ b/core/bot.py @@ -201,7 +201,7 @@ class IRCBot(IRCClient): castDup["type"] = "highlight" self.event(**castDup) else: - if cast["channel"].lower() == self.nickname.lower(): + if cast["channel"].lower() == self.nickname.lower() or not cast["channel"].startswith("#"): cast["mtype"] = cast["type"] cast["type"] = "query" # self.event(**castDup) From d9d3faf860524477de370334e6a8ef5c70c9fcdd Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 23:14:17 +0100 Subject: [PATCH 341/394] Extra debugging for get_first_relay --- modules/helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/helpers.py b/modules/helpers.py index 3c4efdf..3f5c145 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -1,6 +1,6 @@ import main from modules import chankeep - +from util.logging.debug import debug def get_first_relay(net): """ @@ -11,13 +11,16 @@ def get_first_relay(net): """ cur_relay = 0 max_relay = len(main.network[net].relays.keys()) + 1 + debug(f"get_first_relay() {net}: max_relay:{max_relay}") activeRelays = chankeep.getActiveRelays(net) + debug(f"get_first_relay() {net}: activeRelays:{activeRelays}") while cur_relay != max_relay: cur_relay += 1 if cur_relay not in activeRelays: continue name = net + str(cur_relay) if name in main.IRCPool.keys(): + debug(f"get_first_relay() {net}: found relay {name}") return main.IRCPool[name] return None From 5f33ba7f1d900599302eae15a96c4952305c7841 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 23:14:51 +0100 Subject: [PATCH 342/394] Fix typo in module name --- modules/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/helpers.py b/modules/helpers.py index 3f5c145..daff826 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -1,6 +1,6 @@ import main from modules import chankeep -from util.logging.debug import debug +from utils.logging.debug import debug def get_first_relay(net): """ From 3ca5a3452c211b7b1a16532c2955e5324f75735b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 23:17:26 +0100 Subject: [PATCH 343/394] Extra debugging for getting active relays --- modules/chankeep.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index 0ac9bf8..5816c68 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -16,6 +16,7 @@ def getEnabledRelays(net): :return: list of enabled relay numbers """ enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] + debug(f"getEnabledRelays() {net}: {enabledRelays}") return enabledRelays @@ -33,6 +34,7 @@ def getActiveRelays(net): if name in main.IRCPool.keys(): if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: activeRelays.append(i) + debug(f"getActiveRelays() {net}: {activeRelays}") return activeRelays From 2a3c9f80a3f0d345e5f1b2b4b726e7de0598f8f5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 23:18:56 +0100 Subject: [PATCH 344/394] Add even more debugging --- modules/chankeep.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index 5816c68..6c61466 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -32,6 +32,7 @@ def getActiveRelays(net): for i in enabledRelays: name = net + str(i) if name in main.IRCPool.keys(): + debug(f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} conn:{main.IRCPool[name].isconnected}") if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: activeRelays.append(i) debug(f"getActiveRelays() {net}: {activeRelays}") From a82355b66017256f5bc6deab11573ee7983c4973 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 23:36:39 +0100 Subject: [PATCH 345/394] Add more information to relay API return --- api/views.py | 2 ++ modules/chankeep.py | 7 ++++++- modules/helpers.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index d7b5805..0ef1f5a 100644 --- a/api/views.py +++ b/api/views.py @@ -325,11 +325,13 @@ class API(object): to_append["nick"] = main.IRCPool[name].nickname to_append["conn"] = main.IRCPool[name].isconnected to_append["limit"] = main.IRCPool[name].chanlimit + to_append["auth"] = main.IRCPool[name].authenticated else: to_append["chans"] = 0 to_append["nick"] = None to_append["conn"] = False to_append["limit"] = None + to_append["auth"] = None relays.append(to_append) return dumps({"relays": relays}) diff --git a/modules/chankeep.py b/modules/chankeep.py index 6c61466..68eab78 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -32,7 +32,12 @@ def getActiveRelays(net): for i in enabledRelays: name = net + str(i) if name in main.IRCPool.keys(): - debug(f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} conn:{main.IRCPool[name].isconnected}") + debug( + ( + f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} " + f"conn:{main.IRCPool[name].isconnected}" + ) + ) if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: activeRelays.append(i) debug(f"getActiveRelays() {net}: {activeRelays}") diff --git a/modules/helpers.py b/modules/helpers.py index daff826..33b484f 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -2,6 +2,7 @@ import main from modules import chankeep from utils.logging.debug import debug + def get_first_relay(net): """ Get the first relay in the network. From a42c6be1b7a8e226902e0384ac4b92ff5552553d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 13 Aug 2022 23:38:13 +0100 Subject: [PATCH 346/394] LBYL --- modules/regproc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/regproc.py b/modules/regproc.py index facf120..6037ddd 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -168,9 +168,10 @@ def registerTest(c): confirmRegistration(c["net"], c["num"]) return if sinst["checktype"] == "msg": - if sinst["checkmsg"] in c["msg"]: - confirmRegistration(c["net"], c["num"]) - return + if "msg" in c.keys(): + if sinst["checkmsg"] in c["msg"]: + confirmRegistration(c["net"], c["num"]) + return elif sinst["checktype"] == "mode": if c["type"] == "self": if c["mtype"] == "mode": From a204be25c52d68753d6357ee05a7b815df6902c6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 00:01:14 +0100 Subject: [PATCH 347/394] Make channel deletion endpoint accept JSON --- api/views.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 0ef1f5a..9f92aaf 100644 --- a/api/views.py +++ b/api/views.py @@ -435,11 +435,19 @@ class API(object): return dumps({"channels": channels}) - @app.route("/irc/network//channel//", methods=["DELETE"]) + @app.route("/irc/network//channel/", methods=["DELETE"]) @login_required - def irc_network_channel_part(self, request, net, channel): + def irc_network_channel_part(self, request, net): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "channel" not in data: + return dumps({"success": False, "reason": "no channel specified."}) + channel = data["channel"] if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) + print("ABOUT TO PART", channel) parted = chankeep.partSingle(net, channel) if not parted: dumps({"success": False, "reason": "no channels matched."}) From 9b14979f2923048536ab85a2ebcfb9b8d698ef80 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 09:25:01 +0100 Subject: [PATCH 348/394] Use JSON for joining channels and don't shadow auth variable when getting network info --- api/views.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/api/views.py b/api/views.py index 9f92aaf..788a0dc 100644 --- a/api/views.py +++ b/api/views.py @@ -325,15 +325,14 @@ class API(object): to_append["nick"] = main.IRCPool[name].nickname to_append["conn"] = main.IRCPool[name].isconnected to_append["limit"] = main.IRCPool[name].chanlimit - to_append["auth"] = main.IRCPool[name].authenticated + to_append["authed"] = main.IRCPool[name].authenticated else: to_append["chans"] = 0 to_append["nick"] = None to_append["conn"] = False to_append["limit"] = None - to_append["auth"] = None + to_append["authed"] = None relays.append(to_append) - return dumps({"relays": relays}) @app.route("/irc/network///", methods=["POST"]) @@ -435,9 +434,10 @@ class API(object): return dumps({"channels": channels}) + # API version @app.route("/irc/network//channel/", methods=["DELETE"]) @login_required - def irc_network_channel_part(self, request, net): + def irc_network_channel_part_api(self, request, net): try: data = loads(request.content.read()) except JSONDecodeError: @@ -447,12 +447,29 @@ class API(object): channel = data["channel"] if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) - print("ABOUT TO PART", channel) parted = chankeep.partSingle(net, channel) if not parted: dumps({"success": False, "reason": "no channels matched."}) return dumps({"success": True, "relays": parted}) + # API version + @app.route("/irc/network//channel/", methods=["PUT"]) + @login_required + def irc_network_channel_join_api(self, request, net): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "channel" not in data: + return dumps({"success": False, "reason": "no channel specified."}) + channel = data["channel"] + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + joined = chankeep.joinSingle(net, channel) + if not joined: + return dumps({"success": False, "reason": "could not allocate channel to relay."}) + return dumps({"success": True, "relays": joined}) + @app.route("/irc/network//channel//", methods=["PUT"]) @login_required def irc_network_channel_join(self, request, net, channel): From feecf48b9b52a0a2288dcd868cb77cd8cc21de45 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 09:25:54 +0100 Subject: [PATCH 349/394] Add debug statements and only check if network is connected when parting channels --- core/bot.py | 8 +++++--- modules/chankeep.py | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/core/bot.py b/core/bot.py index df3345a..da355ee 100644 --- a/core/bot.py +++ b/core/bot.py @@ -620,22 +620,24 @@ class IRCBot(IRCClient): return sinst = regproc.substitute(self.net, self.num) if not sinst: - error(f"Registration ping failed for {self.net} - {self.num}") + error(f"regPing() {self.net}: registration ping failed for {self.num}") return if self._negativePass is not True: if negativepass is False: self._negativePass = False + debug(f"regPing() {self.net}: negativepass is False for {self.num}") return if negativepass is True: if self._negativePass is None: self._negativePass = True - debug("Positive registration check - %s - %i" % (self.net, self.num)) + debug(f"regPing() {self.net}: positive registration check - {self.num}") if sinst["ping"]: debug("Sending ping - %s - %i" % (self.net, self.num)) self.msg(sinst["entity"], sinst["pingmsg"]) + debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}") return else: - debug("Negative registration for %s - %i" % (self.net, self.num)) + debug("regPing() {self.net}: negative registration check - {self.num}") return if sinst["check"]: diff --git a/modules/chankeep.py b/modules/chankeep.py index 68eab78..beef873 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -20,12 +20,30 @@ def getEnabledRelays(net): return enabledRelays +def getConnectedRelays(net): + """ + Get a list of connected relays for a network. + :param net: network + :rtype: list of int + :return: list of relay numbers + """ + enabledRelays = getEnabledRelays(net) + connectedRelays = [] + for i in enabledRelays: + name = net + str(i) + if name in main.IRCPool.keys(): + if main.IRCPool[name].isconnected: + connectedRelays.append(i) + debug(f"getConnectedRelays() {net}: {connectedRelays}") + return connectedRelays + + def getActiveRelays(net): """ Get a list of active relays for a network. :param net: network :rtype: list of int - :return: list of getEnabledRelays relay numbers + :return: list of relay numbers """ enabledRelays = getEnabledRelays(net) activeRelays = [] @@ -321,7 +339,7 @@ def partSingle(net, channel): :rtype: list of str """ parted = [] - for i in getActiveRelays(net): + for i in getConnectedRelays(net): name = f"{net}{i}" if name in main.IRCPool.keys(): if channel in main.IRCPool[name].channels: From 39059084ef95418eb6174b6e2bea253328090ebe Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 10:58:28 +0100 Subject: [PATCH 350/394] Add allRelaysActive output to network info --- api/views.py | 1 + core/bot.py | 2 +- modules/chankeep.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 788a0dc..842c186 100644 --- a/api/views.py +++ b/api/views.py @@ -187,6 +187,7 @@ class API(object): networks = {} for net in main.network.keys(): networks[net] = { + "active": chankeep.allRelaysActive(net), "relays": len(main.network[net].relays), "channels": userinfo.getTotalChanNum(net), "records": userinfo.getNumWhoEntries(net), diff --git a/core/bot.py b/core/bot.py index da355ee..cab516b 100644 --- a/core/bot.py +++ b/core/bot.py @@ -489,7 +489,7 @@ class IRCBot(IRCClient): if first_relay: if first_relay.wantList is True: first_relay.list(nocheck=True) - debug("recheckList() asking for a list for {self.net} after final relay {self.num} connected") + debug(f"recheckList() asking for a list for {self.net} after final relay {self.num} connected") else: debug(f"recheckList() first relay wantList is False for {self.net} ({first_relay.num})") # name = self.net + "1" diff --git a/modules/chankeep.py b/modules/chankeep.py index beef873..ef5a52d 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -62,6 +62,18 @@ def getActiveRelays(net): return activeRelays +def relayIsActive(net, num): + """ + Check if a relay is active. + :param net: network + :param num: relay number + :rtype: bool + :return: True if relay is active, False otherwise + """ + activeRelays = getActiveRelays(net) + return num in activeRelays + + def allRelaysActive(net): """ Check if all enabled relays are active and authenticated. From 0b20a05b1909072534d37842e9b7fe39d9c8f2fb Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 11:41:29 +0100 Subject: [PATCH 351/394] More debugging for reg tests and getstr command --- commands/getstr.py | 33 +++++++++++++++++++++++++++++++++ conf/templates/help.json | 3 ++- core/bot.py | 13 ++++++++++--- modules/chankeep.py | 20 ++++++++++---------- modules/helpers.py | 6 +++--- modules/regproc.py | 10 ++++++++-- 6 files changed, 66 insertions(+), 19 deletions(-) create mode 100644 commands/getstr.py diff --git a/commands/getstr.py b/commands/getstr.py new file mode 100644 index 0000000..205088e --- /dev/null +++ b/commands/getstr.py @@ -0,0 +1,33 @@ +import main + +from utils.get import getRelay + +class GetstrCommand: + def __init__(self, *args): + self.getstr(*args) + + def getstr(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 3: + net = spl[1] + num = spl[2] + if not net in main.network.keys(): + failure("Network does not exist: %s" % net) + return + if not num.isdigit(): + failure("Must be a number, not %s" % num) + return + num = int(num) + alias = main.alias[num]["nick"].lower() + host, port = getRelay(num) + password = main.config["Relay"]["Password"] + connstr = f"/connect -ssl {host} {port}" + authstr = f"/quote PASS {alias}/{net}:{password}" + obj.send(connstr) + obj.send(authstr) + + else: + incUsage("getstr") + return + else: + incUsage(None) diff --git a/conf/templates/help.json b/conf/templates/help.json index 2d40b4b..3f79c9a 100644 --- a/conf/templates/help.json +++ b/conf/templates/help.json @@ -33,5 +33,6 @@ "authcheck": "authcheck []", "recheckauth": "recheckauth []", "blacklist": "blacklist ", - "email": "email [(domain)|] []" + "email": "email [(domain)|] []", + "getstr": "getstr " } diff --git a/core/bot.py b/core/bot.py index cab516b..36fc7c4 100644 --- a/core/bot.py +++ b/core/bot.py @@ -480,6 +480,8 @@ class IRCBot(IRCClient): chankeep.initialList(self.net, self.num, listinfo) def recheckList(self): + if not main.config["ChanKeep"]["Enabled"]: + return allRelays = chankeep.allRelaysActive(self.net) debug(f"recheckList() all relays for {self.net} {allRelays}") if allRelays: @@ -526,10 +528,11 @@ class IRCBot(IRCClient): if net_inst_chanlimit: if self.chanlimit > net_inst_chanlimit: self.chanlimit = net_inst_chanlimit - warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}") + # warn(f"Chanlimit on {self.net} too high, setting to {self.chanlimit}") if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation - self.recheckList() + if main.config["ChanKeep"]["Enabled"]: + self.recheckList() def seed_prefix(self, prefix): prefix = prefix.replace(")", "") @@ -618,6 +621,8 @@ class IRCBot(IRCClient): def regPing(self, negativepass=None): if self.authenticated: return + if not regproc.needToRegister(self.net): + self.authenticated = True sinst = regproc.substitute(self.net, self.num) if not sinst: error(f"regPing() {self.net}: registration ping failed for {self.num}") @@ -630,7 +635,7 @@ class IRCBot(IRCClient): if negativepass is True: if self._negativePass is None: self._negativePass = True - debug(f"regPing() {self.net}: positive registration check - {self.num}") + debug(f"regPing() {self.net}: negpass:True positive registration check - {self.num}") if sinst["ping"]: debug("Sending ping - %s - %i" % (self.net, self.num)) self.msg(sinst["entity"], sinst["pingmsg"]) @@ -644,11 +649,13 @@ class IRCBot(IRCClient): if sinst["negative"]: self._negativePass = None self.msg(sinst["entity"], sinst["negativemsg"]) + debug(f"regPing() {self.net}: sent negativemsg '{sinst['negativemsg']}' to {sinst['entity']} - {self.num}") return else: self._negativePass = True if sinst["ping"]: self.msg(sinst["entity"], sinst["pingmsg"]) + debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}") return else: self.authenticated = True diff --git a/modules/chankeep.py b/modules/chankeep.py index ef5a52d..896928c 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -16,7 +16,7 @@ def getEnabledRelays(net): :return: list of enabled relay numbers """ enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]] - debug(f"getEnabledRelays() {net}: {enabledRelays}") + # debug(f"getEnabledRelays() {net}: {enabledRelays}") return enabledRelays @@ -34,7 +34,7 @@ def getConnectedRelays(net): if name in main.IRCPool.keys(): if main.IRCPool[name].isconnected: connectedRelays.append(i) - debug(f"getConnectedRelays() {net}: {connectedRelays}") + # debug(f"getConnectedRelays() {net}: {connectedRelays}") return connectedRelays @@ -50,15 +50,15 @@ def getActiveRelays(net): for i in enabledRelays: name = net + str(i) if name in main.IRCPool.keys(): - debug( - ( - f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} " - f"conn:{main.IRCPool[name].isconnected}" - ) - ) + # debug( + # ( + # f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} " + # f"conn:{main.IRCPool[name].isconnected}" + # ) + # ) if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: activeRelays.append(i) - debug(f"getActiveRelays() {net}: {activeRelays}") + # debug(f"getActiveRelays() {net}: {activeRelays}") return activeRelays @@ -84,7 +84,7 @@ def allRelaysActive(net): activeRelays = getActiveRelays(net) enabledRelays = getEnabledRelays(net) relaysActive = len(activeRelays) == len(enabledRelays) - debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})") + # debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})") return relaysActive diff --git a/modules/helpers.py b/modules/helpers.py index 33b484f..8d419ae 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -12,16 +12,16 @@ def get_first_relay(net): """ cur_relay = 0 max_relay = len(main.network[net].relays.keys()) + 1 - debug(f"get_first_relay() {net}: max_relay:{max_relay}") + # debug(f"get_first_relay() {net}: max_relay:{max_relay}") activeRelays = chankeep.getActiveRelays(net) - debug(f"get_first_relay() {net}: activeRelays:{activeRelays}") + # debug(f"get_first_relay() {net}: activeRelays:{activeRelays}") while cur_relay != max_relay: cur_relay += 1 if cur_relay not in activeRelays: continue name = net + str(cur_relay) if name in main.IRCPool.keys(): - debug(f"get_first_relay() {net}: found relay {name}") + # debug(f"get_first_relay() {net}: found relay {name}") return main.IRCPool[name] return None diff --git a/modules/regproc.py b/modules/regproc.py index 6037ddd..364b375 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -146,6 +146,8 @@ def get_unregistered_relays(net=None): def registerTest(c): sinst = substitute(c["net"], c["num"]) name = c["net"] + str(c["num"]) + net = c["net"] + num = c["num"] if sinst["check"] is False: return if "msg" in c.keys() and not c["msg"] is None: @@ -157,24 +159,28 @@ def registerTest(c): confirmRegistration( c["net"], c["num"], negativepass=False ) # Not passed negative check, report back + debug(f"registerTest() {net} - {num} not passed negative:checknegativemsg check, {sinst['checknegativemsg']} present in message") return if sinst["checkendnegative"] in c["msg"]: confirmRegistration( c["net"], c["num"], negativepass=True ) # Passed the negative check, report back + debug(f"registerTest() {net} - {num} passed negative:checkendnegative check, {sinst['checkendnegative']} present in message") return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: confirmRegistration(c["net"], c["num"]) + debug(f"registerTest() {net} - {num} passed ping:checkmsg2 check, {sinst['checkmsg2']} present in message") return - if sinst["checktype"] == "msg": - if "msg" in c.keys(): + if sinst["checktype"] == "msg": if sinst["checkmsg"] in c["msg"]: confirmRegistration(c["net"], c["num"]) + debug(f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, {sinst['checkmsg']} present in message") return elif sinst["checktype"] == "mode": if c["type"] == "self": if c["mtype"] == "mode": if sinst["checkmode"] in c["mode"] and c["status"] is True: confirmRegistration(c["net"], c["num"]) + debug(f"registerTest() {net} - {num} passed checktype:mode:checkmost check, {sinst['checkmode']} present in mode") return From f7d390da326708ea176a352015ae83942fd16639 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 12:43:33 +0100 Subject: [PATCH 352/394] Implement API for authentication management actions --- api/views.py | 33 +++++++++++++++++++++++++++++++++ commands/getstr.py | 4 ++-- core/bot.py | 12 +++++++++--- modules/helpers.py | 19 ++++++++++++++++++- modules/regproc.py | 42 +++++++++++++++++++++++++++++++++++++----- 5 files changed, 99 insertions(+), 11 deletions(-) diff --git a/api/views.py b/api/views.py index 842c186..78e2b51 100644 --- a/api/views.py +++ b/api/views.py @@ -213,6 +213,39 @@ class API(object): network["records"] = userinfo.getNumWhoEntries(net) return dumps(network) + @app.route("/irc/network/auth/", methods=["POST"]) + @login_required + def irc_network_recheckauth(self, request): + try: + data = loads(request.content.read()) + except JSONDecodeError: + return "Invalid JSON" + if "func" not in data: + return dumps({"success": False, "reason": "no function specified."}) + func = data["func"] + + if "net" not in data: + return dumps({"success": False, "reason": "no net specified."}) + net = data["net"] + if not net or net == "None": + nets = main.network.keys() + else: + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + nets = [net] + for net_name in nets: + conns = helpers.get_connected_relays(net_name) + if not conns: + return dumps({"success": False, "reason": f"failed to get instances for {net_name}."}) + if func == "recheckauth": + for conn in conns: + conn.regPing() + elif func == "resetauth": + for conn in conns: + conn.authenticated = False + conn.regPing() + return dumps({"success": True}) + @app.route("/irc/network//", methods=["DELETE"]) @login_required def irc_network_delete(self, request, net): diff --git a/commands/getstr.py b/commands/getstr.py index 205088e..c4fe6a9 100644 --- a/commands/getstr.py +++ b/commands/getstr.py @@ -1,7 +1,7 @@ import main - from utils.get import getRelay + class GetstrCommand: def __init__(self, *args): self.getstr(*args) @@ -11,7 +11,7 @@ class GetstrCommand: if length == 3: net = spl[1] num = spl[2] - if not net in main.network.keys(): + if net not in main.network.keys(): failure("Network does not exist: %s" % net) return if not num.isdigit(): diff --git a/core/bot.py b/core/bot.py index 36fc7c4..5b2b04d 100644 --- a/core/bot.py +++ b/core/bot.py @@ -55,7 +55,7 @@ class IRCBot(IRCClient): self.isconnected = False self.channels = [] self.net = net - self.authenticated = not regproc.needToRegister(self.net) + self.authenticated = not regproc.needToAuth(self.net) self.num = num self.buffer = "" self.name = net + str(num) @@ -621,8 +621,9 @@ class IRCBot(IRCClient): def regPing(self, negativepass=None): if self.authenticated: return - if not regproc.needToRegister(self.net): + if not regproc.needToAuth(self.net): self.authenticated = True + return sinst = regproc.substitute(self.net, self.num) if not sinst: error(f"regPing() {self.net}: registration ping failed for {self.num}") @@ -649,7 +650,12 @@ class IRCBot(IRCClient): if sinst["negative"]: self._negativePass = None self.msg(sinst["entity"], sinst["negativemsg"]) - debug(f"regPing() {self.net}: sent negativemsg '{sinst['negativemsg']}' to {sinst['entity']} - {self.num}") + debug( + ( + f"regPing() {self.net}: sent negativemsg " + f"'{sinst['negativemsg']}' to {sinst['entity']} - {self.num}" + ) + ) return else: self._negativePass = True diff --git a/modules/helpers.py b/modules/helpers.py index 8d419ae..f2a8690 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -1,6 +1,7 @@ import main from modules import chankeep -from utils.logging.debug import debug + +# from utils.logging.debug import debug def get_first_relay(net): @@ -53,3 +54,19 @@ def get_active_relays(net): if name in main.IRCPool.keys(): active_insts.append(main.IRCPool[name]) return active_insts + + +def get_connected_relays(net): + """ + Get all connected instances for the network. + :param net: the network + :return: list of active instances + :rtype: list of IRCPool instances + """ + active_nums = chankeep.getConnectedRelays(net) + active_insts = [] + for num in active_nums: + name = net + str(num) + if name in main.IRCPool.keys(): + active_insts.append(main.IRCPool[name]) + return active_insts diff --git a/modules/regproc.py b/modules/regproc.py index 364b375..edb411c 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -21,6 +21,13 @@ def needToRegister(net): return False +def needToAuth(net): + networkObj = main.network[net] + if networkObj.auth == "none": + return False + return True + + def selectInst(net): if net in main.irc.keys(): inst = deepcopy(main.irc[net]) @@ -159,28 +166,53 @@ def registerTest(c): confirmRegistration( c["net"], c["num"], negativepass=False ) # Not passed negative check, report back - debug(f"registerTest() {net} - {num} not passed negative:checknegativemsg check, {sinst['checknegativemsg']} present in message") + debug( + ( + f"registerTest() {net} - {num} not passed negative:checknegativemsg " + f"check, {sinst['checknegativemsg']} present in message" + ) + ) return if sinst["checkendnegative"] in c["msg"]: confirmRegistration( c["net"], c["num"], negativepass=True ) # Passed the negative check, report back - debug(f"registerTest() {net} - {num} passed negative:checkendnegative check, {sinst['checkendnegative']} present in message") + debug( + ( + f"registerTest() {net} - {num} passed negative:checkendnegative " + f"check, {sinst['checkendnegative']} present in message" + ) + ) return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: confirmRegistration(c["net"], c["num"]) - debug(f"registerTest() {net} - {num} passed ping:checkmsg2 check, {sinst['checkmsg2']} present in message") + debug( + ( + f"registerTest() {net} - {num} passed ping:checkmsg2 " + f"check, {sinst['checkmsg2']} present in message" + ) + ) return if sinst["checktype"] == "msg": if sinst["checkmsg"] in c["msg"]: confirmRegistration(c["net"], c["num"]) - debug(f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, {sinst['checkmsg']} present in message") + debug( + ( + f"registerTest() {net} - {num} passed checktype:msg:checkmsg check, " + f"{sinst['checkmsg']} present in message" + ) + ) return elif sinst["checktype"] == "mode": if c["type"] == "self": if c["mtype"] == "mode": if sinst["checkmode"] in c["mode"] and c["status"] is True: confirmRegistration(c["net"], c["num"]) - debug(f"registerTest() {net} - {num} passed checktype:mode:checkmost check, {sinst['checkmode']} present in mode") + debug( + ( + f"registerTest() {net} - {num} passed checktype:mode:checkmost check, " + f"{sinst['checkmode']} present in mode" + ) + ) return From 060ee4f0d523716ecb244bebf89c24bd8e2902d3 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 13:13:05 +0100 Subject: [PATCH 353/394] Implement manual authentication mode --- api/views.py | 2 +- modules/provision.py | 2 ++ modules/regproc.py | 22 +++++++++++++++++++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/api/views.py b/api/views.py index 78e2b51..0c08a1b 100644 --- a/api/views.py +++ b/api/views.py @@ -668,5 +668,5 @@ class API(object): num = int(num) if num not in main.network[net].relays.keys(): return dumps({"success": False, "reason": f"no relay {num} on {net}"}) - regproc.enableAuthentication(net, num) + regproc.enableAuthentication(net, num, jump=False, run_now=True) return dumps({"success": True}) diff --git a/modules/provision.py b/modules/provision.py index f517236..3b9d6a4 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -43,11 +43,13 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): user = nick.lower() if auth == "sasl": commands["sasl"] = [] + commands["status"].append("UnloadMod nickserv") commands["status"].append("LoadMod sasl") commands["sasl"].append("Mechanism plain") commands["sasl"].append("Set %s %s" % (nick, password)) elif auth == "ns": commands["nickserv"] = [] + commands["status"].append("UnloadMod sasl") commands["status"].append("LoadMod nickserv") commands["nickserv"].append("Set %s" % password) inst = modules.regproc.selectInst(network) diff --git a/modules/regproc.py b/modules/regproc.py index edb411c..00b20c2 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -114,15 +114,31 @@ def confirmRegistration(net, num, negativepass=None): main.saveConf("network") -def enableAuthentication(net, num): +def attemptManualAuthentication(net, num): + sinst = substitute(net, num) + obj = main.network[net] + identifymsg = sinst["identifymsg"] + entity = sinst["entity"] + name = f"{net}{num}" + if name not in main.IRCPool: + return + main.IRCPool[name].msg(entity, identifymsg) + +def enableAuthentication(net, num, jump=True, run_now=False): obj = main.network[net] nick = main.alias[num]["nick"] security = obj.security auth = obj.auth - password = obj.aliases[num]["password"] + name = f"{net}{num}" + if name not in main.IRCPool: + return # uname = main.alias[num]["nick"] + "/" + net + password = main.network[net].aliases[num]["password"] provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth - main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") + if jump: + main.IRCPool[name].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") + if run_now: + attemptManualAuthentication(net, num) if selectInst(net)["check"] is False: confirmRegistration(net, num) From 559e1f4afd289c287013431dd29e749d3c9be2d7 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 13:51:13 +0100 Subject: [PATCH 354/394] Print identification message --- modules/provision.py | 15 ++++++++++----- modules/regproc.py | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/modules/provision.py b/modules/provision.py index 3b9d6a4..e000611 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -7,7 +7,7 @@ from utils.logging.log import warn def provisionUserNetworkData( - num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password + num, nick, altnick, ident, realname, network, host, port, security ): commands = {} stage2commands = {} @@ -37,7 +37,7 @@ def provisionUserNetworkData( deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]]) -def provisionAuthenticationData(num, nick, network, security, auth, password): +def provisionAuthenticationData(num, nick, network, auth, password): commands = {} commands["status"] = [] user = nick.lower() @@ -63,15 +63,20 @@ def provisionAuthenticationData(num, nick, network, security, auth, password): def provisionRelay(num, network): # provision user and network data aliasObj = main.alias[num] # alias = aliasObj["nick"] + nick = aliasObj["nick"] + altnick = aliasObj["altnick"] + ident = aliasObj["ident"] + realname = aliasObj["realname"] provisionUserNetworkData( num, - *aliasObj.values(), + nick, + altnick, + ident, + realname, network, main.network[network].host, main.network[network].port, main.network[network].security, - main.network[network].auth, - main.network[network].aliases[num]["password"], ) if main.config["ConnectOnCreate"]: reactor.callLater(10, main.network[network].start_bot, num) diff --git a/modules/regproc.py b/modules/regproc.py index 00b20c2..685568f 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -120,6 +120,7 @@ def attemptManualAuthentication(net, num): identifymsg = sinst["identifymsg"] entity = sinst["entity"] name = f"{net}{num}" + print(f"SENDING `{identifymsg}` TO `{entity}` ON {name}") if name not in main.IRCPool: return main.IRCPool[name].msg(entity, identifymsg) @@ -127,14 +128,13 @@ def attemptManualAuthentication(net, num): def enableAuthentication(net, num, jump=True, run_now=False): obj = main.network[net] nick = main.alias[num]["nick"] - security = obj.security auth = obj.auth name = f"{net}{num}" if name not in main.IRCPool: return # uname = main.alias[num]["nick"] + "/" + net password = main.network[net].aliases[num]["password"] - provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth + provision.provisionAuthenticationData(num, nick, net, auth, password) # Set up for auth if jump: main.IRCPool[name].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump") if run_now: From 281eb75b26f4a2d8c280b1043681113ad107b6ab Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 15:43:48 +0100 Subject: [PATCH 355/394] Fix variable shadowing --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 0c08a1b..275fa42 100644 --- a/api/views.py +++ b/api/views.py @@ -291,8 +291,8 @@ class API(object): else: chanlimit = int(chanlimit) online_relays = helpers.get_active_relays(net) - for inst in online_relays: - inst.chanlimit = chanlimit + for r in online_relays: + r.chanlimit = chanlimit elif item == "security": security = data[item][0] if security not in ["ssl", "plain"]: From 02739abaf4f68076a95f67407dbec4c9fb737a09 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 15:53:18 +0100 Subject: [PATCH 356/394] Allow current nick substitution in IRC commands --- conf/templates/irc.json | 1 + modules/regproc.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/conf/templates/irc.json b/conf/templates/irc.json index e37a788..6ec2722 100644 --- a/conf/templates/irc.json +++ b/conf/templates/irc.json @@ -4,6 +4,7 @@ "entity": "NickServ", "domains": [], "registermsg": "REGISTER {password} {email}", + "identifymsg": "IDENTIFY {password}", "confirm": "CONFIRM {token}", "check": true, "ping": true, diff --git a/modules/regproc.py b/modules/regproc.py index 685568f..0fc5107 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -2,7 +2,7 @@ from copy import deepcopy from random import choice import main -from modules import provision +from modules import provision, helpers from utils.logging.debug import debug from utils.logging.log import error @@ -61,10 +61,17 @@ def substitute(net, num, token=None): # username = nickname + "/" + net password = main.network[net].aliases[num]["password"] # inst["email"] = inst["email"].replace("{nickname}", nickname) + + name = f"{net}{num}" + if name in main.IRCPool: + curnick = main.IRCPool[name].nickname + else: + curnick = nickname for i in inst.keys(): if not isinstance(inst[i], str): continue inst[i] = inst[i].replace("{nickname}", nickname) + inst[i] = inst[i].replace("{curnick}", curnick) inst[i] = inst[i].replace("{password}", password) inst[i] = inst[i].replace("{email}", email) if token: From e30250603b15ccdd1b6b83dfc0987c764bd24615 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 16:09:32 +0100 Subject: [PATCH 357/394] Convert num to number in registration confirmation --- api/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/views.py b/api/views.py index 275fa42..660e716 100644 --- a/api/views.py +++ b/api/views.py @@ -654,6 +654,9 @@ class API(object): if not len(spl) == 2: return dumps({"success": False, "reason": f"malformed item: {item}"}) net, num = spl + if not num.isdigit(): + return dumps({"success": False, "reason": "invalid num: not a number."}) + num = int(num) if token: regproc.confirmAccount(net, num, token) return dumps({"success": True}) From b62200d41027efc43f14a3b553099f15450a95ec Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 16:26:09 +0100 Subject: [PATCH 358/394] Implement API call to register --- api/views.py | 3 +++ modules/provision.py | 4 +--- modules/regproc.py | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/views.py b/api/views.py index 660e716..643cea1 100644 --- a/api/views.py +++ b/api/views.py @@ -244,6 +244,9 @@ class API(object): for conn in conns: conn.authenticated = False conn.regPing() + elif func == "register": + for conn in conns: + regproc.registerAccount(conn.net, conn.num) return dumps({"success": True}) @app.route("/irc/network//", methods=["DELETE"]) diff --git a/modules/provision.py b/modules/provision.py index e000611..db5ddcd 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -6,9 +6,7 @@ from utils.deliver_relay_commands import deliverRelayCommands from utils.logging.log import warn -def provisionUserNetworkData( - num, nick, altnick, ident, realname, network, host, port, security -): +def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security): commands = {} stage2commands = {} stage2commands["status"] = [] diff --git a/modules/regproc.py b/modules/regproc.py index 0fc5107..a9e83f3 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -2,7 +2,7 @@ from copy import deepcopy from random import choice import main -from modules import provision, helpers +from modules import provision from utils.logging.debug import debug from utils.logging.log import error @@ -89,7 +89,8 @@ def registerAccount(net, num): error("Cannot register for %s: function disabled" % (net)) return False name = net + str(num) - main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"]) + if not main.IRCPool[name].authenticated: + main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"]) def confirmAccount(net, num, token): @@ -123,15 +124,14 @@ def confirmRegistration(net, num, negativepass=None): def attemptManualAuthentication(net, num): sinst = substitute(net, num) - obj = main.network[net] identifymsg = sinst["identifymsg"] entity = sinst["entity"] name = f"{net}{num}" - print(f"SENDING `{identifymsg}` TO `{entity}` ON {name}") if name not in main.IRCPool: return main.IRCPool[name].msg(entity, identifymsg) + def enableAuthentication(net, num, jump=True, run_now=False): obj = main.network[net] nick = main.alias[num]["nick"] From c55a4058b1d9d2339aaaee0c7a27e2d3b0e5a205 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 16:45:40 +0100 Subject: [PATCH 359/394] Use JSON for sending messages --- api/views.py | 7 +++++-- core/bot.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index 643cea1..280fd5b 100644 --- a/api/views.py +++ b/api/views.py @@ -584,16 +584,19 @@ class API(object): listinfo = chankeep.getListInfo(net) return dumps({"success": True, "listinfo": listinfo}) - @app.route("/irc/msg////", methods=["PUT"]) + @app.route("/irc/msg///", methods=["PUT"]) @login_required - def irc_send_message(self, request, net, num, channel): + def irc_send_message(self, request, net, num): try: data = loads(request.content.read()) except JSONDecodeError: return "Invalid JSON" if "msg" not in data: return dumps({"success": False, "reason": "no msg."}) + if "channel" not in data: + return dumps({"success": False, "reason": "no msg."}) msg = data["msg"] + channel = data["channel"] if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) if not num.isdigit(): diff --git a/core/bot.py b/core/bot.py index 5b2b04d..b6c7329 100644 --- a/core/bot.py +++ b/core/bot.py @@ -258,7 +258,10 @@ class IRCBot(IRCClient): def sendmsg(self, channel, msg, in_query=False): query = f"{self.nickname}!*@*" - us = list(userinfo.getWhoSingle(self.net, query)) + res = userinfo.getWhoSingle(self.net, query) + if not res: + res = [] + us = list(res) if len(us) > 0: hostmask = us[0] else: From 1b39b46121b435bea21c2f8c0ec17d1bc583ea29 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 20:44:04 +0100 Subject: [PATCH 360/394] Blacklist channels we are kicked from --- core/bot.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/bot.py b/core/bot.py index b6c7329..5bff113 100644 --- a/core/bot.py +++ b/core/bot.py @@ -220,6 +220,12 @@ class IRCBot(IRCClient): castDup = deepcopy(cast) castDup["mtype"] = cast["type"] castDup["type"] = "self" + if self.net in main.blacklist.keys(): + if cast["channel"] not in main.blacklist[self.net]: + main.blacklist[self.net].append(cast["channel"]) + else: + main.blacklist[self.net] = [cast["channel"]] + main.saveConf("blacklist") self.event(**castDup) # we sent a message/left/joined/kick someone/quit From 4fa5c25e94f77b1b113b7647dd0dbb0932d2097c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 20:58:30 +0100 Subject: [PATCH 361/394] Fix getting all unregistered relays --- modules/regproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/regproc.py b/modules/regproc.py index a9e83f3..8a35461 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -166,7 +166,7 @@ def get_unregistered_relays(net=None): for num in main.network[i].relays.keys(): if not main.network[i].relays[num]["registered"]: nick = main.alias[num]["nick"] - if net in unreg: + if i in unreg: unreg[i].append([nick, num]) else: unreg[i] = [[nick, num]] From 6cdadd23a02fc5132094065e4108fb9de7316024 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 20:58:41 +0100 Subject: [PATCH 362/394] Implement initial WHO loop delay --- conf/templates/config.json | 1 + core/bot.py | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/conf/templates/config.json b/conf/templates/config.json index 9ea4fa5..1a47ad9 100644 --- a/conf/templates/config.json +++ b/conf/templates/config.json @@ -57,6 +57,7 @@ "Prefix": "*" }, "Delays": { + "WhoDelay": 3600, "WhoLoop": 600, "WhoRange": 1800, "Timeout": 30, diff --git a/core/bot.py b/core/bot.py index 5bff113..91b02fd 100644 --- a/core/bot.py +++ b/core/bot.py @@ -690,17 +690,27 @@ class IRCBot(IRCClient): if not self.authenticated: reactor.callLater(10, self.regPing) + def setup_who_loop(self, channel): + # if main.config["Toggles"]["Who"]: + lc = LoopingCall(self.who, channel) + self._getWho[channel] = lc + intrange = main.config["Tweaks"]["Delays"]["WhoRange"] + minint = main.config["Tweaks"]["Delays"]["WhoLoop"] + interval = randint(minint, minint + intrange) + lc.start(interval) + def joined(self, channel): if channel not in self.channels: self.channels.append(channel) self.names(channel).addCallback(self.got_names) if main.config["Toggles"]["Who"]: - lc = LoopingCall(self.who, channel) - self._getWho[channel] = lc - intrange = main.config["Tweaks"]["Delays"]["WhoRange"] - minint = main.config["Tweaks"]["Delays"]["WhoLoop"] - interval = randint(minint, minint + intrange) - lc.start(interval) + reactor.callLater(main.config["Tweaks"]["Delays"]["WhoDelay"], self.setup_who_loop, channel) + # lc = LoopingCall(self.who, channel) + # self._getWho[channel] = lc + # intrange = main.config["Tweaks"]["Delays"]["WhoRange"] + # minint = main.config["Tweaks"]["Delays"]["WhoLoop"] + # interval = randint(minint, minint + intrange) + # lc.start(interval) def botLeft(self, channel): if channel in self.channels: From 153d3dd847b0b31c0f9378e5732577160f8ab40f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 14 Aug 2022 23:58:35 +0100 Subject: [PATCH 363/394] Reset negative pass status when requesting recheck --- api/views.py | 4 ++-- core/bot.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/views.py b/api/views.py index 280fd5b..adbac22 100644 --- a/api/views.py +++ b/api/views.py @@ -239,11 +239,11 @@ class API(object): return dumps({"success": False, "reason": f"failed to get instances for {net_name}."}) if func == "recheckauth": for conn in conns: - conn.regPing() + conn.regPing(reset=True) elif func == "resetauth": for conn in conns: conn.authenticated = False - conn.regPing() + conn.regPing(reset=True) elif func == "register": for conn in conns: regproc.registerAccount(conn.net, conn.num) diff --git a/core/bot.py b/core/bot.py index 91b02fd..4d33c5c 100644 --- a/core/bot.py +++ b/core/bot.py @@ -627,7 +627,7 @@ class IRCBot(IRCClient): # End of Twisted hackery - def regPing(self, negativepass=None): + def regPing(self, negativepass=None, reset=True): if self.authenticated: return if not regproc.needToAuth(self.net): @@ -637,6 +637,9 @@ class IRCBot(IRCClient): if not sinst: error(f"regPing() {self.net}: registration ping failed for {self.num}") return + if reset: + self._negativePass = None + if self._negativePass is not True: if negativepass is False: self._negativePass = False @@ -652,7 +655,7 @@ class IRCBot(IRCClient): debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}") return else: - debug("regPing() {self.net}: negative registration check - {self.num}") + debug(f"regPing() {self.net}: negative registration check - {self.num}") return if sinst["check"]: From 560af8aeb0298dbdbdea57824ae84cbbd8c359de Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:03:12 +0100 Subject: [PATCH 364/394] Lower max_chans to length of LIST if it's shorter --- modules/chankeep.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index 896928c..c49d4dd 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -303,6 +303,8 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # else: # newNums = [] flist = [i[0] for i in listinfo] + if len(flist) > max_chans: + max_chans = len(flist) chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") @@ -315,6 +317,8 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # else: # newNums = [] siglist = [i[0] for i in listinfo if int(i[1]) > mean] + if len(siglist) > max_chans: + max_chans = len(siglist) chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") From bdb2949d174a1a4b4419ef63bafe5a5e7d87ac1f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:04:49 +0100 Subject: [PATCH 365/394] Subtract one from length of list for indices --- modules/chankeep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index c49d4dd..f0365ee 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -304,7 +304,7 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = [] flist = [i[0] for i in listinfo] if len(flist) > max_chans: - max_chans = len(flist) + max_chans = len(flist)-1 chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") @@ -318,7 +318,7 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = [] siglist = [i[0] for i in listinfo if int(i[1]) > mean] if len(siglist) > max_chans: - max_chans = len(siglist) + max_chans = len(siglist)-1 chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") From 1b68568fb78097f205d05b23311a777f5d06836a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:07:29 +0100 Subject: [PATCH 366/394] Add debugging code in keepChannels --- modules/chankeep.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index f0365ee..e191220 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -319,6 +319,8 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): siglist = [i[0] for i in listinfo if int(i[1]) > mean] if len(siglist) > max_chans: max_chans = len(siglist)-1 + debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)} len_max:{len(max_chans)}") + chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") From 8f44f34d0e848a407102bbcf364a8d16da9362d2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:08:11 +0100 Subject: [PATCH 367/394] Fix debugging code in keepChannels --- modules/chankeep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index e191220..3dc2043 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -319,7 +319,7 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): siglist = [i[0] for i in listinfo if int(i[1]) > mean] if len(siglist) > max_chans: max_chans = len(siglist)-1 - debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)} len_max:{len(max_chans)}") + debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)}") chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") From 66e046e15f2af54dfb975d7bf01ded577d91d53a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:26:11 +0100 Subject: [PATCH 368/394] Fix list parsing --- modules/chankeep.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 3dc2043..48e5741 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -302,10 +302,10 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = modules.provision.provisionMultipleRelays(net, needed) # else: # newNums = [] - flist = [i[0] for i in listinfo] if len(flist) > max_chans: max_chans = len(flist)-1 chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + flist = [i[0] for i in listinfo] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") populateChans(net, chosen) @@ -316,12 +316,12 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = modules.provision.provisionMultipleRelays(net, needed) # else: # newNums = [] - siglist = [i[0] for i in listinfo if int(i[1]) > mean] + siglist = [i for i in listinfo if int(i[1]) > mean] + chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] + siglist = [i[0] for i in siglist] if len(siglist) > max_chans: max_chans = len(siglist)-1 debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)}") - - chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") populateChans(net, chosen) From ffed420c112f6817988cf5f34d0e56355da65251 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:27:16 +0100 Subject: [PATCH 369/394] Fix variable placement --- modules/chankeep.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 48e5741..607dddb 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -302,10 +302,11 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = modules.provision.provisionMultipleRelays(net, needed) # else: # newNums = [] - if len(flist) > max_chans: - max_chans = len(flist)-1 - chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] flist = [i[0] for i in listinfo] + if len(flist) > max_chans: + max_chans = len(flist) - 1 + chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") populateChans(net, chosen) @@ -317,10 +318,11 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # else: # newNums = [] siglist = [i for i in listinfo if int(i[1]) > mean] + if len(siglist) > max_chans: + max_chans = len(siglist) - 1 chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] siglist = [i[0] for i in siglist] - if len(siglist) > max_chans: - max_chans = len(siglist)-1 + debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)}") debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") From 731c6a2fd1c5bf7c8dab20b7ce7868ad26f549e0 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:29:08 +0100 Subject: [PATCH 370/394] Pass a list instead of listinfo --- modules/chankeep.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 607dddb..7f955fe 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -306,6 +306,7 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): if len(flist) > max_chans: max_chans = len(flist) - 1 chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] + chosen = [i[0] for i in chosen] debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") trace(f"keepChannels() {net}: joining:{chosen}") @@ -321,7 +322,7 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): if len(siglist) > max_chans: max_chans = len(siglist) - 1 chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] - siglist = [i[0] for i in siglist] + chosen = [i[0] for i in chosen] debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)}") debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") From 6f44921647da1c47207837a61ce5cfacf64bee1b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:36:36 +0100 Subject: [PATCH 371/394] Figure out the channel parsing logic --- modules/chankeep.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 7f955fe..eaef333 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -302,15 +302,16 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = modules.provision.provisionMultipleRelays(net, needed) # else: # newNums = [] - flist = [i[0] for i in listinfo] - if len(flist) > max_chans: - max_chans = len(flist) - 1 - chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans] - chosen = [i[0] for i in chosen] + listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1]) + if len(listinfo_sort) > max_chans: + max_chans = len(listinfo_sort) - 1 - debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels") - trace(f"keepChannels() {net}: joining:{chosen}") - populateChans(net, chosen) + flist = [i[0] for i in listinfo_sort] + + flist = flist[:max_chans] + debug(f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels") + trace(f"keepChannels() {net}: joining:{flist}") + populateChans(net, flist) else: # needed = sigrelay - len(getActiveRelays(net)) # if needed: @@ -318,16 +319,18 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # newNums = modules.provision.provisionMultipleRelays(net, needed) # else: # newNums = [] - siglist = [i for i in listinfo if int(i[1]) > mean] - if len(siglist) > max_chans: - max_chans = len(siglist) - 1 - chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans] - chosen = [i[0] for i in chosen] + listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1]) + if len(listinfo_sort) > max_chans: + max_chans = len(listinfo_sort) - 1 - debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(siglist)}") - debug(f"keepChannels() {net}: joining {len(chosen)}/{len(siglist)} channels") - trace(f"keepChannels() {net}: joining:{chosen}") - populateChans(net, chosen) + siglist = [i[0] for i in listinfo if int(i[1]) > mean] + + siglist = siglist[:max_chans] + + debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}") + debug(f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels") + trace(f"keepChannels() {net}: joining:{siglist}") + populateChans(net, siglist) notifyJoin(net) From f4225b622f6fdbc092c319af717ce3088f76f1af Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 00:39:22 +0100 Subject: [PATCH 372/394] Add more debugging information --- modules/chankeep.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/chankeep.py b/modules/chankeep.py index eaef333..97f4c82 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -320,12 +320,16 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # else: # newNums = [] listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1]) + debug(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}") if len(listinfo_sort) > max_chans: max_chans = len(listinfo_sort) - 1 + debug(f"keepChannels() {net}: new max_chans:{max_chans}") siglist = [i[0] for i in listinfo if int(i[1]) > mean] + debug(f"keepChannels() {net}: new siglist:{siglist}") siglist = siglist[:max_chans] + debug(f"keepChannels() {net}: truncated siglist:{siglist}") debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}") debug(f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels") From e64aaf99d81395d48e4ce3b8d30b04eb74d487ea Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 373/394] Reorder API endpoints to prevent clashing --- api/views.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/api/views.py b/api/views.py index adbac22..59d12d0 100644 --- a/api/views.py +++ b/api/views.py @@ -194,25 +194,6 @@ class API(object): } return dumps(networks) - @app.route("/irc/network//", methods=["POST"]) - @login_required - def irc_network(self, request, net): - if net not in main.network.keys(): - return dumps({"success": False, "reason": "no such net."}) - inst = main.network[net] - network = {} - network["net"] = inst.net - network["auth"] = inst.auth - network["host"] = inst.host - network["last"] = inst.last - network["port"] = inst.port - network["security"] = inst.security - network["relays"] = len(inst.relays) - network["chanlimit"] = inst.chanlimit - network["channels"] = userinfo.getTotalChanNum(net) - network["records"] = userinfo.getNumWhoEntries(net) - return dumps(network) - @app.route("/irc/network/auth/", methods=["POST"]) @login_required def irc_network_recheckauth(self, request): @@ -249,6 +230,25 @@ class API(object): regproc.registerAccount(conn.net, conn.num) return dumps({"success": True}) + @app.route("/irc/network//", methods=["POST"]) + @login_required + def irc_network(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + inst = main.network[net] + network = {} + network["net"] = inst.net + network["auth"] = inst.auth + network["host"] = inst.host + network["last"] = inst.last + network["port"] = inst.port + network["security"] = inst.security + network["relays"] = len(inst.relays) + network["chanlimit"] = inst.chanlimit + network["channels"] = userinfo.getTotalChanNum(net) + network["records"] = userinfo.getNumWhoEntries(net) + return dumps(network) + @app.route("/irc/network//", methods=["DELETE"]) @login_required def irc_network_delete(self, request, net): From d0ea3bb221081b9e5cd976b1c7cde861e6cd979b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 374/394] Change authentication endpoint --- api/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 59d12d0..5b409ab 100644 --- a/api/views.py +++ b/api/views.py @@ -194,7 +194,7 @@ class API(object): } return dumps(networks) - @app.route("/irc/network/auth/", methods=["POST"]) + @app.route("/irc/auth/", methods=["POST"]) @login_required def irc_network_recheckauth(self, request): try: @@ -214,6 +214,7 @@ class API(object): if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) nets = [net] + print("NETS", nets) for net_name in nets: conns = helpers.get_connected_relays(net_name) if not conns: @@ -227,12 +228,14 @@ class API(object): conn.regPing(reset=True) elif func == "register": for conn in conns: + print("REGISTER ACCOUNT", conn.net, conn.num) regproc.registerAccount(conn.net, conn.num) return dumps({"success": True}) @app.route("/irc/network//", methods=["POST"]) @login_required def irc_network(self, request, net): + print("IRC_NETWORK") if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) inst = main.network[net] From 659162ebc6c95623bf03bbfc0f81c5a4118aa7f1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 375/394] Fix IRC config mutation --- api/views.py | 3 --- modules/regproc.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/api/views.py b/api/views.py index 5b409ab..a8d44ce 100644 --- a/api/views.py +++ b/api/views.py @@ -214,7 +214,6 @@ class API(object): if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) nets = [net] - print("NETS", nets) for net_name in nets: conns = helpers.get_connected_relays(net_name) if not conns: @@ -228,14 +227,12 @@ class API(object): conn.regPing(reset=True) elif func == "register": for conn in conns: - print("REGISTER ACCOUNT", conn.net, conn.num) regproc.registerAccount(conn.net, conn.num) return dumps({"success": True}) @app.route("/irc/network//", methods=["POST"]) @login_required def irc_network(self, request, net): - print("IRC_NETWORK") if net not in main.network.keys(): return dumps({"success": False, "reason": "no such net."}) inst = main.network[net] diff --git a/modules/regproc.py b/modules/regproc.py index 8a35461..5183f1b 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -35,7 +35,7 @@ def selectInst(net): if i not in inst: inst[i] = main.irc["_"][i] else: - inst = main.irc["_"] + inst = deepcopy(main.irc["_"]) return inst From 7efde28d99abdc51e7abf0ae6aaf5361dbfaf67a Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 376/394] Look before you leap to confirming registrations --- modules/regproc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/regproc.py b/modules/regproc.py index 5183f1b..e06a94f 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -96,7 +96,8 @@ def registerAccount(net, num): def confirmAccount(net, num, token): sinst = substitute(net, num, token=token) name = net + str(num) - main.IRCPool[name].msg(sinst["entity"], sinst["confirm"]) + if name in main.IRCPool: + main.IRCPool[name].msg(sinst["entity"], sinst["confirm"]) enableAuthentication(net, num) From b25cb1699f0d83f5d508e3114c768f192368a26b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 377/394] Set the channel limit on connected relays, not active --- api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index a8d44ce..12269ba 100644 --- a/api/views.py +++ b/api/views.py @@ -293,7 +293,7 @@ class API(object): return dumps({"success": False, "reason": "invalid chanlimit: not a number."}) else: chanlimit = int(chanlimit) - online_relays = helpers.get_active_relays(net) + online_relays = helpers.get_connected_relays(net) for r in online_relays: r.chanlimit = chanlimit elif item == "security": From 53ee69540ff0e7777bd9df5274ca535c5e971285 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 378/394] Fix double messages and regPing logic --- core/bot.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/core/bot.py b/core/bot.py index 4d33c5c..48e0374 100644 --- a/core/bot.py +++ b/core/bot.py @@ -640,20 +640,23 @@ class IRCBot(IRCClient): if reset: self._negativePass = None - if self._negativePass is not True: + if self._negativePass is None: + # We have failed, the blacklisted message has been found if negativepass is False: self._negativePass = False debug(f"regPing() {self.net}: negativepass is False for {self.num}") return + # End of negative output reached with no blacklisted message if negativepass is True: - if self._negativePass is None: + if self._negativePass is None: # check if it's not failed yet self._negativePass = True debug(f"regPing() {self.net}: negpass:True positive registration check - {self.num}") - if sinst["ping"]: - debug("Sending ping - %s - %i" % (self.net, self.num)) - self.msg(sinst["entity"], sinst["pingmsg"]) - debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}") - return + # if sinst["ping"]: + # debug("Sending ping - %s - %i" % (self.net, self.num)) + # self.msg(sinst["entity"], sinst["pingmsg"]) + # debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' + # to {sinst['entity']} - {self.num}") + # return else: debug(f"regPing() {self.net}: negative registration check - {self.num}") return From 2942929478ebd91c44dfbb111a2f06cf4eab7c71 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 379/394] Improve regPing negative handling logic --- core/bot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/bot.py b/core/bot.py index 48e0374..0a1c820 100644 --- a/core/bot.py +++ b/core/bot.py @@ -647,7 +647,7 @@ class IRCBot(IRCClient): debug(f"regPing() {self.net}: negativepass is False for {self.num}") return # End of negative output reached with no blacklisted message - if negativepass is True: + elif negativepass is True: if self._negativePass is None: # check if it's not failed yet self._negativePass = True debug(f"regPing() {self.net}: negpass:True positive registration check - {self.num}") @@ -660,10 +660,8 @@ class IRCBot(IRCClient): else: debug(f"regPing() {self.net}: negative registration check - {self.num}") return - - if sinst["check"]: + else: if sinst["negative"]: - self._negativePass = None self.msg(sinst["entity"], sinst["negativemsg"]) debug( ( @@ -674,6 +672,8 @@ class IRCBot(IRCClient): return else: self._negativePass = True + + if sinst["check"]: if sinst["ping"]: self.msg(sinst["entity"], sinst["pingmsg"]) debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' to {sinst['entity']} - {self.num}") From ce32ab4722efda08eadf28d5f98a15ca2aa65a5b Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 380/394] Improve regPing debugging --- core/bot.py | 23 +++++++++++++++-------- modules/regproc.py | 12 ------------ 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/core/bot.py b/core/bot.py index 0a1c820..4a6b7ad 100644 --- a/core/bot.py +++ b/core/bot.py @@ -640,23 +640,30 @@ class IRCBot(IRCClient): if reset: self._negativePass = None + debug( + (f"regProc() {self.net} - {self.num}: _negativePass:{self._negativepass} " f"negativepass:{negativepass}") + ) if self._negativePass is None: # We have failed, the blacklisted message has been found if negativepass is False: self._negativePass = False - debug(f"regPing() {self.net}: negativepass is False for {self.num}") + debug( + ( + f"registerTest() {self.net} - {self.num} not passed negative:checknegativemsg " + f"check, {sinst['checknegativemsg']} present in message" + ) + ) return # End of negative output reached with no blacklisted message elif negativepass is True: if self._negativePass is None: # check if it's not failed yet self._negativePass = True - debug(f"regPing() {self.net}: negpass:True positive registration check - {self.num}") - # if sinst["ping"]: - # debug("Sending ping - %s - %i" % (self.net, self.num)) - # self.msg(sinst["entity"], sinst["pingmsg"]) - # debug(f"regPing() {self.net}: sent ping '{sinst['pingmsg']}' - # to {sinst['entity']} - {self.num}") - # return + debug( + ( + f"registerTest() {self.net} - {self.num} passed negative:checkendnegative " + f"check, {sinst['checkendnegative']} present in message" + ) + ) else: debug(f"regPing() {self.net}: negative registration check - {self.num}") return diff --git a/modules/regproc.py b/modules/regproc.py index e06a94f..b96dda2 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -190,23 +190,11 @@ def registerTest(c): confirmRegistration( c["net"], c["num"], negativepass=False ) # Not passed negative check, report back - debug( - ( - f"registerTest() {net} - {num} not passed negative:checknegativemsg " - f"check, {sinst['checknegativemsg']} present in message" - ) - ) return if sinst["checkendnegative"] in c["msg"]: confirmRegistration( c["net"], c["num"], negativepass=True ) # Passed the negative check, report back - debug( - ( - f"registerTest() {net} - {num} passed negative:checkendnegative " - f"check, {sinst['checkendnegative']} present in message" - ) - ) return if sinst["ping"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: From d0268810861fcff288dc77857c318b9f1bac5406 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 381/394] Properly format string --- core/bot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/bot.py b/core/bot.py index 4a6b7ad..bdeb43b 100644 --- a/core/bot.py +++ b/core/bot.py @@ -640,9 +640,7 @@ class IRCBot(IRCClient): if reset: self._negativePass = None - debug( - (f"regProc() {self.net} - {self.num}: _negativePass:{self._negativepass} " f"negativepass:{negativepass}") - ) + debug(f"regProc() {self.net} - {self.num}: _negativePass:{self._negativepass} negativepass:{negativepass}") if self._negativePass is None: # We have failed, the blacklisted message has been found if negativepass is False: From 415a0b113543f9bdffdf77a2a57ce83d9c04c071 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 382/394] Fix debug statements and amend function names --- core/bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/bot.py b/core/bot.py index bdeb43b..a368e69 100644 --- a/core/bot.py +++ b/core/bot.py @@ -640,14 +640,14 @@ class IRCBot(IRCClient): if reset: self._negativePass = None - debug(f"regProc() {self.net} - {self.num}: _negativePass:{self._negativepass} negativepass:{negativepass}") + debug(f"regPing() {self.net} - {self.num}: _negativePass:{self._negativePass} negativepass:{negativepass}") if self._negativePass is None: # We have failed, the blacklisted message has been found if negativepass is False: self._negativePass = False debug( ( - f"registerTest() {self.net} - {self.num} not passed negative:checknegativemsg " + f"regPing() {self.net} - {self.num} not passed negative:checknegativemsg " f"check, {sinst['checknegativemsg']} present in message" ) ) @@ -658,7 +658,7 @@ class IRCBot(IRCClient): self._negativePass = True debug( ( - f"registerTest() {self.net} - {self.num} passed negative:checkendnegative " + f"regPing() {self.net} - {self.num} passed negative:checkendnegative " f"check, {sinst['checkendnegative']} present in message" ) ) From e4c1d80250dc7466adbc01beaf6c57c1812cc9ae Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 383/394] Only run pingmsg after negative has completed --- core/bot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index a368e69..45d7191 100644 --- a/core/bot.py +++ b/core/bot.py @@ -676,8 +676,10 @@ class IRCBot(IRCClient): ) return else: - self._negativePass = True + self._negativePass = True # if it's disabled, we still want the next block to run + # Only run if negativepass has completed, or exempted as above + if self._negativePass: if sinst["check"]: if sinst["ping"]: self.msg(sinst["entity"], sinst["pingmsg"]) From 0b69893e172844caaccee41fea1f1df2c948fadb Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 17:59:31 +0100 Subject: [PATCH 384/394] Fix query handling and don't send a fake message --- api/views.py | 14 +++++++++++--- core/bot.py | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/views.py b/api/views.py index 12269ba..588120d 100644 --- a/api/views.py +++ b/api/views.py @@ -608,13 +608,21 @@ class API(object): if name not in main.IRCPool.keys(): return dumps({"success": False, "reason": f"relay {num} not on {net}"}) # We are in a query + in_query = False + if "nick" in data: + nick = data["nick"] + if nick == channel: + in_query = True + else: + nick = None if channel == main.IRCPool[name].nickname: - if "nick" not in data: + in_query = True + if nick: return dumps({"success": False, "reason": "no nick specified to query"}) else: - main.IRCPool[name].sendmsg(data["nick"], msg, in_query=True) + main.IRCPool[name].sendmsg(nick, msg, in_query=in_query) else: - main.IRCPool[name].sendmsg(channel, msg) + main.IRCPool[name].sendmsg(channel, msg, in_query=in_query) return dumps({"success": True, "message": f"sent message to {channel} on {name}"}) @app.route("/irc/nick///", methods=["GET"]) diff --git a/core/bot.py b/core/bot.py index 45d7191..45dd84f 100644 --- a/core/bot.py +++ b/core/bot.py @@ -272,7 +272,7 @@ class IRCBot(IRCClient): hostmask = us[0] else: # Close enough... - hostmask = f"{self.nickname}!*@{self.servername}" + hostmask = f"{self.nickname}!{self.nickname}@{self.servername}" warn(f"Could not get a hostname, using {hostmask}") nick, ident, host = parsen(hostmask) # We sent someone a query reply @@ -280,7 +280,7 @@ class IRCBot(IRCClient): self.event(type="self", mtype="msg", channel=self.nickname, nick=channel, ident=ident, host=host, msg=msg) else: self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) - self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) + # self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) self.msg(channel, msg) def get(self, var): From ed3c8497bc88b565811dfa6b1ecff949b249ab04 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 19:15:00 +0100 Subject: [PATCH 385/394] Switch debugging statements to trace in ChanKeep --- modules/chankeep.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 97f4c82..157f643 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -320,18 +320,18 @@ def keepChannels(net, listinfo, mean, sigrelay, relay): # else: # newNums = [] listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1]) - debug(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}") + trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}") if len(listinfo_sort) > max_chans: max_chans = len(listinfo_sort) - 1 debug(f"keepChannels() {net}: new max_chans:{max_chans}") siglist = [i[0] for i in listinfo if int(i[1]) > mean] - debug(f"keepChannels() {net}: new siglist:{siglist}") + trace(f"keepChannels() {net}: new siglist:{siglist}") siglist = siglist[:max_chans] - debug(f"keepChannels() {net}: truncated siglist:{siglist}") + trace(f"keepChannels() {net}: truncated siglist:{siglist}") - debug(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}") + trace(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}") debug(f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels") trace(f"keepChannels() {net}: joining:{siglist}") populateChans(net, siglist) From a65098c22219d9f3d98b3912674f6ec943185ebc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 19:15:12 +0100 Subject: [PATCH 386/394] Add sinst fetch and fix message send logic --- api/views.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 588120d..0ad5f0b 100644 --- a/api/views.py +++ b/api/views.py @@ -617,7 +617,7 @@ class API(object): nick = None if channel == main.IRCPool[name].nickname: in_query = True - if nick: + if not nick: return dumps({"success": False, "reason": "no nick specified to query"}) else: main.IRCPool[name].sendmsg(nick, msg, in_query=in_query) @@ -687,3 +687,13 @@ class API(object): return dumps({"success": False, "reason": f"no relay {num} on {net}"}) regproc.enableAuthentication(net, num, jump=False, run_now=True) return dumps({"success": True}) + + @app.route("/irc/sinst//", methods=["GET"]) + @login_required + def irc_get_authentity(self, request, net): + if net not in main.network.keys(): + return dumps({"success": False, "reason": "no such net."}) + auth = regproc.selectInst(net) + if not auth: + return dumps({"success": False, "reason": "error getting results."}) + return dumps({"success": True, "sinst": auth}) From b61316d8050cf3716e370c8494a0d68f8b5787e2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 19:24:42 +0100 Subject: [PATCH 387/394] Detect queries if nick and channel are the same --- api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 0ad5f0b..443e546 100644 --- a/api/views.py +++ b/api/views.py @@ -615,7 +615,7 @@ class API(object): in_query = True else: nick = None - if channel == main.IRCPool[name].nickname: + if channel == main.IRCPool[name].nickname or nick == channel: in_query = True if not nick: return dumps({"success": False, "reason": "no nick specified to query"}) From 8816024d90a778c2a7f83565c9dfb16270082b52 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 15 Aug 2022 19:49:21 +0100 Subject: [PATCH 388/394] Re-add fake messages --- core/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bot.py b/core/bot.py index 45dd84f..1554423 100644 --- a/core/bot.py +++ b/core/bot.py @@ -280,7 +280,7 @@ class IRCBot(IRCClient): self.event(type="self", mtype="msg", channel=self.nickname, nick=channel, ident=ident, host=host, msg=msg) else: self.event(type="self", mtype="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) - # self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) + self.event(type="msg", channel=channel, nick=self.nickname, ident=ident, host=host, msg=msg) self.msg(channel, msg) def get(self, var): From 07f1fff125c66fbbd0ebbc51fe18e68d1db4af6d Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 18 Aug 2022 07:20:30 +0100 Subject: [PATCH 389/394] Switch to siphash --- docker/requirements.prod.txt | 2 +- requirements.txt | 2 +- utils/dedup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/requirements.prod.txt b/docker/requirements.prod.txt index 5f94628..90c5a96 100644 --- a/docker/requirements.prod.txt +++ b/docker/requirements.prod.txt @@ -5,5 +5,5 @@ redis pyYaML python-logstash service_identity -csiphash +siphashc Klein diff --git a/requirements.txt b/requirements.txt index f6a2dcf..7dbea1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ redis pyYaML python-logstash service_identity -csiphash +siphashc Klein diff --git a/utils/dedup.py b/utils/dedup.py index 0565a0d..b4c44e3 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -2,7 +2,7 @@ from copy import deepcopy from datetime import datetime from json import dumps -from csiphash import siphash24 +from siphashc import siphash import main from utils.logging.debug import debug @@ -13,7 +13,7 @@ def dedup(numName, b): if "ts" in c.keys(): del c["ts"] c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]] - castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) + castHash = siphash(main.hashKey, dumps(c, sort_keys=True)) del c["approxtime"] isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) if isDuplicate: From 49b0b9db4600d80904c05286acd79752ef6dcfa5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 16 Aug 2022 22:01:35 +0100 Subject: [PATCH 390/394] Implement deduplicating channels --- commands/dedup.py | 24 +++++++++++++ core/bot.py | 2 ++ modules/chankeep.py | 82 ++++++++++++++++++++++++++++++++++++++++++++- modules/userinfo.py | 15 +++++++-- 4 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 commands/dedup.py diff --git a/commands/dedup.py b/commands/dedup.py new file mode 100644 index 0000000..168b955 --- /dev/null +++ b/commands/dedup.py @@ -0,0 +1,24 @@ +import main +from modules import chankeep +from modules import helpers +from json import dumps + +class DedupCommand: + def __init__(self, *args): + self.dedup(*args) + + def dedup(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length == 1: + dupes = chankeep.getDuplicateChannels() + chankeep.partChannels(dupes) + info(dumps(dupes)) + return + elif length == 2: + if spl[1] not in main.network.keys(): + failure("No such network: %s" % spl[1]) + return + dupes = chankeep.getDuplicateChannels(spl[1]) + chankeep.partChannels(dupes) + info(dumps(dupes)) + return diff --git a/core/bot.py b/core/bot.py index 1554423..d4b095d 100644 --- a/core/bot.py +++ b/core/bot.py @@ -628,6 +628,8 @@ class IRCBot(IRCClient): # End of Twisted hackery def regPing(self, negativepass=None, reset=True): + if not main.config["AutoReg"]: + return if self.authenticated: return if not regproc.needToAuth(self.net): diff --git a/modules/chankeep.py b/modules/chankeep.py index 157f643..716b791 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -4,10 +4,90 @@ from math import ceil from twisted.internet.threads import deferToThread import main +from modules import helpers from utils.logging.debug import debug, trace from utils.logging.log import error, log, warn +def getAllChannels(net=None): + """ + Get a list of all channels on all relays. + :return: list of channels + """ + channels = {} + if not net: + nets = main.network.keys() + else: + nets = [net] + for net in nets: + relays = helpers.get_connected_relays(net) + for relay in relays: + if net not in channels: + channels[net] = {} + if relay.num not in channels[net]: + channels[net][relay.num] = [] + for channel in relay.channels: + channels[net][relay.num].append(channel) + # debug(f"getAllChannels(): {channels}") + return channels + + +def getDuplicateChannels(net=None, total=False): + """ + Get a list of duplicate channels. + :return: list of duplicate channels + """ + allChans = getAllChannels(net) + duplicates = {} + for net in allChans.keys(): + net_chans = [] + inst = {} + # add all the channels from this network to a list + for num in allChans[net].keys(): + net_chans.extend(allChans[net][num]) + for channel in net_chans: + count_chan = net_chans.count(channel) + # I don't know why but it works + # this is used in userinfo.delChannels + set_min = 1 + if total: + set_min = 0 + if count_chan > set_min: + inst[channel] = count_chan + if inst: + duplicates[net] = inst + + if total: + return duplicates + + to_part = {} + for net in allChans: + if net in duplicates: + for num in allChans[net].keys(): + for channel in allChans[net][num]: + if channel in duplicates[net].keys(): + if duplicates[net][channel] > 1: + if net not in to_part: + to_part[net] = {} + if num not in to_part[net]: + to_part[net][num] = [] + to_part[net][num].append(channel) + duplicates[net][channel] -= 1 + + return to_part + + +def partChannels(data): + for net in data: + for num in data[net]: + name = f"{net}{num}" + if name in main.IRCPool.keys(): + for channel in data[net][num]: + if channel in main.IRCPool[name].channels: + main.IRCPool[name].part(channel) + log(f"Parted {channel} on {net} - {num}") + + def getEnabledRelays(net): """ Get a list of enabled relays for a network. @@ -241,7 +321,7 @@ def minifyChans(net, listinfo, as_list=False): if not allRelaysActive(net): error("All relays for %s are not active, cannot minify list" % net) return False - for i in getActiveRelays(net): + for i in getConnectedRelays(net): name = net + str(i) for x in main.IRCPool[name].channels: if as_list: diff --git a/modules/userinfo.py b/modules/userinfo.py index d281674..5180394 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,7 +1,8 @@ from twisted.internet.threads import deferToThread import main -from utils.logging.debug import trace +from modules import chankeep +from utils.logging.debug import debug, trace from utils.logging.log import warn from utils.parsing import parsen @@ -289,5 +290,15 @@ def _delChannels(net, channels): def delChannels(net, channels): # we have left a channel trace("Purging channel %s for %s" % (", ".join(channels), net)) - deferToThread(_delChannels, net, channels) + dupes = chankeep.getDuplicateChannels(net, total=True) + print("dupes: %s" % dupes) + if not dupes: + deferToThread(_delChannels, net, channels) + else: + for channel in channels: + if channel in dupes[net]: + if dupes[net][channel] != 0: + channels.remove(channel) + debug(f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it") + deferToThread(_delChannels, net, channels) # d.addCallback(testCallback) From c7941bfcda48ad0ba526a3b05705067f4510f241 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 16 Aug 2022 23:01:42 +0100 Subject: [PATCH 391/394] Log authentication messages --- modules/regproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/regproc.py b/modules/regproc.py index b96dda2..d6d3c28 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -130,7 +130,7 @@ def attemptManualAuthentication(net, num): name = f"{net}{num}" if name not in main.IRCPool: return - main.IRCPool[name].msg(entity, identifymsg) + main.IRCPool[name].sendmsg(entity, identifymsg, in_query=True) def enableAuthentication(net, num, jump=True, run_now=False): From 2a2f24f57041ae84c86fbbcb84f6e713b990e527 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 26 Aug 2022 22:17:12 +0100 Subject: [PATCH 392/394] Fix getting first relay when they are not sequential --- modules/chankeep.py | 2 +- modules/helpers.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/chankeep.py b/modules/chankeep.py index 716b791..d4f1b39 100644 --- a/modules/chankeep.py +++ b/modules/chankeep.py @@ -138,7 +138,7 @@ def getActiveRelays(net): # ) if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected: activeRelays.append(i) - # debug(f"getActiveRelays() {net}: {activeRelays}") + debug(f"getActiveRelays() {net}: {activeRelays}") return activeRelays diff --git a/modules/helpers.py b/modules/helpers.py index f2a8690..05d5bf2 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -1,7 +1,7 @@ import main from modules import chankeep -# from utils.logging.debug import debug +from utils.logging.debug import debug def get_first_relay(net): @@ -12,10 +12,11 @@ def get_first_relay(net): :return: IRCPool instance for the IRC bot """ cur_relay = 0 - max_relay = len(main.network[net].relays.keys()) + 1 - # debug(f"get_first_relay() {net}: max_relay:{max_relay}") + max_relay = max(main.network[net].relays.keys()) + debug(f"get_first_relay() {net}: max_relay:{max_relay}") activeRelays = chankeep.getActiveRelays(net) - # debug(f"get_first_relay() {net}: activeRelays:{activeRelays}") + + debug(f"get_first_relay() {net}: activeRelays:{activeRelays}") while cur_relay != max_relay: cur_relay += 1 if cur_relay not in activeRelays: From 2731713ede59df100fa5f20fb38426c74ddac73c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 27 Aug 2022 11:19:28 +0100 Subject: [PATCH 393/394] Fix error when no email can be found --- commands/dedup.py | 5 +++-- modules/helpers.py | 1 - modules/regproc.py | 9 ++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/commands/dedup.py b/commands/dedup.py index 168b955..0a286fc 100644 --- a/commands/dedup.py +++ b/commands/dedup.py @@ -1,7 +1,8 @@ +from json import dumps + import main from modules import chankeep -from modules import helpers -from json import dumps + class DedupCommand: def __init__(self, *args): diff --git a/modules/helpers.py b/modules/helpers.py index 05d5bf2..edd7ac2 100644 --- a/modules/helpers.py +++ b/modules/helpers.py @@ -1,6 +1,5 @@ import main from modules import chankeep - from utils.logging.debug import debug diff --git a/modules/regproc.py b/modules/regproc.py index d6d3c28..e9e2e1f 100644 --- a/modules/regproc.py +++ b/modules/regproc.py @@ -55,8 +55,7 @@ def substitute(net, num, token=None): email = f"{alias['nickname']}@{domain}" gotemail = True if not gotemail: - error(f"Could not get email for {net} - {num}") - return False + inst["email"] = False nickname = alias["nick"] # username = nickname + "/" + net password = main.network[net].aliases[num]["password"] @@ -73,7 +72,8 @@ def substitute(net, num, token=None): inst[i] = inst[i].replace("{nickname}", nickname) inst[i] = inst[i].replace("{curnick}", curnick) inst[i] = inst[i].replace("{password}", password) - inst[i] = inst[i].replace("{email}", email) + if gotemail: + inst[i] = inst[i].replace("{email}", email) if token: inst[i] = inst[i].replace("{token}", token) return inst @@ -85,6 +85,9 @@ def registerAccount(net, num): if not sinst: error(f"Register account failed for {net} - {num}") return + if not sinst["email"]: + error(f"Could not get email for {net} - {num}") + return if not sinst["register"]: error("Cannot register for %s: function disabled" % (net)) return False From 122fdca5db4abfdca77f6d6a90a910e34ff34831 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 6 Sep 2022 12:50:09 +0100 Subject: [PATCH 394/394] Moved files to subdirectory --- .env.example => legacy/.env.example | 0 .gitignore => legacy/.gitignore | 0 .../.pre-commit-config.yaml | 0 {api => legacy/api}/views.py | 0 {commands => legacy/commands}/__init__.py | 0 {commands => legacy/commands}/admall.py | 0 {commands => legacy/commands}/alias.py | 0 {commands => legacy/commands}/all.py | 0 {commands => legacy/commands}/allc.py | 0 {commands => legacy/commands}/authcheck.py | 0 {commands => legacy/commands}/auto.py | 0 {commands => legacy/commands}/blacklist.py | 0 {commands => legacy/commands}/chans.py | 0 {commands => legacy/commands}/cmd.py | 0 {commands => legacy/commands}/confirm.py | 0 {commands => legacy/commands}/dedup.py | 0 {commands => legacy/commands}/disable.py | 0 {commands => legacy/commands}/dist.py | 0 {commands => legacy/commands}/email.py | 0 {commands => legacy/commands}/enable.py | 0 {commands => legacy/commands}/exec.py | 0 {commands => legacy/commands}/getstr.py | 0 {commands => legacy/commands}/help.py | 0 {commands => legacy/commands}/join.py | 0 {commands => legacy/commands}/list.py | 0 {commands => legacy/commands}/load.py | 0 {commands => legacy/commands}/loadmod.py | 0 {commands => legacy/commands}/logout.py | 0 {commands => legacy/commands}/mod.py | 0 {commands => legacy/commands}/msg.py | 0 {commands => legacy/commands}/network.py | 0 {commands => legacy/commands}/part.py | 0 {commands => legacy/commands}/pass.py | 0 {commands => legacy/commands}/pending.py | 0 {commands => legacy/commands}/recheckauth.py | 0 {commands => legacy/commands}/reg.py | 0 {commands => legacy/commands}/relay.py | 0 {commands => legacy/commands}/save.py | 0 {commands => legacy/commands}/stats.py | 0 {commands => legacy/commands}/swho.py | 0 {commands => legacy/commands}/token.py | 0 {commands => legacy/commands}/users.py | 0 {commands => legacy/commands}/who.py | 0 {conf => legacy/conf}/templates/aliasdata.json | 0 {conf => legacy/conf}/templates/config.json | 0 {conf => legacy/conf}/templates/help.json | 0 {conf => legacy/conf}/templates/irc.json | 0 {core => legacy/core}/bot.py | 0 {core => legacy/core}/logstash.py | 0 {core => legacy/core}/parser.py | 0 {core => legacy/core}/relay.py | 0 {core => legacy/core}/server.py | 0 docker-compose.yml => legacy/docker-compose.yml | 0 {docker => legacy/docker}/Dockerfile | 0 .../docker}/docker-compose.prod.yml | 0 {docker => legacy/docker}/redis.conf | 0 {docker => legacy/docker}/requirements.prod.txt | 0 main.py => legacy/main.py | 0 {modules => legacy/modules}/__init__.py | 0 {modules => legacy/modules}/alias.py | 0 {modules => legacy/modules}/chankeep.py | 0 {modules => legacy/modules}/counters.py | 0 {modules => legacy/modules}/helpers.py | 0 {modules => legacy/modules}/monitor.py | 0 {modules => legacy/modules}/network.py | 0 {modules => legacy/modules}/provision.py | 0 {modules => legacy/modules}/regproc.py | 0 {modules => legacy/modules}/userinfo.py | 0 requirements.txt => legacy/requirements.txt | 0 runtest.sh => legacy/runtest.sh | 0 legacy/stack.env | 17 +++++++++++++++++ {tests => legacy/tests}/test_chankeep.py | 0 threshold => legacy/threshold | 0 {utils => legacy/utils}/cleanup.py | 0 {utils => legacy/utils}/dedup.py | 0 .../utils}/deliver_relay_commands.py | 0 {utils => legacy/utils}/get.py | 0 .../utils}/loaders/command_loader.py | 0 .../utils}/loaders/single_loader.py | 0 {utils => legacy/utils}/logging/debug.py | 0 {utils => legacy/utils}/logging/log.py | 0 {utils => legacy/utils}/logging/send.py | 0 {utils => legacy/utils}/parsing.py | 0 83 files changed, 17 insertions(+) rename .env.example => legacy/.env.example (100%) rename .gitignore => legacy/.gitignore (100%) rename .pre-commit-config.yaml => legacy/.pre-commit-config.yaml (100%) rename {api => legacy/api}/views.py (100%) rename {commands => legacy/commands}/__init__.py (100%) rename {commands => legacy/commands}/admall.py (100%) rename {commands => legacy/commands}/alias.py (100%) rename {commands => legacy/commands}/all.py (100%) rename {commands => legacy/commands}/allc.py (100%) rename {commands => legacy/commands}/authcheck.py (100%) rename {commands => legacy/commands}/auto.py (100%) rename {commands => legacy/commands}/blacklist.py (100%) rename {commands => legacy/commands}/chans.py (100%) rename {commands => legacy/commands}/cmd.py (100%) rename {commands => legacy/commands}/confirm.py (100%) rename {commands => legacy/commands}/dedup.py (100%) rename {commands => legacy/commands}/disable.py (100%) rename {commands => legacy/commands}/dist.py (100%) rename {commands => legacy/commands}/email.py (100%) rename {commands => legacy/commands}/enable.py (100%) rename {commands => legacy/commands}/exec.py (100%) rename {commands => legacy/commands}/getstr.py (100%) rename {commands => legacy/commands}/help.py (100%) rename {commands => legacy/commands}/join.py (100%) rename {commands => legacy/commands}/list.py (100%) rename {commands => legacy/commands}/load.py (100%) rename {commands => legacy/commands}/loadmod.py (100%) rename {commands => legacy/commands}/logout.py (100%) rename {commands => legacy/commands}/mod.py (100%) rename {commands => legacy/commands}/msg.py (100%) rename {commands => legacy/commands}/network.py (100%) rename {commands => legacy/commands}/part.py (100%) rename {commands => legacy/commands}/pass.py (100%) rename {commands => legacy/commands}/pending.py (100%) rename {commands => legacy/commands}/recheckauth.py (100%) rename {commands => legacy/commands}/reg.py (100%) rename {commands => legacy/commands}/relay.py (100%) rename {commands => legacy/commands}/save.py (100%) rename {commands => legacy/commands}/stats.py (100%) rename {commands => legacy/commands}/swho.py (100%) rename {commands => legacy/commands}/token.py (100%) rename {commands => legacy/commands}/users.py (100%) rename {commands => legacy/commands}/who.py (100%) rename {conf => legacy/conf}/templates/aliasdata.json (100%) rename {conf => legacy/conf}/templates/config.json (100%) rename {conf => legacy/conf}/templates/help.json (100%) rename {conf => legacy/conf}/templates/irc.json (100%) rename {core => legacy/core}/bot.py (100%) rename {core => legacy/core}/logstash.py (100%) rename {core => legacy/core}/parser.py (100%) rename {core => legacy/core}/relay.py (100%) rename {core => legacy/core}/server.py (100%) rename docker-compose.yml => legacy/docker-compose.yml (100%) rename {docker => legacy/docker}/Dockerfile (100%) rename {docker => legacy/docker}/docker-compose.prod.yml (100%) rename {docker => legacy/docker}/redis.conf (100%) rename {docker => legacy/docker}/requirements.prod.txt (100%) rename main.py => legacy/main.py (100%) rename {modules => legacy/modules}/__init__.py (100%) rename {modules => legacy/modules}/alias.py (100%) rename {modules => legacy/modules}/chankeep.py (100%) rename {modules => legacy/modules}/counters.py (100%) rename {modules => legacy/modules}/helpers.py (100%) rename {modules => legacy/modules}/monitor.py (100%) rename {modules => legacy/modules}/network.py (100%) rename {modules => legacy/modules}/provision.py (100%) rename {modules => legacy/modules}/regproc.py (100%) rename {modules => legacy/modules}/userinfo.py (100%) rename requirements.txt => legacy/requirements.txt (100%) rename runtest.sh => legacy/runtest.sh (100%) create mode 100644 legacy/stack.env rename {tests => legacy/tests}/test_chankeep.py (100%) rename threshold => legacy/threshold (100%) rename {utils => legacy/utils}/cleanup.py (100%) rename {utils => legacy/utils}/dedup.py (100%) rename {utils => legacy/utils}/deliver_relay_commands.py (100%) rename {utils => legacy/utils}/get.py (100%) rename {utils => legacy/utils}/loaders/command_loader.py (100%) rename {utils => legacy/utils}/loaders/single_loader.py (100%) rename {utils => legacy/utils}/logging/debug.py (100%) rename {utils => legacy/utils}/logging/log.py (100%) rename {utils => legacy/utils}/logging/send.py (100%) rename {utils => legacy/utils}/parsing.py (100%) diff --git a/.env.example b/legacy/.env.example similarity index 100% rename from .env.example rename to legacy/.env.example diff --git a/.gitignore b/legacy/.gitignore similarity index 100% rename from .gitignore rename to legacy/.gitignore diff --git a/.pre-commit-config.yaml b/legacy/.pre-commit-config.yaml similarity index 100% rename from .pre-commit-config.yaml rename to legacy/.pre-commit-config.yaml diff --git a/api/views.py b/legacy/api/views.py similarity index 100% rename from api/views.py rename to legacy/api/views.py diff --git a/commands/__init__.py b/legacy/commands/__init__.py similarity index 100% rename from commands/__init__.py rename to legacy/commands/__init__.py diff --git a/commands/admall.py b/legacy/commands/admall.py similarity index 100% rename from commands/admall.py rename to legacy/commands/admall.py diff --git a/commands/alias.py b/legacy/commands/alias.py similarity index 100% rename from commands/alias.py rename to legacy/commands/alias.py diff --git a/commands/all.py b/legacy/commands/all.py similarity index 100% rename from commands/all.py rename to legacy/commands/all.py diff --git a/commands/allc.py b/legacy/commands/allc.py similarity index 100% rename from commands/allc.py rename to legacy/commands/allc.py diff --git a/commands/authcheck.py b/legacy/commands/authcheck.py similarity index 100% rename from commands/authcheck.py rename to legacy/commands/authcheck.py diff --git a/commands/auto.py b/legacy/commands/auto.py similarity index 100% rename from commands/auto.py rename to legacy/commands/auto.py diff --git a/commands/blacklist.py b/legacy/commands/blacklist.py similarity index 100% rename from commands/blacklist.py rename to legacy/commands/blacklist.py diff --git a/commands/chans.py b/legacy/commands/chans.py similarity index 100% rename from commands/chans.py rename to legacy/commands/chans.py diff --git a/commands/cmd.py b/legacy/commands/cmd.py similarity index 100% rename from commands/cmd.py rename to legacy/commands/cmd.py diff --git a/commands/confirm.py b/legacy/commands/confirm.py similarity index 100% rename from commands/confirm.py rename to legacy/commands/confirm.py diff --git a/commands/dedup.py b/legacy/commands/dedup.py similarity index 100% rename from commands/dedup.py rename to legacy/commands/dedup.py diff --git a/commands/disable.py b/legacy/commands/disable.py similarity index 100% rename from commands/disable.py rename to legacy/commands/disable.py diff --git a/commands/dist.py b/legacy/commands/dist.py similarity index 100% rename from commands/dist.py rename to legacy/commands/dist.py diff --git a/commands/email.py b/legacy/commands/email.py similarity index 100% rename from commands/email.py rename to legacy/commands/email.py diff --git a/commands/enable.py b/legacy/commands/enable.py similarity index 100% rename from commands/enable.py rename to legacy/commands/enable.py diff --git a/commands/exec.py b/legacy/commands/exec.py similarity index 100% rename from commands/exec.py rename to legacy/commands/exec.py diff --git a/commands/getstr.py b/legacy/commands/getstr.py similarity index 100% rename from commands/getstr.py rename to legacy/commands/getstr.py diff --git a/commands/help.py b/legacy/commands/help.py similarity index 100% rename from commands/help.py rename to legacy/commands/help.py diff --git a/commands/join.py b/legacy/commands/join.py similarity index 100% rename from commands/join.py rename to legacy/commands/join.py diff --git a/commands/list.py b/legacy/commands/list.py similarity index 100% rename from commands/list.py rename to legacy/commands/list.py diff --git a/commands/load.py b/legacy/commands/load.py similarity index 100% rename from commands/load.py rename to legacy/commands/load.py diff --git a/commands/loadmod.py b/legacy/commands/loadmod.py similarity index 100% rename from commands/loadmod.py rename to legacy/commands/loadmod.py diff --git a/commands/logout.py b/legacy/commands/logout.py similarity index 100% rename from commands/logout.py rename to legacy/commands/logout.py diff --git a/commands/mod.py b/legacy/commands/mod.py similarity index 100% rename from commands/mod.py rename to legacy/commands/mod.py diff --git a/commands/msg.py b/legacy/commands/msg.py similarity index 100% rename from commands/msg.py rename to legacy/commands/msg.py diff --git a/commands/network.py b/legacy/commands/network.py similarity index 100% rename from commands/network.py rename to legacy/commands/network.py diff --git a/commands/part.py b/legacy/commands/part.py similarity index 100% rename from commands/part.py rename to legacy/commands/part.py diff --git a/commands/pass.py b/legacy/commands/pass.py similarity index 100% rename from commands/pass.py rename to legacy/commands/pass.py diff --git a/commands/pending.py b/legacy/commands/pending.py similarity index 100% rename from commands/pending.py rename to legacy/commands/pending.py diff --git a/commands/recheckauth.py b/legacy/commands/recheckauth.py similarity index 100% rename from commands/recheckauth.py rename to legacy/commands/recheckauth.py diff --git a/commands/reg.py b/legacy/commands/reg.py similarity index 100% rename from commands/reg.py rename to legacy/commands/reg.py diff --git a/commands/relay.py b/legacy/commands/relay.py similarity index 100% rename from commands/relay.py rename to legacy/commands/relay.py diff --git a/commands/save.py b/legacy/commands/save.py similarity index 100% rename from commands/save.py rename to legacy/commands/save.py diff --git a/commands/stats.py b/legacy/commands/stats.py similarity index 100% rename from commands/stats.py rename to legacy/commands/stats.py diff --git a/commands/swho.py b/legacy/commands/swho.py similarity index 100% rename from commands/swho.py rename to legacy/commands/swho.py diff --git a/commands/token.py b/legacy/commands/token.py similarity index 100% rename from commands/token.py rename to legacy/commands/token.py diff --git a/commands/users.py b/legacy/commands/users.py similarity index 100% rename from commands/users.py rename to legacy/commands/users.py diff --git a/commands/who.py b/legacy/commands/who.py similarity index 100% rename from commands/who.py rename to legacy/commands/who.py diff --git a/conf/templates/aliasdata.json b/legacy/conf/templates/aliasdata.json similarity index 100% rename from conf/templates/aliasdata.json rename to legacy/conf/templates/aliasdata.json diff --git a/conf/templates/config.json b/legacy/conf/templates/config.json similarity index 100% rename from conf/templates/config.json rename to legacy/conf/templates/config.json diff --git a/conf/templates/help.json b/legacy/conf/templates/help.json similarity index 100% rename from conf/templates/help.json rename to legacy/conf/templates/help.json diff --git a/conf/templates/irc.json b/legacy/conf/templates/irc.json similarity index 100% rename from conf/templates/irc.json rename to legacy/conf/templates/irc.json diff --git a/core/bot.py b/legacy/core/bot.py similarity index 100% rename from core/bot.py rename to legacy/core/bot.py diff --git a/core/logstash.py b/legacy/core/logstash.py similarity index 100% rename from core/logstash.py rename to legacy/core/logstash.py diff --git a/core/parser.py b/legacy/core/parser.py similarity index 100% rename from core/parser.py rename to legacy/core/parser.py diff --git a/core/relay.py b/legacy/core/relay.py similarity index 100% rename from core/relay.py rename to legacy/core/relay.py diff --git a/core/server.py b/legacy/core/server.py similarity index 100% rename from core/server.py rename to legacy/core/server.py diff --git a/docker-compose.yml b/legacy/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to legacy/docker-compose.yml diff --git a/docker/Dockerfile b/legacy/docker/Dockerfile similarity index 100% rename from docker/Dockerfile rename to legacy/docker/Dockerfile diff --git a/docker/docker-compose.prod.yml b/legacy/docker/docker-compose.prod.yml similarity index 100% rename from docker/docker-compose.prod.yml rename to legacy/docker/docker-compose.prod.yml diff --git a/docker/redis.conf b/legacy/docker/redis.conf similarity index 100% rename from docker/redis.conf rename to legacy/docker/redis.conf diff --git a/docker/requirements.prod.txt b/legacy/docker/requirements.prod.txt similarity index 100% rename from docker/requirements.prod.txt rename to legacy/docker/requirements.prod.txt diff --git a/main.py b/legacy/main.py similarity index 100% rename from main.py rename to legacy/main.py diff --git a/modules/__init__.py b/legacy/modules/__init__.py similarity index 100% rename from modules/__init__.py rename to legacy/modules/__init__.py diff --git a/modules/alias.py b/legacy/modules/alias.py similarity index 100% rename from modules/alias.py rename to legacy/modules/alias.py diff --git a/modules/chankeep.py b/legacy/modules/chankeep.py similarity index 100% rename from modules/chankeep.py rename to legacy/modules/chankeep.py diff --git a/modules/counters.py b/legacy/modules/counters.py similarity index 100% rename from modules/counters.py rename to legacy/modules/counters.py diff --git a/modules/helpers.py b/legacy/modules/helpers.py similarity index 100% rename from modules/helpers.py rename to legacy/modules/helpers.py diff --git a/modules/monitor.py b/legacy/modules/monitor.py similarity index 100% rename from modules/monitor.py rename to legacy/modules/monitor.py diff --git a/modules/network.py b/legacy/modules/network.py similarity index 100% rename from modules/network.py rename to legacy/modules/network.py diff --git a/modules/provision.py b/legacy/modules/provision.py similarity index 100% rename from modules/provision.py rename to legacy/modules/provision.py diff --git a/modules/regproc.py b/legacy/modules/regproc.py similarity index 100% rename from modules/regproc.py rename to legacy/modules/regproc.py diff --git a/modules/userinfo.py b/legacy/modules/userinfo.py similarity index 100% rename from modules/userinfo.py rename to legacy/modules/userinfo.py diff --git a/requirements.txt b/legacy/requirements.txt similarity index 100% rename from requirements.txt rename to legacy/requirements.txt diff --git a/runtest.sh b/legacy/runtest.sh similarity index 100% rename from runtest.sh rename to legacy/runtest.sh diff --git a/legacy/stack.env b/legacy/stack.env new file mode 100644 index 0000000..b04bec9 --- /dev/null +++ b/legacy/stack.env @@ -0,0 +1,17 @@ +THRESHOLD_LISTENER_HOST=0.0.0.0 +THRESHOLD_LISTENER_PORT=13867 +THRESHOLD_LISTENER_SSL=1 + +THRESHOLD_RELAY_ENABLED=0 +THRESHOLD_RELAY_HOST=0.0.0.0 +THRESHOLD_RELAY_PORT=13868 +THRESHOLD_RELAY_SSL=1 + +THRESHOLD_API_ENABLED=1 +THRESHOLD_API_HOST=0.0.0.0 +THRESHOLD_API_PORT=13869 +PORTAINER_GIT_DIR=.. + +THRESHOLD_CONFIG_DIR=../conf/live/ +THRESHOLD_TEMPLATE_DIR=../conf/example/ +THRESHOLD_CERT_DIR=../conf/cert/ \ No newline at end of file diff --git a/tests/test_chankeep.py b/legacy/tests/test_chankeep.py similarity index 100% rename from tests/test_chankeep.py rename to legacy/tests/test_chankeep.py diff --git a/threshold b/legacy/threshold similarity index 100% rename from threshold rename to legacy/threshold diff --git a/utils/cleanup.py b/legacy/utils/cleanup.py similarity index 100% rename from utils/cleanup.py rename to legacy/utils/cleanup.py diff --git a/utils/dedup.py b/legacy/utils/dedup.py similarity index 100% rename from utils/dedup.py rename to legacy/utils/dedup.py diff --git a/utils/deliver_relay_commands.py b/legacy/utils/deliver_relay_commands.py similarity index 100% rename from utils/deliver_relay_commands.py rename to legacy/utils/deliver_relay_commands.py diff --git a/utils/get.py b/legacy/utils/get.py similarity index 100% rename from utils/get.py rename to legacy/utils/get.py diff --git a/utils/loaders/command_loader.py b/legacy/utils/loaders/command_loader.py similarity index 100% rename from utils/loaders/command_loader.py rename to legacy/utils/loaders/command_loader.py diff --git a/utils/loaders/single_loader.py b/legacy/utils/loaders/single_loader.py similarity index 100% rename from utils/loaders/single_loader.py rename to legacy/utils/loaders/single_loader.py diff --git a/utils/logging/debug.py b/legacy/utils/logging/debug.py similarity index 100% rename from utils/logging/debug.py rename to legacy/utils/logging/debug.py diff --git a/utils/logging/log.py b/legacy/utils/logging/log.py similarity index 100% rename from utils/logging/log.py rename to legacy/utils/logging/log.py diff --git a/utils/logging/send.py b/legacy/utils/logging/send.py similarity index 100% rename from utils/logging/send.py rename to legacy/utils/logging/send.py diff --git a/utils/parsing.py b/legacy/utils/parsing.py similarity index 100% rename from utils/parsing.py rename to legacy/utils/parsing.py