Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
pull from: druid
merge into: Pathogen:master
Conversation 1 Commits 263 Files Changed 128 +363 -236
44 changed files with 363 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 2fecd98978 - Show all commits

2
commands/alias.py
View File

@ -32,7 +32,7 @@ class AliasCommand:
failure("Must be a number, not %s" % spl[2])
return
num = int(spl[2])
if not num in main.alias.keys():
if num not in main.alias.keys():
failure("No such alias: %i" % num)
return
failed = False

2
commands/all.py
View File

@ -1,5 +1,5 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class AllCommand:

2
commands/allc.py
View File

@ -1,5 +1,5 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class AllcCommand:

1
commands/chans.py
View File

@ -1,4 +1,3 @@
import main
import modules.userinfo as userinfo

3
commands/cmd.py
View File

@ -1,5 +1,4 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class CmdCommand:

6
commands/disable.py
View File

@ -1,5 +1,5 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class DisableCommand:
@ -19,13 +19,13 @@ class DisableCommand:
name = spl[1] + spl[2]
if not spl[1] in main.IRCPool.keys():
info("Note - instance not running, proceeding anyway")
if not relayNum in main.network[spl[1]].relays.keys():
if relayNum not in main.network[spl[1]].relays.keys():
failure("No such relay: %s in network %s" % (spl[2], spl[1]))
return
main.network[spl[1]].relays[relayNum]["enabled"] = False
user = main.alias[relayNum]["nick"]
network = spl[1]
relay = main.network[spl[1]].relays[relayNum]
# relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]}
deliverRelayCommands(relayNum, commands, user=user + "/" + network)
main.saveConf("network")

6
commands/email.py
View File

@ -17,7 +17,7 @@ class EmailCommand:
if "@" in domain:
failure("Not a domain: %s" % domain)
return
if not domain in main.irc["_"]["domains"]:
if domain not in main.irc["_"]["domains"]:
main.irc["_"]["domains"].append(domain)
success("Successfully added domain %s to default config" % domain)
main.saveConf("irc")
@ -41,7 +41,7 @@ class EmailCommand:
else:
num = int(spl[2])
if spl[1] == "add":
if not num in main.alias.keys():
if num not in main.alias.keys():
failure("No such alias: %i" % num)
return
if not spl[3] in main.alias[num]["emails"]:
@ -53,7 +53,7 @@ class EmailCommand:
failure("Email already exists in alias %i: %s" % (num, spl[3]))
return
elif spl[1] == "del":
if not num in main.alias.keys():
if num not in main.alias.keys():
failure("No such alias: %i" % num)
return
if spl[3] in main.alias[num]["emails"]:

2
commands/enable.py
View File

@ -1,5 +1,5 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class EnableCommand:

3
commands/exec.py
View File

@ -1,6 +1,3 @@
import main
class ExecCommand:
def __init__(self, *args):
self.exec(*args)

6
commands/list.py
View File

@ -9,7 +9,7 @@ class ListCommand:
if authed:
if length == 1:
for i in main.network.keys():
if not 1 in main.network[i].relays.keys():
if 1 not in main.network[i].relays.keys():
info("Network has no first instance: %s" % i)
continue
if not i + "1" in main.IRCPool.keys():
@ -22,10 +22,10 @@ class ListCommand:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
if not 1 in main.network[spl[1]].relays.keys():
if 1 not in main.network[spl[1]].relays.keys():
failure("Network has no first instance")
return
if not spl[1] + "1" in main.IRCPool.keys():
if spl[1] + "1" not in main.IRCPool.keys():
failure("No IRC instance: %s - 1" % spl[1])
return
main.IRCPool[spl[1] + "1"].list()

3
commands/loadmod.py
View File

@ -1,4 +1,3 @@
import main
from utils.loaders.single_loader import loadSingle
@ -10,7 +9,7 @@ class LoadmodCommand:
if authed:
if length == 2:
rtrn = loadSingle(spl[1])
if rtrn == True:
if rtrn is True:
success("Loaded module: %s" % spl[1])
return
elif rtrn == "RELOAD":

3
commands/logout.py
View File

@ -1,6 +1,3 @@
import main
class LogoutCommand:
def __init__(self, *args):
self.logout(*args)

5
commands/mod.py
View File

@ -1,5 +1,4 @@
import main
from yaml import dump
class ModCommand:
@ -16,8 +15,8 @@ class ModCommand:
try:
setattr(main.network[spl[1]], spl[2], spl[3])
except e:
failure("Something went wrong.")
except Exception as e:
failure(f"Something went wrong: {e}")
return
main.saveConf("network")

2
commands/msg.py
View File

@ -12,7 +12,7 @@ class MsgCommand:
failure("Network does not exist: %s" % spl[1])
return
if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %" % (spl[2], spl[1]))
failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
return
if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1])

5
commands/recheckauth.py
View File

@ -8,9 +8,7 @@ class RecheckauthCommand:
def recheckauth(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
if authed:
if length == 1:
results = []
for i in main.IRCPool.keys():
num = main.IRCPool[i].num
net = main.IRCPool[i].net
main.IRCPool[i].authenticated = False
main.IRCPool[i].regPing()
@ -20,9 +18,8 @@ class RecheckauthCommand:
if not spl[1] in main.network.keys():
failure("No such network: %s" % spl[1])
return
results = []
for i in main.IRCPool.keys():
num = main.IRCPool[i].num
# num = main.IRCPool[i].num
net = main.IRCPool[i].net
if not net == spl[1]:
continue

6
commands/stats.py
View File

@ -26,7 +26,7 @@ class StatsCommand:
stats.append("User records: %s" % numWhoEntries)
stats.append("Events/min: %s" % main.lastMinuteSample)
counterEvents = count.getEvents()
if counterEvents == None:
if counterEvents is None:
stats.append("No counters records")
else:
stats.append("Counters:")
@ -43,7 +43,7 @@ class StatsCommand:
numNodes = 0
for i in main.IRCPool.keys():
if "".join([x for x in i if not x in digits]) == spl[1]:
if "".join([x for x in i if x not in digits]) == spl[1]:
numChannels += len(main.IRCPool[i].channels)
found = True
numNodes += 1
@ -54,7 +54,7 @@ class StatsCommand:
stats.append("User records: %s" % numWhoEntries)
stats.append("Endpoints: %s" % numNodes)
counterEvents = count.getEvents(spl[1])
if counterEvents == None:
if counterEvents is None:
stats.append("No counters records")
else:
stats.append("Counters:")

1
commands/users.py
View File

@ -1,4 +1,3 @@
import main
import modules.userinfo as userinfo

1
commands/who.py
View File

@ -1,4 +1,3 @@
import main
import modules.userinfo as userinfo

6
conf/example/config.json
View File

@ -12,14 +12,12 @@
},
"Key": "key.pem",
"Certificate": "cert.pem",
"RedisSocket": "/var/run/redis/redis.sock",
"RedisDBEphemeral": 2,
"RedisDBPersistent": 3,
"RedisSocket": "/tmp/redis.sock",
"UsePassword": true,
"ConnectOnCreate": false,
"AutoReg": false,
"Debug": false,
"Trace": false,
"Trace", false,
"Relay": {
"Host": "127.0.0.1",
"Port": "201x",

1
conf/example/irc.json
View File

@ -3,7 +3,6 @@
"register": true,
"entity": "NickServ",
"email": "{nickname}@domain.com",
"domains": [],
"registermsg": "REGISTER {password} {email}",
"confirm": "CONFIRM {token}",
"check": true,

118
core/bot.py
View File

@ -2,16 +2,14 @@ from twisted.internet.protocol import ReconnectingClientFactory
from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.internet import reactor, task
from twisted.internet import reactor
from twisted.words.protocols.irc import (
symbolic_to_numeric,
numeric_to_symbolic,
lowDequote,
IRCBadMessage,
)
import sys
from string import digits
from random import randint
from copy import deepcopy
from datetime import datetime
@ -24,19 +22,13 @@ from modules import regproc
from core.relay import sendRelayNotification
from utils.dedup import dedup
from utils.get import getRelay
import main
from utils.logging.log import *
from utils.logging.debug import *
from utils.logging.send import *
from utils.logging.log import log, warn, error
from utils.logging.debug import debug
from utils.logging.send import sendAll
from utils.parsing import parsen
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from utils.deliver_relay_commands import deliverRelayCommands
# Copied from the Twisted source so we can fix a bug
def parsemsg(s):
@ -64,69 +56,6 @@ def parsemsg(s):
return prefix, command, args
class IRCRelay(IRCClient):
def __init__(self, num, relayCommands, user, stage2):
self.isconnected = False
self.buffer = ""
if user == None:
self.user = main.config["Relay"]["User"]
else:
self.user = user.lower()
password = main.config["Relay"]["Password"]
self.nickname = "relay"
self.realname = "relay"
self.username = self.user
self.password = self.user + ":" + password
self.relayCommands = relayCommands
self.num = num
self.stage2 = stage2
self.loop = None
def privmsg(self, user, channel, msg):
nick, ident, host = parsen(user)
for i in main.ZNCErrors:
if i in msg:
error("ZNC issue:", msg)
if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
nick = nick[1:]
if nick in self.relayCommands.keys():
sendAll("[%s] %s -> %s" % (self.num, nick, msg))
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
log("%s: relay password mismatch" % self.num)
sendAll("%s: relay password mismatch" % self.num)
def sendStage2(self):
# [["user", {"sasl": ["message1", "message2"]}], []]
if not len(self.stage2) == 0:
user = self.stage2[0].pop(0)
commands = self.stage2[0].pop(0)
del self.stage2[0]
deliverRelayCommands(self.num, commands, user, self.stage2)
def signedOn(self):
if not self.isconnected:
self.isconnected = True
# log("signed on as a relay: %s" % self.num)
sleeptime = 0
increment = 0.8
for i in self.relayCommands.keys():
for x in self.relayCommands[i]:
reactor.callLater(
sleeptime,
self.msg,
main.config["Tweaks"]["ZNC"]["Prefix"] + i,
x,
)
sleeptime += increment
increment += 0.8
if not self.stage2 == None:
reactor.callLater(sleeptime, self.sendStage2)
reactor.callLater(sleeptime + 5, self.transport.loseConnection)
return
class IRCBot(IRCClient):
def __init__(self, net, num):
self.isconnected = False
@ -188,7 +117,7 @@ class IRCBot(IRCClient):
sleeptime = 0.0
increment = 0.8
for i in channels:
if not i in self.channels:
if i not in self.channels:
if self.net in main.blacklist.keys():
if i in main.blacklist[self.net]:
debug("Not joining blacklisted channel %s on %s - %i" % (i, self.net, self.num))
@ -215,7 +144,7 @@ class IRCBot(IRCClient):
del main.TempChan[self.net]
def event(self, **cast):
if not "ts" in cast.keys():
if "ts" not in cast.keys():
cast["ts"] = str(datetime.now().isoformat())
# remove odd stuff
@ -302,16 +231,16 @@ class IRCBot(IRCClient):
# we have been mentioned in a msg/notice/action/part/quit/topic message
if "msg" in cast.keys(): # Don't highlight queries
if not cast["msg"] == None:
if cast["msg"] is not None:
if self.nickname.lower() in cast["msg"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "highlight"
self.event(**castDup)
if not "net" in cast.keys():
if "net" not in cast.keys():
cast["net"] = self.net
if not "num" in cast.keys():
if "num" not in cast.keys():
cast["num"] = self.num
if not self.authenticated:
regproc.registerTest(cast)
@ -535,7 +464,7 @@ class IRCBot(IRCClient):
allRelays = chankeep.allRelaysActive(self.net)
if allRelays:
name = self.net + "1"
if main.IRCPool[name].wantList == True:
if main.IRCPool[name].wantList is True:
main.IRCPool[name].list(nocheck=True)
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
if self.num == 1: # Only one instance should do a list
@ -559,7 +488,7 @@ class IRCBot(IRCClient):
try:
self.chanlimit = int(chanlimit)
except TypeError:
warn("Invalid chanlimit: %s" % i)
warn("Invalid chanlimit: %s" % chanlimit)
if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's not limited
if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation
@ -625,7 +554,7 @@ class IRCBot(IRCClient):
self.userLeft(prefix, channel, message)
def irc_QUIT(self, prefix, params):
nick = prefix.split("!")[0]
# nick = prefix.split("!")[0]
self.userQuit(prefix, params[0])
def irc_NICK(self, prefix, params):
@ -656,12 +585,12 @@ class IRCBot(IRCClient):
if not sinst:
error(f"Registration ping failed for {self.net} - {self.num}")
return
if not self._negativePass == True:
if negativepass == False:
if self._negativePass is not True:
if negativepass is False:
self._negativePass = False
return
if negativepass == True:
if self._negativePass == None:
if negativepass is True:
if self._negativePass is None:
self._negativePass = True
debug("Positive registration check - %s - %i" % (self.net, self.num))
if sinst["ping"]:
@ -701,7 +630,7 @@ class IRCBot(IRCClient):
reactor.callLater(10, self.regPing)
def joined(self, channel):
if not channel in self.channels:
if channel not in self.channels:
self.channels.append(channel)
self.names(channel).addCallback(self.got_names)
if main.config["Toggles"]["Who"]:
@ -749,7 +678,7 @@ class IRCBot(IRCClient):
# format
return # stop right there sir!
chans = userinfo.getChanList(self.net, cast["nick"])
if chans == None:
if chans is None:
error("No channels returned for chanless event: %s" % cast)
# self.event(**cast) -- no, should NEVER happen
return
@ -780,9 +709,10 @@ class IRCBot(IRCClient):
)
# TODO: strip out relay functionality
class IRCBotFactory(ReconnectingClientFactory):
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
if net == None:
if net is None:
self.num = num
self.net = None
self.name = "relay - %i" % num
@ -801,11 +731,9 @@ class IRCBotFactory(ReconnectingClientFactory):
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
def buildProtocol(self, addr):
if self.relay == False:
if self.relay is False:
entry = IRCBot(self.net, self.num)
main.IRCPool[self.name] = entry
else:
entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
self.client = entry
return entry
@ -813,7 +741,7 @@ class IRCBotFactory(ReconnectingClientFactory):
def clientConnectionLost(self, connector, reason):
if not self.relay:
userinfo.delChannels(self.net, self.client.channels)
if not self.client == None:
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
@ -836,7 +764,7 @@ class IRCBotFactory(ReconnectingClientFactory):
# ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
if not self.client == None:
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []

77
core/factory.py Normal file
View File

@ -0,0 +1,77 @@
class IRCBotFactory(ReconnectingClientFactory):
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
if net is None:
self.num = num
self.net = None
self.name = "relay - %i" % num
self.relay = True
else:
self.name = net + str(num)
self.num = num
self.net = net
self.relay = False
self.client = None
self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
self.factor = main.config["Tweaks"]["Delays"]["Factor"]
self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
def buildProtocol(self, addr):
if self.relay is False:
entry = IRCBot(self.net, self.num)
main.IRCPool[self.name] = entry
else:
entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
self.client = entry
return entry
def clientConnectionLost(self, connector, reason):
if not self.relay:
userinfo.delChannels(self.net, self.client.channels)
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
error = reason.getErrorMessage()
if not self.relay:
log("%s - %i: connection lost: %s" % (self.net, self.num, error))
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "lost",
"message": error,
"ts": ctime,
}
)
self.retry(connector)
# ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
error = reason.getErrorMessage()
log("%s - %i: connection failed: %s" % (self.net, self.num, error))
if not self.relay:
sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "failed",
"message": error,
"ts": ctime,
}
)
self.retry(connector)
# ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

5
core/logstash.py
View File

@ -1,9 +1,8 @@
import logstash
import logging
from json import dumps, loads
from json import dumps
import main
from utils.logging.log import *
logger = None
@ -22,7 +21,7 @@ def init_logstash():
def sendLogstashNotification(text):
if not logger == None:
if logger is not None:
logger.info(dumps(text))
return True
return False

17
core/parser.py
View File

@ -1,6 +1,6 @@
import main
from utils.logging.log import *
from utils.logging.send import *
from utils.logging.log import warn
from utils.logging.send import sendSuccess, sendFailure, sendInfo, incorrectUsage
def parseCommand(addr, authed, data):
@ -12,17 +12,12 @@ def parseCommand(addr, authed, data):
warn("Got connection object with no instance in the address pool")
return
success = lambda data: sendSuccess(addr, data)
failure = lambda data: sendFailure(addr, data)
info = lambda data: sendInfo(addr, data)
success = lambda data: sendSuccess(addr, data) # noqa: E731
failure = lambda data: sendFailure(addr, data) # noqa: E731
info = lambda data: sendInfo(addr, data) # noqa: E731
incUsage = lambda mode: incorrectUsage(addr, mode)
incUsage = lambda mode: incorrectUsage(addr, mode) # noqa: E731
length = len(spl)
if len(spl) > 0:
cmd = spl[0]
else:
failure("No text was sent")
return
if spl[0] in main.CommandMap.keys():
main.CommandMap[spl[0]](addr, authed, data, obj, spl, success, failure, info, incUsage, length)
return

16
core/relay.py
View File

@ -1,9 +1,7 @@
from twisted.internet.protocol import Protocol, Factory, ClientFactory
from twisted.internet.protocol import Protocol, Factory
from json import dumps, loads
from copy import deepcopy
import main
from utils.logging.log import *
from utils.logging.log import log, warn
validTypes = [
"msg",
@ -47,10 +45,10 @@ class Relay(Protocol):
data = data.decode("utf-8", "replace")
try:
parsed = loads(data)
except:
except: # noqa: E722
self.sendErr("MALFORMED")
return
if not "type" in parsed.keys():
if "type" not in parsed.keys():
self.sendErr("NOTYPE")
return
if parsed["type"] == "hello":
@ -87,7 +85,7 @@ class Relay(Protocol):
self.sendErr("NOTLIST")
return
for i in lst:
if not i in validTypes:
if i not in validTypes:
self.sendErr("NONEXISTANT")
return
if i in self.subscriptions:
@ -102,10 +100,10 @@ class Relay(Protocol):
self.sendErr("NOTLIST")
return
for i in lst:
if not i in validTypes:
if i not in validTypes:
self.sendErr("NONEXISTANT")
return
if not i in self.subscriptions:
if i not in self.subscriptions:
self.sendErr("NOTSUBSCRIBED")
return
del self.subscriptions[i]

6
core/server.py
View File

@ -1,6 +1,6 @@
from twisted.internet.protocol import Protocol, Factory, ClientFactory
from twisted.internet.protocol import Protocol, Factory
import main
from utils.logging.log import *
from utils.logging.log import log, warn
from core.parser import parseCommand
@ -9,7 +9,7 @@ class Server(Protocol):
def __init__(self, addr):
self.addr = addr
self.authed = False
if main.config["UsePassword"] == False:
if main.config["UsePassword"] is False:
self.authed = True
def send(self, data):

9
main.py
View File

@ -4,8 +4,6 @@ from redis import StrictRedis
from string import digits
from os import urandom
from utils.logging.log import *
# List of errors ZNC can give us
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
@ -59,11 +57,12 @@ lastMinuteSample = 0
hashKey = urandom(16)
lastEvents = {}
# Get networks that are currently online and dedupliate
def liveNets():
networks = set()
for i in IRCPool.keys():
networks.add("".join([x for x in i if not x in digits]))
networks.add("".join([x for x in i if x not in digits]))
return networks
@ -107,6 +106,6 @@ def initMain():
global r, g
initConf()
r = StrictRedis(
unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]
unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"] # noqa
) # Ephemeral - flushed on quit
g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent
g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # noqa

4
modules/alias.py
View File

@ -15,7 +15,7 @@ def generate_alias():
rand = random.randint(1, 4)
while rand == 1:
split = random.randint(0, len(nick) - 1)
nick = nick[:split] + nick[split + 1 :]
nick = nick[:split] + nick[split + 1 :] # noqa: E203
rand = random.randint(1, 4)
rand = random.randint(1, 3)
if rand == 1 or rand == 4:
@ -53,7 +53,7 @@ def generate_alias():
ident = namebase.split(" ")[0]
ident = ident[:10]
elif rand == 6:
ident = re.sub("\s", "", namebase).lower()
ident = re.sub("\s", "", namebase).lower() # noqa: W605
ident = ident[:10]
realname = nick

6
modules/chankeep.py
View File

@ -1,6 +1,6 @@
import main
from utils.logging.log import *
from utils.logging.debug import *
from utils.logging.log import log, warn, error
from utils.logging.debug import debug
from copy import deepcopy
from math import ceil
import modules.provision
@ -166,7 +166,7 @@ def _initialList(net, num, listinfo, chanlimit):
sigrelay = ceil(siglength / chanlimit)
relay = ceil(listlength / chanlimit)
netbase = "list.%s" % net
# netbase = "list.%s" % net
abase = "analytics.list.%s" % net
p = main.g.pipeline()
p.hset(abase, "mean", mean)

8
modules/counters.py
View File

@ -3,11 +3,11 @@ from twisted.internet.task import LoopingCall
def event(name, eventType):
if not "local" in main.counters.keys():
if "local" not in main.counters.keys():
main.counters["local"] = {}
if not "global" in main.counters.keys():
if "global" not in main.counters.keys():
main.counters["global"] = {}
if not name in main.counters["local"].keys():
if name not in main.counters["local"].keys():
main.counters["local"][name] = {}
if eventType not in main.counters["local"][name].keys():
main.counters["local"][name][eventType] = 0
@ -21,7 +21,7 @@ def event(name, eventType):
def getEvents(name=None):
if name == None:
if name is None:
if "global" in main.counters.keys():
return main.counters["global"]
else:

7
modules/monitor.py
View File

@ -1,11 +1,6 @@
from copy import deepcopy
from json import dumps
import main
from core.relay import sendRelayNotification
from core.logstash import sendLogstashNotification
from modules import userinfo
from modules import regproc
from utils.dedup import dedup
order = [
@ -29,7 +24,7 @@ order = [
def parsemeta(numName, c):
if not "channel" in c.keys():
if "channel" not in c.keys():
c["channel"] = None
# metadata scraping
# need to check if this was received from a relay

7
modules/network.py
View File

@ -1,13 +1,12 @@
from twisted.internet.ssl import DefaultOpenSSLContextFactory
import json
from modules import alias
from modules.chankeep import nukeNetwork
from modules.regproc import needToRegister
from twisted.internet import reactor
from core.bot import IRCBot, IRCBotFactory
from core.bot import IRCBotFactory
import main
from utils.logging.log import *
from utils.logging.log import log
from utils.get import getRelay
@ -41,7 +40,7 @@ class Network:
"registered": registered,
}
password = alias.generate_password()
if not num in main.alias.keys():
if num not in main.alias.keys():
main.alias[num] = alias.generate_alias()
main.saveConf("alias")
self.aliases[num] = {"password": password}

3
modules/provision.py
View File

@ -1,6 +1,5 @@
import main
from utils.deliver_relay_commands import deliverRelayCommands
from utils.logging.log import *
from twisted.internet import reactor
import modules.regproc
@ -64,7 +63,7 @@ def provisionAuthenticationData(num, nick, network, security, auth, password):
def provisionRelay(num, network): # provision user and network data
aliasObj = main.alias[num]
print("ALIASOBJ FALUES", aliasObj.values())
alias = aliasObj["nick"]
# alias = aliasObj["nick"]
provisionUserNetworkData(
num,
*aliasObj.values(),

24
modules/regproc.py
View File

@ -1,7 +1,7 @@
import main
from modules import provision
from utils.logging.log import *
from utils.logging.debug import *
from utils.logging.log import error
from utils.logging.debug import debug
from copy import deepcopy
from random import choice
@ -24,7 +24,7 @@ def selectInst(net):
if net in main.irc.keys():
inst = deepcopy(main.irc[net])
for i in main.irc["_"].keys():
if not i in inst:
if i not in inst:
inst[i] = main.irc["_"][i]
else:
inst = main.irc["_"]
@ -50,7 +50,7 @@ def substitute(net, num, token=None):
error(f"Could not get email for {net} - {num}")
return False
nickname = alias["nick"]
username = nickname + "/" + net
# username = nickname + "/" + net
password = main.network[net].aliases[num]["password"]
# inst["email"] = inst["email"].replace("{nickname}", nickname)
for i in inst.keys():
@ -88,7 +88,7 @@ def confirmRegistration(net, num, negativepass=None):
obj = main.network[net]
name = net + str(num)
if name in main.IRCPool.keys():
if not negativepass == None:
if negativepass is not None:
main.IRCPool[name].regPing(negativepass=negativepass)
return
debug("Relay authenticated: %s - %i" % (net, num))
@ -100,7 +100,7 @@ def confirmRegistration(net, num, negativepass=None):
if main.IRCPool[name]._regAttempt:
try:
main.IRCPool[name]._regAttempt.cancel()
except:
except: # noqa
pass
obj.relays[num]["registered"] = True
main.saveConf("network")
@ -112,22 +112,22 @@ def enableAuthentication(net, num):
security = obj.security
auth = obj.auth
password = obj.aliases[num]["password"]
uname = main.alias[num]["nick"] + "/" + net
# uname = main.alias[num]["nick"] + "/" + net
provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth
main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump")
if selectInst(net)["check"] == False:
if selectInst(net)["check"] is False:
confirmRegistration(net, num)
def registerTest(c):
sinst = substitute(c["net"], c["num"])
name = c["net"] + str(c["num"])
if sinst["check"] == False:
if sinst["check"] is False:
return
if "msg" in c.keys() and not c["msg"] == None:
if "msg" in c.keys() and not c["msg"] is None:
if sinst["negative"]:
if name in main.IRCPool.keys():
if not main.IRCPool[name]._negativePass == True:
if main.IRCPool[name]._negativePass is not True:
if c["type"] == "query" and c["nick"] == sinst["entity"]:
if sinst["checknegativemsg"] in c["msg"]:
confirmRegistration(
@ -150,6 +150,6 @@ def registerTest(c):
elif sinst["checktype"] == "mode":
if c["type"] == "self":
if c["mtype"] == "mode":
if sinst["checkmode"] in c["mode"] and c["status"] == True:
if sinst["checkmode"] in c["mode"] and c["status"] is True:
confirmRegistration(c["net"], c["num"])
return

11
modules/userinfo.py
View File

@ -1,9 +1,8 @@
from twisted.internet.threads import deferToThread
from string import digits
import main
from utils.logging.log import *
from utils.logging.debug import debug, trace
from utils.logging.log import warn
from utils.logging.debug import trace
from utils.parsing import parsen
@ -97,7 +96,7 @@ def _initialUsers(name, channel, users):
def initialUsers(name, channel, users):
trace("Initialising WHO records for %s on %s" % (channel, name))
d = deferToThread(_initialUsers, name, channel, users)
deferToThread(_initialUsers, name, channel, users)
# d.addCallback(testCallback)
@ -114,7 +113,7 @@ def _initialNames(name, channel, names):
def initialNames(name, channel, names):
trace("Initialising NAMES records for %s on %s" % (channel, name))
d = deferToThread(_initialNames, name, channel, names)
deferToThread(_initialNames, name, channel, names)
# d.addCallback(testCallback)
@ -260,5 +259,5 @@ def _delChannels(net, channels):
def delChannels(net, channels): # we have left a channel
trace("Purging channel %s for %s" % (", ".join(channels), net))
d = deferToThread(_delChannels, net, channels)
deferToThread(_delChannels, net, channels)
# d.addCallback(testCallback)

31
threshold
View File

@ -10,33 +10,38 @@ from signal import signal, SIGINT
from sys import stdout, stderr # Import again because we want to override
from codecs import getwriter # fix printing odd shit to the terminal
stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know
stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using
# unicode quit messages for no reason
import main
main.initMain()
from utils.cleanup import handler
signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine
from utils.logging.log import log
from utils.loaders.command_loader import loadCommands
from core.server import ServerFactory
from core.relay import RelayFactory
import modules.counters
import core.logstash
main.initMain()
if "--debug" in sys.argv: # yes really
main.config["Debug"] = True
if "--trace" in sys.argv:
main.config["Trace"] = True
from utils.logging.log import *
from utils.loaders.command_loader import loadCommands
from core.server import Server, ServerFactory
from core.relay import Relay, RelayFactory
import modules.counters
loadCommands()
import core.logstash
core.logstash.init_logstash()
signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine
stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know
stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using
# unicode quit messages for no reason
if __name__ == "__main__":
listener = ServerFactory()
if main.config["Listener"]["UseSSL"] == True:
if main.config["Listener"]["UseSSL"] is True:
reactor.listenSSL(
main.config["Listener"]["Port"],
listener,
@ -59,7 +64,7 @@ if __name__ == "__main__":
log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]))
if main.config["RelayAPI"]["Enabled"]:
relay = RelayFactory()
if main.config["RelayAPI"]["UseSSL"] == True:
if main.config["RelayAPI"]["UseSSL"] is True:
reactor.listenSSL(
main.config["RelayAPI"]["Port"],
relay,

3
utils/cleanup.py
View File

@ -1,8 +1,7 @@
import main
from twisted.internet import reactor
from utils.logging.debug import debug
from utils.logging.log import *
import sys
from utils.logging.log import log
def handler(sig, frame):

2
utils/dedup.py
View File

@ -19,7 +19,7 @@ def dedup(numName, b):
return True
if numName in main.lastEvents.keys():
main.lastEvents[numName].insert(0, castHash)
main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]]
main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] # noqa
else:
main.lastEvents[numName] = [castHash]
return False

159
utils/deliver_relay_commands.py
View File

@ -1,7 +1,162 @@
import main
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from twisted.internet import reactor
from twisted.words.protocols.irc import IRCClient
from twisted.internet.protocol import ReconnectingClientFactory
from utils.parsing import parsen
from utils.logging.log import log, error
from utils.logging.send import sendAll
from modules import userinfo
from datetime import datetime
from core.relay import sendRelayNotification
from utils.get import getRelay
# TODO: strip out non-relay functionality
class IRCRelay(IRCClient):
def __init__(self, num, relayCommands, user, stage2):
self.isconnected = False
self.buffer = ""
if user is None:
self.user = main.config["Relay"]["User"]
else:
self.user = user.lower()
password = main.config["Relay"]["Password"]
self.nickname = "relay"
self.realname = "relay"
self.username = self.user
self.password = self.user + ":" + password
self.relayCommands = relayCommands
self.num = num
self.stage2 = stage2
self.loop = None
def privmsg(self, user, channel, msg):
nick, ident, host = parsen(user)
for i in main.ZNCErrors:
if i in msg:
error("ZNC issue:", msg)
if nick[0] == main.config["Tweaks"]["ZNC"]["Prefix"]:
nick = nick[1:]
if nick in self.relayCommands.keys():
sendAll("[%s] %s -> %s" % (self.num, nick, msg))
def irc_ERR_PASSWDMISMATCH(self, prefix, params):
log("%s: relay password mismatch" % self.num)
sendAll("%s: relay password mismatch" % self.num)
def sendStage2(self):
# [["user", {"sasl": ["message1", "message2"]}], []]
if not len(self.stage2) == 0:
user = self.stage2[0].pop(0)
commands = self.stage2[0].pop(0)
del self.stage2[0]
deliverRelayCommands(self.num, commands, user, self.stage2)
def signedOn(self):
if not self.isconnected:
self.isconnected = True
# log("signed on as a relay: %s" % self.num)
sleeptime = 0
increment = 0.8
for i in self.relayCommands.keys():
for x in self.relayCommands[i]:
reactor.callLater(
sleeptime,
self.msg,
main.config["Tweaks"]["ZNC"]["Prefix"] + i,
x,
)
sleeptime += increment
increment += 0.8
if self.stage2 is not None:
reactor.callLater(sleeptime, self.sendStage2)
reactor.callLater(sleeptime + 5, self.transport.loseConnection)
return
class IRCRelayFactory(ReconnectingClientFactory):
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
if net is None:
self.num = num
self.net = None
self.name = "relay - %i" % num
self.relay = True
else:
self.name = net + str(num)
self.num = num
self.net = net
self.relay = False
self.client = None
self.maxDelay = main.config["Tweaks"]["Delays"]["MaxDelay"]
self.initialDelay = main.config["Tweaks"]["Delays"]["InitialDelay"]
self.factor = main.config["Tweaks"]["Delays"]["Factor"]
self.jitter = main.config["Tweaks"]["Delays"]["Jitter"]
self.relayCommands, self.user, self.stage2 = relayCommands, user, stage2
def buildProtocol(self, addr):
entry = IRCRelay(self.num, self.relayCommands, self.user, self.stage2)
self.client = entry
return entry
def clientConnectionLost(self, connector, reason):
if not self.relay:
userinfo.delChannels(self.net, self.client.channels)
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
error = reason.getErrorMessage()
if not self.relay:
log("%s - %i: connection lost: %s" % (self.net, self.num, error))
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "lost",
"message": error,
"ts": ctime,
}
)
self.retry(connector)
# ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
if self.client is not None:
self.client.isconnected = False
self.client.authenticated = False
self.client.channels = []
error = reason.getErrorMessage()
log("%s - %i: connection failed: %s" % (self.net, self.num, error))
if not self.relay:
sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat())
sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "failed",
"message": error,
"ts": ctime,
}
)
self.retry(connector)
# ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
keyFN = main.certPath + main.config["Key"]
certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace"))
bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
bot = IRCRelayFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
host, port = getRelay(num)
rct = reactor.connectSSL(host, port, bot, contextFactory)
reactor.connectSSL(host, port, bot, contextFactory)

4
utils/get.py
View File

@ -4,8 +4,8 @@ import main
def getRelay(num):
host = main.config["Relay"]["Host"].replace("x", str(num))
port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2)))
user = main.config["Relay"]["User"]
password = main.config["Relay"]["Password"]
# user = main.config["Relay"]["User"]
# password = main.config["Relay"]["Password"]
try:
port = int(port)
except ValueError:

5
utils/loaders/command_loader.py
View File

@ -1,8 +1,7 @@
from os import listdir
from utils.logging.debug import debug
from utils.logging.log import *
import commands
from utils.logging.log import error
from main import CommandMap
@ -14,7 +13,7 @@ def loadCommands(allowDup=False):
className = commandName.capitalize() + "Command"
# try:
module = __import__("commands.%s" % commandName)
if not commandName in CommandMap:
if commandName not in CommandMap:
CommandMap[commandName] = getattr(getattr(module, commandName), className)
debug("Registered command: %s" % commandName)
else:

2
utils/loaders/single_loader.py
View File

@ -3,8 +3,6 @@ from importlib import reload
import sys
from utils.logging.debug import debug
from utils.logging.log import *
import commands
from main import CommandMap

1
utils/logging/debug.py
View File

@ -1,5 +1,6 @@
import main
# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main
# to read the setting
def debug(*data):

2
utils/logging/send.py
View File

@ -32,7 +32,7 @@ def sendAll(data):
def incorrectUsage(addr, mode):
if mode == None:
if mode is None:
sendFailure(addr, "Incorrect usage")
return
if mode in main.help.keys():