Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
pull from: druid
merge into: Pathogen:master
Conversation 1 Commits 263 Files Changed 128 +547 -278
60 changed files with 547 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7c855e09c0 - Show all commits

1
commands/admall.py
View File

@ -1,6 +1,7 @@
import main import main
from core.bot import deliverRelayCommands from core.bot import deliverRelayCommands
class AdmallCommand: class AdmallCommand:
def __init__(self, *args): def __init__(self, *args):
self.admall(*args) self.admall(*args)

3
commands/alias.py
View File

@ -2,6 +2,7 @@ import main
from yaml import dump from yaml import dump
from modules import alias from modules import alias
class AliasCommand: class AliasCommand:
def __init__(self, *args): def __init__(self, *args):
self.alias(*args) self.alias(*args)
@ -21,7 +22,7 @@ class AliasCommand:
if len(main.alias.keys()) == 0: if len(main.alias.keys()) == 0:
nextNum = 1 nextNum = 1
else: else:
nextNum = max(main.alias.keys())+1 nextNum = max(main.alias.keys()) + 1
main.alias[nextNum] = alias.generate_alias() main.alias[nextNum] = alias.generate_alias()
success("Generated new alias: %i" % nextNum) success("Generated new alias: %i" % nextNum)
main.saveConf("alias") main.saveConf("alias")

5
commands/all.py
View File

@ -1,6 +1,7 @@
import main import main
from core.bot import deliverRelayCommands from core.bot import deliverRelayCommands
class AllCommand: class AllCommand:
def __init__(self, *args): def __init__(self, *args):
self.all(*args) self.all(*args)
@ -14,8 +15,8 @@ class AllCommand:
net = main.network[i].relays[x]["net"] net = main.network[i].relays[x]["net"]
alias = main.alias[x]["nick"] alias = main.alias[x]["nick"]
commands = {spl[1]: [" ".join(spl[2:])]} commands = {spl[1]: [" ".join(spl[2:])]}
success("Sending commands to relay %s as user %s" % (num, alias+"/"+net)) success("Sending commands to relay %s as user %s" % (num, alias + "/" + net))
deliverRelayCommands(num, commands, user=alias+"/"+net) deliverRelayCommands(num, commands, user=alias + "/" + net)
return return
else: else:
incUsage("all") incUsage("all")

13
commands/allc.py
View File

@ -1,6 +1,7 @@
import main import main
from core.bot import deliverRelayCommands from core.bot import deliverRelayCommands
class AllcCommand: class AllcCommand:
def __init__(self, *args): def __init__(self, *args):
self.allc(*args) self.allc(*args)
@ -16,9 +17,11 @@ class AllcCommand:
targets.append((i, x)) targets.append((i, x))
elif spl[1] == "alias": elif spl[1] == "alias":
for i in main.network.keys(): for i in main.network.keys():
[targets.append((i, x)) for x in main.alias.keys() if [
main.alias[x]["nick"] == spl[2] and targets.append((i, x))
x in main.network[i].aliases.keys()] for x in main.alias.keys()
if main.alias[x]["nick"] == spl[2] and x in main.network[i].aliases.keys()
]
else: else:
incUsage("allc") incUsage("allc")
return return
@ -30,8 +33,8 @@ class AllcCommand:
num = i[1] num = i[1]
alias = main.alias[num]["nick"] alias = main.alias[num]["nick"]
commands = {spl[3]: [" ".join(spl[4:])]} commands = {spl[3]: [" ".join(spl[4:])]}
success("Sending commands to relay %i as user %s" % (num, alias+"/"+net)) success("Sending commands to relay %i as user %s" % (num, alias + "/" + net))
deliverRelayCommands(num, commands, user=alias+"/"+net) deliverRelayCommands(num, commands, user=alias + "/" + net)
return return
else: else:
incUsage("allc") incUsage("allc")

1
commands/authcheck.py
View File

@ -1,5 +1,6 @@
import main import main
class AuthcheckCommand: class AuthcheckCommand:
def __init__(self, *args): def __init__(self, *args):
self.authcheck(*args) self.authcheck(*args)

1
commands/auto.py
View File

@ -1,6 +1,7 @@
import main import main
from modules import provision from modules import provision
class AutoCommand: class AutoCommand:
def __init__(self, *args): def __init__(self, *args):
self.auto(*args) self.auto(*args)

1
commands/blacklist.py
View File

@ -1,6 +1,7 @@
import main import main
from yaml import dump from yaml import dump
class BlacklistCommand: class BlacklistCommand:
def __init__(self, *args): def __init__(self, *args):
self.blacklist(*args) self.blacklist(*args)

3
commands/chans.py
View File

@ -1,6 +1,7 @@
import main import main
import modules.userinfo as userinfo import modules.userinfo as userinfo
class ChansCommand: class ChansCommand:
def __init__(self, *args): def __init__(self, *args):
self.chans(*args) self.chans(*args)
@ -16,7 +17,7 @@ class ChansCommand:
rtrn += "Matches from: %s" % i rtrn += "Matches from: %s" % i
rtrn += "\n" rtrn += "\n"
for x in result[i]: for x in result[i]:
rtrn += (x) rtrn += x
rtrn += "\n" rtrn += "\n"
rtrn += "\n" rtrn += "\n"
info(rtrn) info(rtrn)

1
commands/cmd.py
View File

@ -1,6 +1,7 @@
import main import main
from core.bot import deliverRelayCommands from core.bot import deliverRelayCommands
class CmdCommand: class CmdCommand:
def __init__(self, *args): def __init__(self, *args):
self.cmd(*args) self.cmd(*args)

1
commands/confirm.py
View File

@ -1,6 +1,7 @@
import main import main
from modules import regproc from modules import regproc
class ConfirmCommand: class ConfirmCommand:
def __init__(self, *args): def __init__(self, *args):
self.confirm(*args) self.confirm(*args)

5
commands/disable.py
View File

@ -1,6 +1,7 @@
import main import main
from core.bot import deliverRelayCommands from core.bot import deliverRelayCommands
class DisableCommand: class DisableCommand:
def __init__(self, *args): def __init__(self, *args):
self.disable(*args) self.disable(*args)
@ -15,7 +16,7 @@ class DisableCommand:
failure("Must be a number, not %s" % spl[2]) failure("Must be a number, not %s" % spl[2])
return return
relayNum = int(spl[2]) relayNum = int(spl[2])
name = spl[1]+spl[2] name = spl[1] + spl[2]
if not spl[1] in main.IRCPool.keys(): if not spl[1] in main.IRCPool.keys():
info("Note - instance not running, proceeding anyway") info("Note - instance not running, proceeding anyway")
if not relayNum in main.network[spl[1]].relays.keys(): if not relayNum in main.network[spl[1]].relays.keys():
@ -26,7 +27,7 @@ class DisableCommand:
network = spl[1] network = spl[1]
relay = main.network[spl[1]].relays[relayNum] relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]} commands = {"status": ["Disconnect"]}
deliverRelayCommands(relayNum, commands, user=user+"/"+network) deliverRelayCommands(relayNum, commands, user=user + "/" + network)
main.saveConf("network") main.saveConf("network")
if name in main.ReactorPool.keys(): if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys(): if name in main.FactoryPool.keys():

1
commands/dist.py
View File

@ -1,6 +1,7 @@
import main import main
from subprocess import run, PIPE from subprocess import run, PIPE
class DistCommand: class DistCommand:
def __init__(self, *args): def __init__(self, *args):
self.dist(*args) self.dist(*args)

3
commands/email.py
View File

@ -1,6 +1,7 @@
import main import main
from yaml import dump from yaml import dump
class EmailCommand: class EmailCommand:
def __init__(self, *args): def __init__(self, *args):
self.email(*args) self.email(*args)
@ -21,7 +22,7 @@ class EmailCommand:
else: else:
failure("Domain already exists in default config: %s" % domain) failure("Domain already exists in default config: %s" % domain)
return return
else: else:
num = int(spl[2]) num = int(spl[2])
if spl[1] == "add": if spl[1] == "add":

5
commands/enable.py
View File

@ -1,6 +1,7 @@
import main import main
from core.bot import deliverRelayCommands from core.bot import deliverRelayCommands
class EnableCommand: class EnableCommand:
def __init__(self, *args): def __init__(self, *args):
self.enable(*args) self.enable(*args)
@ -22,9 +23,9 @@ class EnableCommand:
user = main.alias[int(spl[2])]["nick"] user = main.alias[int(spl[2])]["nick"]
network = spl[1] network = spl[1]
commands = {"status": ["Connect"]} commands = {"status": ["Connect"]}
deliverRelayCommands(int(spl[2]), commands, user=user+"/"+network) deliverRelayCommands(int(spl[2]), commands, user=user + "/" + network)
main.saveConf("network") main.saveConf("network")
if not spl[1]+spl[2] in main.IRCPool.keys(): if not spl[1] + spl[2] in main.IRCPool.keys():
main.network[spl[1]].start_bot(int(spl[2])) main.network[spl[1]].start_bot(int(spl[2]))
else: else:
pass pass

1
commands/exec.py
View File

@ -1,5 +1,6 @@
import main import main
class ExecCommand: class ExecCommand:
def __init__(self, *args): def __init__(self, *args):
self.exec(*args) self.exec(*args)

1
commands/help.py
View File

@ -1,5 +1,6 @@
import main import main
class HelpCommand: class HelpCommand:
def __init__(self, *args): def __init__(self, *args):
self.help(*args) self.help(*args)

9
commands/join.py
View File

@ -1,6 +1,7 @@
import main import main
import modules.chankeep import modules.chankeep
class JoinCommand: class JoinCommand:
def __init__(self, *args): def __init__(self, *args):
self.join(*args) self.join(*args)
@ -21,10 +22,10 @@ class JoinCommand:
if not int(spl[2]) in main.network[spl[1]].relays.keys(): if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay %s does not exist on network %s" % (spl[2], spl[1])) failure("Relay %s does not exist on network %s" % (spl[2], spl[1]))
return return
if not spl[1]+spl[2] in main.IRCPool.keys(): if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1]) failure("Name has no instance: %s" % spl[1])
return return
main.IRCPool[spl[1]+spl[2]].join(spl[3]) main.IRCPool[spl[1] + spl[2]].join(spl[3])
success("Joined %s" % spl[3]) success("Joined %s" % spl[3])
return return
elif length == 5: elif length == 5:
@ -34,10 +35,10 @@ class JoinCommand:
if not int(spl[2]) in main.network[spl[1]].relays.keys(): if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %", (spl[2], spl[1])) failure("Relay % does not exist on network %", (spl[2], spl[1]))
return return
if not spl[1]+spl[2] in main.IRCPool.keys(): if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1]) failure("Name has no instance: %s" % spl[1])
return return
main.IRCPool[spl[1]+spl[2]].join(spl[3], spl[4]) main.IRCPool[spl[1] + spl[2]].join(spl[3], spl[4])
success("Joined %s with key %s" % (spl[3], spl[4])) success("Joined %s with key %s" % (spl[3], spl[4]))
return return
else: else:

9
commands/list.py
View File

@ -1,5 +1,6 @@
import main import main
class ListCommand: class ListCommand:
def __init__(self, *args): def __init__(self, *args):
self.list(*args) self.list(*args)
@ -11,10 +12,10 @@ class ListCommand:
if not 1 in main.network[i].relays.keys(): if not 1 in main.network[i].relays.keys():
info("Network has no first instance: %s" % i) info("Network has no first instance: %s" % i)
continue continue
if not i+"1" in main.IRCPool.keys(): if not i + "1" in main.IRCPool.keys():
info("No IRC instance: %s - 1" % i) info("No IRC instance: %s - 1" % i)
continue continue
main.IRCPool[i+"1"].list() main.IRCPool[i + "1"].list()
success("Requested list with first instance of %s" % i) success("Requested list with first instance of %s" % i)
return return
elif length == 2: elif length == 2:
@ -24,10 +25,10 @@ class ListCommand:
if not 1 in main.network[spl[1]].relays.keys(): if not 1 in main.network[spl[1]].relays.keys():
failure("Network has no first instance") failure("Network has no first instance")
return return
if not spl[1]+"1" in main.IRCPool.keys(): if not spl[1] + "1" in main.IRCPool.keys():
failure("No IRC instance: %s - 1" % spl[1]) failure("No IRC instance: %s - 1" % spl[1])
return return
main.IRCPool[spl[1]+"1"].list() main.IRCPool[spl[1] + "1"].list()
success("Requested list with first instance of %s" % spl[1]) success("Requested list with first instance of %s" % spl[1])
return return
else: else:

1
commands/load.py
View File

@ -1,5 +1,6 @@
import main import main
class LoadCommand: class LoadCommand:
def __init__(self, *args): def __init__(self, *args):
self.load(*args) self.load(*args)

1
commands/loadmod.py
View File

@ -1,6 +1,7 @@
import main import main
from utils.loaders.single_loader import loadSingle from utils.loaders.single_loader import loadSingle
class LoadmodCommand: class LoadmodCommand:
def __init__(self, *args): def __init__(self, *args):
self.loadmod(*args) self.loadmod(*args)

1
commands/logout.py
View File

@ -1,5 +1,6 @@
import main import main
class LogoutCommand: class LogoutCommand:
def __init__(self, *args): def __init__(self, *args):
self.logout(*args) self.logout(*args)

3
commands/mod.py
View File

@ -1,6 +1,7 @@
import main import main
from yaml import dump from yaml import dump
class ModCommand: class ModCommand:
# This could be greatly improved, but not really important right now # This could be greatly improved, but not really important right now
def __init__(self, *args): def __init__(self, *args):
@ -23,7 +24,7 @@ class ModCommand:
success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1])) success("Successfully set key %s to %s on %s" % (spl[2], spl[3], spl[1]))
return return
# Find a better way to do this # Find a better way to do this
#elif length == 6: # elif length == 6:
# if not spl[1] in main.network.keys(): # if not spl[1] in main.network.keys():
# failure("Network does not exist: %s" % spl[1]) # failure("Network does not exist: %s" % spl[1])
# return # return

7
commands/msg.py
View File

@ -1,5 +1,6 @@
import main import main
class MsgCommand: class MsgCommand:
def __init__(self, *args): def __init__(self, *args):
self.msg(*args) self.msg(*args)
@ -13,12 +14,12 @@ class MsgCommand:
if not int(spl[2]) in main.network[spl[1]].relays.keys(): if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %" % (spl[2], spl[1])) failure("Relay % does not exist on network %" % (spl[2], spl[1]))
return return
if not spl[1]+spl[2] in main.IRCPool.keys(): if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1]) failure("Name has no instance: %s" % spl[1])
return return
if not spl[3] in main.IRCPool[spl[1]+spl[2]].channels: if not spl[3] in main.IRCPool[spl[1] + spl[2]].channels:
info("Bot not on channel: %s" % spl[3]) info("Bot not on channel: %s" % spl[3])
main.IRCPool[spl[1]+spl[2]].msg(spl[3], " ".join(spl[4:])) main.IRCPool[spl[1] + spl[2]].msg(spl[3], " ".join(spl[4:]))
success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1])) success("Sent %s to %s on relay %s on network %s" % (" ".join(spl[4:]), spl[3], spl[2], spl[1]))
return return
else: else:

3
commands/network.py
View File

@ -3,6 +3,7 @@ from yaml import dump
from modules.network import Network from modules.network import Network
from string import digits from string import digits
class NetworkCommand: class NetworkCommand:
def __init__(self, *args): def __init__(self, *args):
self.network(*args) self.network(*args)
@ -38,7 +39,7 @@ class NetworkCommand:
elif length == 3: elif length == 3:
if spl[1] == "del": if spl[1] == "del":
if spl[2] in main.network.keys(): if spl[2] in main.network.keys():
main.network[spl[2]].seppuku() # ;( main.network[spl[2]].seppuku() # ;(
del main.network[spl[2]] del main.network[spl[2]]
success("Successfully removed network: %s" % spl[2]) success("Successfully removed network: %s" % spl[2])
main.saveConf("network") main.saveConf("network")

7
commands/part.py
View File

@ -1,5 +1,6 @@
import main import main
class PartCommand: class PartCommand:
def __init__(self, *args): def __init__(self, *args):
self.part(*args) self.part(*args)
@ -13,10 +14,10 @@ class PartCommand:
if not int(spl[2]) in main.network[spl[1]].relays.keys(): if not int(spl[2]) in main.network[spl[1]].relays.keys():
failure("Relay % does not exist on network %", (spl[2], spl[1])) failure("Relay % does not exist on network %", (spl[2], spl[1]))
return return
if not spl[1]+spl[2] in main.IRCPool.keys(): if not spl[1] + spl[2] in main.IRCPool.keys():
failure("Name has no instance: %s" % spl[1]+spl[2]) failure("Name has no instance: %s" % spl[1] + spl[2])
return return
main.IRCPool[spl[1]+spl[2]].part(spl[3]) main.IRCPool[spl[1] + spl[2]].part(spl[3])
success("Left %s" % spl[3]) success("Left %s" % spl[3])
return return
else: else:

1
commands/pass.py
View File

@ -1,5 +1,6 @@
import main import main
class PassCommand: class PassCommand:
def __init__(self, *args): def __init__(self, *args):
self.password(*args) self.password(*args)

1
commands/pending.py
View File

@ -1,5 +1,6 @@
import main import main
class PendingCommand: class PendingCommand:
def __init__(self, *args): def __init__(self, *args):
self.pending(*args) self.pending(*args)

1
commands/recheckauth.py
View File

@ -1,5 +1,6 @@
import main import main
class RecheckauthCommand: class RecheckauthCommand:
def __init__(self, *args): def __init__(self, *args):
self.recheckauth(*args) self.recheckauth(*args)

1
commands/reg.py
View File

@ -1,6 +1,7 @@
import main import main
from modules import regproc from modules import regproc
class RegCommand: class RegCommand:
def __init__(self, *args): def __init__(self, *args):
self.reg(*args) self.reg(*args)

1
commands/relay.py
View File

@ -1,6 +1,7 @@
import main import main
from yaml import dump from yaml import dump
class RelayCommand: class RelayCommand:
def __init__(self, *args): def __init__(self, *args):
self.relay(*args) self.relay(*args)

1
commands/save.py
View File

@ -1,5 +1,6 @@
import main import main
class SaveCommand: class SaveCommand:
def __init__(self, *args): def __init__(self, *args):
self.save(*args) self.save(*args)

1
commands/stats.py
View File

@ -3,6 +3,7 @@ import modules.counters as count
import modules.userinfo as userinfo import modules.userinfo as userinfo
from string import digits from string import digits
class StatsCommand: class StatsCommand:
def __init__(self, *args): def __init__(self, *args):
self.stats(*args) self.stats(*args)

1
commands/swho.py
View File

@ -1,5 +1,6 @@
import main import main
class SwhoCommand: class SwhoCommand:
def __init__(self, *args): def __init__(self, *args):
self.swho(*args) self.swho(*args)

12
commands/token.py
View File

@ -2,6 +2,7 @@ import main
from yaml import dump from yaml import dump
from uuid import uuid4 from uuid import uuid4
class TokenCommand: class TokenCommand:
def __init__(self, *args): def __init__(self, *args):
self.token(*args) self.token(*args)
@ -31,11 +32,12 @@ class TokenCommand:
elif length == 4: elif length == 4:
if spl[1] == "add": if spl[1] == "add":
if not spl[2] in main.tokens.keys(): if not spl[2] in main.tokens.keys():
if spl[3] in ["relay"]: # more to come! if spl[3] in ["relay"]: # more to come!
main.tokens[spl[2]] = {"hello": str(uuid4()), main.tokens[spl[2]] = {
"usage": spl[3], "hello": str(uuid4()),
"counter": str(uuid4()), "usage": spl[3],
} "counter": str(uuid4()),
}
main.saveConf("tokens") main.saveConf("tokens")
success("Successfully created token %s:" % spl[2]) success("Successfully created token %s:" % spl[2])
info(dump(main.tokens[spl[2]])) info(dump(main.tokens[spl[2]]))

3
commands/users.py
View File

@ -1,6 +1,7 @@
import main import main
import modules.userinfo as userinfo import modules.userinfo as userinfo
class UsersCommand: class UsersCommand:
def __init__(self, *args): def __init__(self, *args):
self.users(*args) self.users(*args)
@ -16,7 +17,7 @@ class UsersCommand:
rtrn += "Matches from: %s" % i rtrn += "Matches from: %s" % i
rtrn += "\n" rtrn += "\n"
for x in result[i]: for x in result[i]:
rtrn += (x) rtrn += x
rtrn += "\n" rtrn += "\n"
rtrn += "\n" rtrn += "\n"
info(rtrn) info(rtrn)

3
commands/who.py
View File

@ -1,6 +1,7 @@
import main import main
import modules.userinfo as userinfo import modules.userinfo as userinfo
class WhoCommand: class WhoCommand:
def __init__(self, *args): def __init__(self, *args):
self.who(*args) self.who(*args)
@ -14,7 +15,7 @@ class WhoCommand:
rtrn += "Matches from: %s" % i rtrn += "Matches from: %s" % i
rtrn += "\n" rtrn += "\n"
for x in result[i]: for x in result[i]:
rtrn += (x) rtrn += x
rtrn += "\n" rtrn += "\n"
rtrn += "\n" rtrn += "\n"
info(rtrn) info(rtrn)

220
core/bot.py
View File

@ -3,7 +3,12 @@ from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from twisted.internet import reactor, task from twisted.internet import reactor, task
from twisted.words.protocols.irc import symbolic_to_numeric, numeric_to_symbolic, lowDequote, IRCBadMessage from twisted.words.protocols.irc import (
symbolic_to_numeric,
numeric_to_symbolic,
lowDequote,
IRCBadMessage,
)
import sys import sys
from string import digits from string import digits
@ -29,16 +34,15 @@ from utils.parsing import parsen
from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.ssl import DefaultOpenSSLContextFactory
def deliverRelayCommands(num, relayCommands, user=None, stage2=None): def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
keyFN = main.certPath+main.config["Key"] keyFN = main.certPath + main.config["Key"]
certFN = main.certPath+main.config["Certificate"] certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace"))
certFN.encode("utf-8", "replace"))
bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2) bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
host, port = getRelay(num) host, port = getRelay(num)
rct = reactor.connectSSL(host, rct = reactor.connectSSL(host, port, bot, contextFactory)
port,
bot, contextFactory)
# Copied from the Twisted source so we can fix a bug # Copied from the Twisted source so we can fix a bug
def parsemsg(s): def parsemsg(s):
@ -50,21 +54,22 @@ def parsemsg(s):
@return: A tuple of (prefix, command, args). @return: A tuple of (prefix, command, args).
@rtype: L{tuple} @rtype: L{tuple}
""" """
prefix = '' prefix = ""
trailing = [] trailing = []
if not s: if not s:
raise IRCBadMessage("Empty line.") raise IRCBadMessage("Empty line.")
if s[0:1] == ':': if s[0:1] == ":":
prefix, s = s[1:].split(' ', 1) prefix, s = s[1:].split(" ", 1)
if s.find(' :') != -1: if s.find(" :") != -1:
s, trailing = s.split(' :', 1) s, trailing = s.split(" :", 1)
args = s.split(' ') # Twisted bug fixed by adding an argument to split() args = s.split(" ") # Twisted bug fixed by adding an argument to split()
args.append(trailing) args.append(trailing)
else: else:
args = s.split(' ') # And again args = s.split(" ") # And again
command = args.pop(0) command = args.pop(0)
return prefix, command, args return prefix, command, args
class IRCRelay(IRCClient): class IRCRelay(IRCClient):
def __init__(self, num, relayCommands, user, stage2): def __init__(self, num, relayCommands, user, stage2):
self.isconnected = False self.isconnected = False
@ -77,7 +82,7 @@ class IRCRelay(IRCClient):
self.nickname = "relay" self.nickname = "relay"
self.realname = "relay" self.realname = "relay"
self.username = self.user self.username = self.user
self.password = self.user+":"+password self.password = self.user + ":" + password
self.relayCommands = relayCommands self.relayCommands = relayCommands
self.num = num self.num = num
@ -109,19 +114,25 @@ class IRCRelay(IRCClient):
def signedOn(self): def signedOn(self):
if not self.isconnected: if not self.isconnected:
self.isconnected = True self.isconnected = True
#log("signed on as a relay: %s" % self.num) # log("signed on as a relay: %s" % self.num)
sleeptime = 0 sleeptime = 0
increment = 0.8 increment = 0.8
for i in self.relayCommands.keys(): for i in self.relayCommands.keys():
for x in self.relayCommands[i]: for x in self.relayCommands[i]:
reactor.callLater(sleeptime, self.msg, main.config["Tweaks"]["ZNC"]["Prefix"]+i, x) reactor.callLater(
sleeptime,
self.msg,
main.config["Tweaks"]["ZNC"]["Prefix"] + i,
x,
)
sleeptime += increment sleeptime += increment
increment += 0.8 increment += 0.8
if not self.stage2 == None: if not self.stage2 == None:
reactor.callLater(sleeptime, self.sendStage2) reactor.callLater(sleeptime, self.sendStage2)
reactor.callLater(sleeptime+5, self.transport.loseConnection) reactor.callLater(sleeptime + 5, self.transport.loseConnection)
return return
class IRCBot(IRCClient): class IRCBot(IRCClient):
def __init__(self, net, num): def __init__(self, net, num):
self.isconnected = False self.isconnected = False
@ -135,26 +146,26 @@ class IRCBot(IRCClient):
relay = main.network[self.net].relays[num] relay = main.network[self.net].relays[num]
self.nickname = alias["nick"] self.nickname = alias["nick"]
self.realname = alias["realname"] self.realname = alias["realname"]
self.username = alias["nick"].lower()+"/"+relay["net"] self.username = alias["nick"].lower() + "/" + relay["net"]
self.password = main.config["Relay"]["Password"] self.password = main.config["Relay"]["Password"]
self.userinfo = None # self.userinfo = None #
self.fingerReply = None # self.fingerReply = None #
self.versionName = None # Don't give out information self.versionName = None # Don't give out information
self.versionNum = None # self.versionNum = None #
self.versionEnv = None # self.versionEnv = None #
self.sourceURL = None # self.sourceURL = None #
self._getWho = {} # LoopingCall objects -- needed to be able to stop them self._getWho = {} # LoopingCall objects -- needed to be able to stop them
self._tempWho = {} # temporary storage for gathering WHO info self._tempWho = {} # temporary storage for gathering WHO info
self._tempNames = {} # temporary storage for gathering NAMES info self._tempNames = {} # temporary storage for gathering NAMES info
self._tempList = ([], []) # temporary storage for gathering LIST info self._tempList = ([], []) # temporary storage for gathering LIST info
self.listOngoing = False # we are currently receiving a LIST self.listOngoing = False # we are currently receiving a LIST
self.listRetried = False # we asked and got nothing so asked again self.listRetried = False # we asked and got nothing so asked again
self.listAttempted = False # we asked for a list self.listAttempted = False # we asked for a list
self.listSimple = False # after asking again we got the list, so use the simple self.listSimple = False # after asking again we got the list, so use the simple
# syntax from now on # syntax from now on
self.wantList = False # we want to send a LIST, but not all relays are active yet self.wantList = False # we want to send a LIST, but not all relays are active yet
self.chanlimit = 0 self.chanlimit = 0
self.prefix = {} self.prefix = {}
self.servername = None self.servername = None
@ -214,8 +225,8 @@ class IRCBot(IRCClient):
cast["time"] = str(datetime.now().isoformat()) cast["time"] = str(datetime.now().isoformat())
# remove odd stuff # remove odd stuff
for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over
if cast[i] == "": # a dictionary that changes length with each iteration if cast[i] == "": # a dictionary that changes length with each iteration
del cast[i] del cast[i]
# remove server stuff # remove server stuff
if "muser" in cast.keys(): if "muser" in cast.keys():
@ -259,11 +270,11 @@ class IRCBot(IRCClient):
if "channel" in cast.keys(): if "channel" in cast.keys():
if cast["type"] == "mode": if cast["type"] == "mode":
if cast["channel"].lower() == self.nickname.lower(): if cast["channel"].lower() == self.nickname.lower():
#castDup = deepcopy(cast) # castDup = deepcopy(cast)
cast["mtype"] = cast["type"] cast["mtype"] = cast["type"]
cast["type"] = "self" cast["type"] = "self"
#self.event(**castDup) # self.event(**castDup)
if cast["modearg"]: # check if modearg is non-NoneType if cast["modearg"]: # check if modearg is non-NoneType
if self.nickname.lower() == cast["modearg"].lower(): if self.nickname.lower() == cast["modearg"].lower():
castDup = deepcopy(cast) castDup = deepcopy(cast)
castDup["mtype"] = cast["type"] castDup["mtype"] = cast["type"]
@ -273,7 +284,7 @@ class IRCBot(IRCClient):
if cast["channel"].lower() == self.nickname.lower(): if cast["channel"].lower() == self.nickname.lower():
cast["mtype"] = cast["type"] cast["mtype"] = cast["type"]
cast["type"] = "query" cast["type"] = "query"
#self.event(**castDup) # self.event(**castDup)
# Don't call self.event for this one because queries are not events on a # Don't call self.event for this one because queries are not events on a
# channel, but we still want to see them # channel, but we still want to see them
@ -296,7 +307,7 @@ class IRCBot(IRCClient):
castDup["type"] = "self" castDup["type"] = "self"
# we have been mentioned in a msg/notice/action/part/quit/topic message # we have been mentioned in a msg/notice/action/part/quit/topic message
if "msg" in cast.keys(): # Don't highlight queries if "msg" in cast.keys(): # Don't highlight queries
if not cast["msg"] == None: if not cast["msg"] == None:
if self.nickname.lower() in cast["msg"].lower(): if self.nickname.lower() in cast["msg"].lower():
castDup = deepcopy(cast) castDup = deepcopy(cast)
@ -373,7 +384,16 @@ class IRCBot(IRCClient):
return return
n = self._tempWho[channel][1] n = self._tempWho[channel][1]
n.append([nick, nick, host, server, status, realname]) n.append([nick, nick, host, server, status, realname])
self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, channel=channel, server=server, status=status) self.event(
type="who",
nick=nick,
ident=ident,
host=host,
realname=realname,
channel=channel,
server=server,
status=status,
)
def irc_RPL_ENDOFWHO(self, prefix, params): def irc_RPL_ENDOFWHO(self, prefix, params):
channel = params[1] channel = params[1]
@ -390,7 +410,10 @@ class IRCBot(IRCClient):
def sanit(self, data): def sanit(self, data):
if len(data) >= 1: if len(data) >= 1:
if data[0] in self.prefix.keys(): if data[0] in self.prefix.keys():
return (self.prefix[data[0]], data[1:]) # would use a set but it's possible these are the same return (
self.prefix[data[0]],
data[1:],
) # would use a set but it's possible these are the same
return (None, data) return (None, data)
else: else:
return (None, False) return (None, False)
@ -405,7 +428,7 @@ class IRCBot(IRCClient):
def irc_RPL_NAMREPLY(self, prefix, params): def irc_RPL_NAMREPLY(self, prefix, params):
channel = params[2] channel = params[2]
nicklist = params[3].split(' ') nicklist = params[3].split(" ")
if channel not in self._tempNames: if channel not in self._tempNames:
return return
n = self._tempNames[channel][1] n = self._tempNames[channel][1]
@ -438,7 +461,7 @@ class IRCBot(IRCClient):
self._tempList[0].append(d) self._tempList[0].append(d)
if self.listSimple: if self.listSimple:
self.sendLine("LIST") self.sendLine("LIST")
return d # return early if we know what to do return d # return early if we know what to do
if noargs: if noargs:
self.sendLine("LIST") self.sendLine("LIST")
@ -460,8 +483,8 @@ class IRCBot(IRCClient):
return return
else: else:
if nocheck: if nocheck:
allRelays = True # override the system - if this is allRelays = True # override the system - if this is
else: # specified, we already did this else: # specified, we already did this
allRelays = chankeep.allRelaysActive(self.net) allRelays = chankeep.allRelaysActive(self.net)
if not allRelays: if not allRelays:
self.wantList = True self.wantList = True
@ -481,7 +504,7 @@ class IRCBot(IRCClient):
self._tempList[1].append([channel, users, topic]) self._tempList[1].append([channel, users, topic])
def irc_RPL_LISTEND(self, prefix, params): def irc_RPL_LISTEND(self, prefix, params):
if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there if not len(self._tempList[0]) > 0: # there are no callbacks, can't do anything there
debug("We didn't ask for this LIST, discarding") debug("We didn't ask for this LIST, discarding")
self._tempList[0].clear() self._tempList[0].clear()
self._tempList[1].clear() self._tempList[1].clear()
@ -510,18 +533,18 @@ class IRCBot(IRCClient):
self.listSimple = True self.listSimple = True
def got_list(self, listinfo): def got_list(self, listinfo):
if len(listinfo) == 0: # probably ngircd not supporting LIST >0 if len(listinfo) == 0: # probably ngircd not supporting LIST >0
return return
chankeep.initialList(self.net, self.num, listinfo, self.chanlimit) chankeep.initialList(self.net, self.num, listinfo, self.chanlimit)
def recheckList(self): def recheckList(self):
allRelays = chankeep.allRelaysActive(self.net) allRelays = chankeep.allRelaysActive(self.net)
if allRelays: if allRelays:
name = self.net+"1" name = self.net + "1"
if main.IRCPool[name].wantList == True: if main.IRCPool[name].wantList == True:
main.IRCPool[name].list(nocheck=True) main.IRCPool[name].list(nocheck=True)
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num)) debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
if self.num == 1: # Only one instance should do a list if self.num == 1: # Only one instance should do a list
if self.chanlimit: if self.chanlimit:
if allRelays: if allRelays:
self.list() self.list()
@ -532,24 +555,26 @@ class IRCBot(IRCClient):
self.checkChannels() self.checkChannels()
def seed_chanlimit(self, chanlimit): def seed_chanlimit(self, chanlimit):
if not main.network[self.net].relays[self.num]["registered"]: #TODO: add check for register request sent, only send it once if not main.network[self.net].relays[self.num][
"registered"
]: # TODO: add check for register request sent, only send it once
if main.config["AutoReg"]: if main.config["AutoReg"]:
self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num) self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num)
#regproc.registerAccount(self.net, self.num) # regproc.registerAccount(self.net, self.num)
try: try:
self.chanlimit = int(chanlimit) self.chanlimit = int(chanlimit)
except TypeError: except TypeError:
warn("Invalid chanlimit: %s" % i) warn("Invalid chanlimit: %s" % i)
if self.chanlimit == 0: if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's not limited self.chanlimit = 200 # don't take the piss if it's not limited
if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation
self.recheckList() self.recheckList()
def seed_prefix(self, prefix): def seed_prefix(self, prefix):
prefix = prefix.replace(")", "") prefix = prefix.replace(")", "")
prefix = prefix.replace("(", "") prefix = prefix.replace("(", "")
length = len(prefix) length = len(prefix)
half = int(length/2) half = int(length / 2)
prefixToMode = dict(zip(prefix[half:], prefix[:half])) prefixToMode = dict(zip(prefix[half:], prefix[:half]))
self.prefix = prefixToMode self.prefix = prefixToMode
@ -585,7 +610,7 @@ class IRCBot(IRCClient):
# the hostname and other useful information in the functions # the hostname and other useful information in the functions
# that these call by default # that these call by default
def irc_JOIN(self, prefix, params): def irc_JOIN(self, prefix, params):
nick = prefix.split('!')[0] nick = prefix.split("!")[0]
channel = params[-1] channel = params[-1]
if nick == self.nickname: if nick == self.nickname:
self.joined(channel) self.joined(channel)
@ -593,7 +618,7 @@ class IRCBot(IRCClient):
self.userJoined(prefix, channel) self.userJoined(prefix, channel)
def irc_PART(self, prefix, params): def irc_PART(self, prefix, params):
nick = prefix.split('!')[0] nick = prefix.split("!")[0]
channel = params[0] channel = params[0]
if len(params) >= 2: if len(params) >= 2:
message = params[1] message = params[1]
@ -605,11 +630,11 @@ class IRCBot(IRCClient):
self.userLeft(prefix, channel, message) self.userLeft(prefix, channel, message)
def irc_QUIT(self, prefix, params): def irc_QUIT(self, prefix, params):
nick = prefix.split('!')[0] nick = prefix.split("!")[0]
self.userQuit(prefix, params[0]) self.userQuit(prefix, params[0])
def irc_NICK(self, prefix, params): def irc_NICK(self, prefix, params):
nick = prefix.split('!', 1)[0] nick = prefix.split("!", 1)[0]
if nick == self.nickname: if nick == self.nickname:
self.nickChanged(prefix, params[0]) self.nickChanged(prefix, params[0])
else: else:
@ -626,6 +651,7 @@ class IRCBot(IRCClient):
channel = params[0] channel = params[0]
newtopic = params[1] newtopic = params[1]
self.topicUpdated(prefix, channel, newtopic) self.topicUpdated(prefix, channel, newtopic)
# End of Twisted hackery # End of Twisted hackery
def regPing(self, negativepass=None): def regPing(self, negativepass=None):
@ -664,7 +690,15 @@ class IRCBot(IRCClient):
def signedOn(self): def signedOn(self):
log("signed on: %s - %i" % (self.net, self.num)) log("signed on: %s - %i" % (self.net, self.num))
ctime = str(datetime.now().isoformat()) ctime = str(datetime.now().isoformat())
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon", "time": ctime}) sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "signedon",
"time": ctime,
}
)
if not self.authenticated: if not self.authenticated:
reactor.callLater(10, self.regPing) reactor.callLater(10, self.regPing)
@ -677,7 +711,7 @@ class IRCBot(IRCClient):
self._getWho[channel] = lc self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"] intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
minint = main.config["Tweaks"]["Delays"]["WhoLoop"] minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
interval = randint(minint, minint+intrange) interval = randint(minint, minint + intrange)
lc.start(interval) lc.start(interval)
def botLeft(self, channel): def botLeft(self, channel):
@ -687,10 +721,11 @@ class IRCBot(IRCClient):
lc = self._getWho[channel] lc = self._getWho[channel]
lc.stop() lc.stop()
del self._getWho[channel] del self._getWho[channel]
userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this userinfo.delChannels(self.net, [channel]) # < we do not need to deduplicate this
#log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # log("Can no longer cover %s, removing records" % channel)# as it will only be matched once --
# other bots have different nicknames so # other bots have different nicknames so
def left(self, user, channel, message): # even if they saw it, they wouldn't react
def left(self, user, channel, message): # even if they saw it, they wouldn't react
self.event(type="part", muser=user, channel=channel, msg=message) self.event(type="part", muser=user, channel=channel, msg=message)
self.botLeft(channel) self.botLeft(channel)
@ -711,14 +746,14 @@ class IRCBot(IRCClient):
def chanlessEvent(self, cast): def chanlessEvent(self, cast):
cast["time"] = str(datetime.now().isoformat()) cast["time"] = str(datetime.now().isoformat())
cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"]) cast["nick"], cast["ident"], cast["host"] = parsen(cast["muser"])
if dedup(self.name, cast): # Needs to be kept self.name until the dedup if dedup(self.name, cast): # Needs to be kept self.name until the dedup
# function is converted to the new net, num # function is converted to the new net, num
# format # format
return # stop right there sir! return # stop right there sir!
chans = userinfo.getChanList(self.net, cast["nick"]) chans = userinfo.getChanList(self.net, cast["nick"])
if chans == None: if chans == None:
error("No channels returned for chanless event: %s" % cast) error("No channels returned for chanless event: %s" % cast)
# self.event(**cast) -- no, should NEVER happen # self.event(**cast) -- no, should NEVER happen
return return
# getChansSingle returns all channels of the user, we only want to use # getChansSingle returns all channels of the user, we only want to use
# ones we have common with them # ones we have common with them
@ -737,7 +772,15 @@ class IRCBot(IRCClient):
argList = list(args) argList = list(args)
modeList = [i for i in modes] modeList = [i for i in modes]
for a, m in zip(argList, modeList): for a, m in zip(argList, modeList):
self.event(type="mode", muser=user, channel=channel, mode=m, status=toset, modearg=a) self.event(
type="mode",
muser=user,
channel=channel,
mode=m,
status=toset,
modearg=a,
)
class IRCBotFactory(ReconnectingClientFactory): class IRCBotFactory(ReconnectingClientFactory):
def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None): def __init__(self, net, num=None, relayCommands=None, user=None, stage2=None):
@ -747,7 +790,7 @@ class IRCBotFactory(ReconnectingClientFactory):
self.name = "relay - %i" % num self.name = "relay - %i" % num
self.relay = True self.relay = True
else: else:
self.name = net+str(num) self.name = net + str(num)
self.num = num self.num = num
self.net = net self.net = net
self.relay = False self.relay = False
@ -781,9 +824,18 @@ class IRCBotFactory(ReconnectingClientFactory):
log("%s - %i: connection lost: %s" % (self.net, self.num, error)) log("%s - %i: connection lost: %s" % (self.net, self.num, error))
sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error)) sendAll("%s - %i: connection lost: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat()) ctime = str(datetime.now().isoformat())
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "lost", "message": error, "time": ctime}) sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "lost",
"message": error,
"time": ctime,
}
)
self.retry(connector) self.retry(connector)
#ReconnectingClientFactory.clientConnectionLost(self, connector, reason) # ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason): def clientConnectionFailed(self, connector, reason):
if not self.client == None: if not self.client == None:
@ -795,7 +847,15 @@ class IRCBotFactory(ReconnectingClientFactory):
if not self.relay: if not self.relay:
sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error)) sendAll("%s - %s: connection failed: %s" % (self.net, self.num, error))
ctime = str(datetime.now().isoformat()) ctime = str(datetime.now().isoformat())
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "failed", "message": error, "time": ctime}) sendRelayNotification(
{
"type": "conn",
"net": self.net,
"num": self.num,
"status": "failed",
"message": error,
"time": ctime,
}
)
self.retry(connector) self.retry(connector)
#ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) # ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

13
core/logstash.py
View File

@ -6,11 +6,20 @@ import main
from utils.logging.log import * from utils.logging.log import *
logger = None logger = None
def init_logstash(): def init_logstash():
global logger global logger
logger = logging.getLogger('ingest') logger = logging.getLogger("ingest")
logger.setLevel(logging.INFO) logger.setLevel(logging.INFO)
logger.addHandler(logstash.TCPLogstashHandler(main.config["Logstash"]["Host"], int(main.config["Logstash"]["Port"]), version=1)) logger.addHandler(
logstash.TCPLogstashHandler(
main.config["Logstash"]["Host"],
int(main.config["Logstash"]["Port"]),
version=1,
)
)
def sendLogstashNotification(text): def sendLogstashNotification(text):
if not logger == None: if not logger == None:

3
core/parser.py
View File

@ -2,8 +2,9 @@ import main
from utils.logging.log import * from utils.logging.log import *
from utils.logging.send import * from utils.logging.send import *
def parseCommand(addr, authed, data): def parseCommand(addr, authed, data):
#call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length) # call command modules with: (addr, authed, data, spl, success, failure, info, incUsage, length)
spl = data.split() spl = data.split()
if addr in main.connections.keys(): if addr in main.connections.keys():
obj = main.connections[addr] obj = main.connections[addr]

24
core/relay.py
View File

@ -5,7 +5,25 @@ from copy import deepcopy
import main import main
from utils.logging.log import * from utils.logging.log import *
validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight"] validTypes = [
"msg",
"notice",
"action",
"who",
"part",
"join",
"kick",
"quit",
"nick",
"topic",
"mode",
"conn",
"znc",
"query",
"self",
"highlight",
]
class Relay(Protocol): class Relay(Protocol):
def __init__(self, addr): def __init__(self, addr):
@ -108,7 +126,7 @@ class Relay(Protocol):
def connectionMade(self): def connectionMade(self):
log("Relay connection from %s:%s" % (self.addr.host, self.addr.port)) log("Relay connection from %s:%s" % (self.addr.host, self.addr.port))
#self.send("Greetings.") # self.send("Greetings.")
def connectionLost(self, reason): def connectionLost(self, reason):
self.authed = False self.authed = False
@ -118,6 +136,7 @@ class Relay(Protocol):
else: else:
warn("Tried to remove a non-existant relay connection.") warn("Tried to remove a non-existant relay connection.")
class RelayFactory(Factory): class RelayFactory(Factory):
def buildProtocol(self, addr): def buildProtocol(self, addr):
entry = Relay(addr) entry = Relay(addr)
@ -131,6 +150,7 @@ class RelayFactory(Factory):
else: else:
return return
def sendRelayNotification(cast): def sendRelayNotification(cast):
for i in main.relayConnections.keys(): for i in main.relayConnections.keys():
if main.relayConnections[i].authed: if main.relayConnections[i].authed:

4
core/server.py
View File

@ -4,6 +4,7 @@ from utils.logging.log import *
from core.parser import parseCommand from core.parser import parseCommand
class Server(Protocol): class Server(Protocol):
def __init__(self, addr): def __init__(self, addr):
self.addr = addr self.addr = addr
@ -18,7 +19,7 @@ class Server(Protocol):
def dataReceived(self, data): def dataReceived(self, data):
data = data.decode("utf-8", "replace") data = data.decode("utf-8", "replace")
#log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data))) # log("Data received from %s:%s -- %s" % (self.addr.host, self.addr.port, repr(data)))
if "\n" in data: if "\n" in data:
splitData = [x for x in data.split("\n") if x] splitData = [x for x in data.split("\n") if x]
if "\n" in data: if "\n" in data:
@ -39,6 +40,7 @@ class Server(Protocol):
else: else:
warn("Tried to remove a non-existant connection.") warn("Tried to remove a non-existant connection.")
class ServerFactory(Factory): class ServerFactory(Factory):
def buildProtocol(self, addr): def buildProtocol(self, addr):
entry = Server(addr) entry = Server(addr)

28
main.py
View File

@ -7,10 +7,7 @@ from os import urandom
from utils.logging.log import * from utils.logging.log import *
# List of errors ZNC can give us # List of errors ZNC can give us
ZNCErrors = ["Error:", ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
"Unable to load",
"does not exist",
"doesn't exist"]
configPath = "conf/" configPath = "conf/"
certPath = "cert/" certPath = "cert/"
@ -25,9 +22,8 @@ filemap = {
"alias": ["alias.json", "provisioned alias data", "json"], "alias": ["alias.json", "provisioned alias data", "json"],
"irc": ["irc.json", "IRC network definitions", "json"], "irc": ["irc.json", "IRC network definitions", "json"],
"blacklist": ["blacklist.json", "IRC channel blacklist", "json"], "blacklist": ["blacklist.json", "IRC channel blacklist", "json"],
# Binary (pickle) configs # Binary (pickle) configs
"network": ["network.dat", "network list", "pickle"] "network": ["network.dat", "network list", "pickle"],
} }
# Connections to the plain-text interface # Connections to the plain-text interface
@ -70,43 +66,45 @@ def liveNets():
networks.add("".join([x for x in i if not x in digits])) networks.add("".join([x for x in i if not x in digits]))
return networks return networks
def saveConf(var): def saveConf(var):
if filemap[var][2] == "json": if filemap[var][2] == "json":
with open(configPath+filemap[var][0], "w") as f: with open(configPath + filemap[var][0], "w") as f:
json.dump(globals()[var], f, indent=4) json.dump(globals()[var], f, indent=4)
elif filemap[var][2] == "pickle": elif filemap[var][2] == "pickle":
with open(configPath+filemap[var][0], "wb") as f: with open(configPath + filemap[var][0], "wb") as f:
pickle.dump(globals()[var], f) pickle.dump(globals()[var], f)
else: else:
raise Exception("invalid format") raise Exception("invalid format")
def loadConf(var): def loadConf(var):
if filemap[var][2] == "json": if filemap[var][2] == "json":
with open(configPath+filemap[var][0], "r") as f: with open(configPath + filemap[var][0], "r") as f:
globals()[var] = json.load(f) globals()[var] = json.load(f)
if var == "alias": if var == "alias":
# This is a workaround to convert all the keys into integers since JSON # This is a workaround to convert all the keys into integers since JSON
# turns them into strings... # turns them into strings...
# Dammit Jason! # Dammit Jason!
global alias global alias
alias = {int(x):y for x, y in alias.items()} alias = {int(x): y for x, y in alias.items()}
elif filemap[var][2] == "pickle": elif filemap[var][2] == "pickle":
try: try:
with open(configPath+filemap[var][0], "rb") as f: with open(configPath + filemap[var][0], "rb") as f:
globals()[var] = pickle.load(f) globals()[var] = pickle.load(f)
except FileNotFoundError: except FileNotFoundError:
globals()[var] = {} globals()[var] = {}
else: else:
raise Exception("invalid format") raise Exception("invalid format")
def initConf(): def initConf():
for i in filemap.keys(): for i in filemap.keys():
loadConf(i) loadConf(i)
def initMain(): def initMain():
global r, g global r, g
initConf() initConf()
r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit r = StrictRedis(unix_socket_path=config["RedisSocket"], db=0) # Ephemeral - flushed on quit
g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent g = StrictRedis(unix_socket_path=config["RedisSocket"], db=1) # Persistent

14
modules/alias.py
View File

@ -2,9 +2,11 @@ import main
import random import random
import re import re
def generate_password(): def generate_password():
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])
def generate_alias(): def generate_alias():
nick = random.choice(main.aliasdata["stubs"]) nick = random.choice(main.aliasdata["stubs"])
rand = random.randint(1, 2) rand = random.randint(1, 2)
@ -12,8 +14,8 @@ def generate_alias():
nick = nick.capitalize() nick = nick.capitalize()
rand = random.randint(1, 4) rand = random.randint(1, 4)
while rand == 1: while rand == 1:
split = random.randint(0, len(nick)-1) split = random.randint(0, len(nick) - 1)
nick = nick[:split] + nick[split+1:] nick = nick[:split] + nick[split + 1 :]
rand = random.randint(1, 4) rand = random.randint(1, 4)
rand = random.randint(1, 3) rand = random.randint(1, 3)
if rand == 1 or rand == 4: if rand == 1 or rand == 4:
@ -63,4 +65,10 @@ def generate_alias():
if rand == 3 or rand == 4: if rand == 3 or rand == 4:
realname = realname.capitalize() realname = realname.capitalize()
return {"nick": nick, "altnick": altnick, "ident": ident, "realname": realname, "emails": []} return {
"nick": nick,
"altnick": altnick,
"ident": ident,
"realname": realname,
"emails": [],
}

68
modules/chankeep.py
View File

@ -6,11 +6,12 @@ from math import ceil
import modules.provision import modules.provision
from twisted.internet.threads import deferToThread from twisted.internet.threads import deferToThread
def allRelaysActive(net): def allRelaysActive(net):
relayNum = len(main.network[net].relays.keys()) relayNum = len(main.network[net].relays.keys())
existNum = 0 existNum = 0
for i in main.network[net].relays.keys(): for i in main.network[net].relays.keys():
name = net+str(i) name = net + str(i)
if name in main.IRCPool.keys(): if name in main.IRCPool.keys():
if main.IRCPool[name].authenticated: if main.IRCPool[name].authenticated:
existNum += 1 existNum += 1
@ -18,20 +19,22 @@ def allRelaysActive(net):
return True return True
return False return False
def getChanFree(net, new): def getChanFree(net, new):
chanfree = {} chanfree = {}
chanlimits = set() chanlimits = set()
for i in main.network[net].relays.keys(): for i in main.network[net].relays.keys():
if i in new: if i in new:
continue continue
name = net+str(i) name = net + str(i)
chanfree[i] = main.IRCPool[name].chanlimit-len(main.IRCPool[name].channels) chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)
chanlimits.add(main.IRCPool[name].chanlimit) chanlimits.add(main.IRCPool[name].chanlimit)
if not len(chanlimits) == 1: if not len(chanlimits) == 1:
error("Network %s has servers with different CHANLIMIT values" % net) error("Network %s has servers with different CHANLIMIT values" % net)
return False return False
return (chanfree, chanlimits.pop()) return (chanfree, chanlimits.pop())
def emptyChanAllocate(net, flist, relay, new): def emptyChanAllocate(net, flist, relay, new):
chanfree = getChanFree(net, new) chanfree = getChanFree(net, new)
if not chanfree: if not chanfree:
@ -41,10 +44,10 @@ def emptyChanAllocate(net, flist, relay, new):
allocated = {} allocated = {}
toalloc = len(flist) toalloc = len(flist)
if toalloc > sum(chanfree[0].values()): if toalloc > sum(chanfree[0].values()):
correction = round(toalloc-sum(chanfree[0].values()) / chanfree[1]) correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1])
#print("correction", correction) # print("correction", correction)
warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net)) warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net))
#newNums = modules.provision.provisionMultipleRelays(net, correction) # newNums = modules.provision.provisionMultipleRelays(net, correction)
return False return False
for i in chanfree[0].keys(): for i in chanfree[0].keys():
for x in range(chanfree[0][i]): for x in range(chanfree[0][i]):
@ -56,8 +59,9 @@ def emptyChanAllocate(net, flist, relay, new):
allocated[i] = [flist.pop()] allocated[i] = [flist.pop()]
return allocated return allocated
def populateChans(net, clist, relay, new): def populateChans(net, clist, relay, new):
#divided = array_split(clist, relay) # divided = array_split(clist, relay)
allocated = emptyChanAllocate(net, clist, relay, new) allocated = emptyChanAllocate(net, clist, relay, new)
if not allocated: if not allocated:
return return
@ -67,18 +71,20 @@ def populateChans(net, clist, relay, new):
else: else:
main.TempChan[net] = {i: allocated[i]} main.TempChan[net] = {i: allocated[i]}
def notifyJoin(net): def notifyJoin(net):
for i in main.network[net].relays.keys(): for i in main.network[net].relays.keys():
name = net+str(i) name = net + str(i)
if name in main.IRCPool.keys(): if name in main.IRCPool.keys():
main.IRCPool[name].checkChannels() main.IRCPool[name].checkChannels()
def minifyChans(net, listinfo): def minifyChans(net, listinfo):
if not allRelaysActive(net): if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net) error("All relays for %s are not active, cannot minify list" % net)
return False return False
for i in main.network[net].relays.keys(): for i in main.network[net].relays.keys():
name = net+str(i) name = net + str(i)
for x in main.IRCPool[name].channels: for x in main.IRCPool[name].channels:
for y in listinfo: for y in listinfo:
if y[0] == x: if y[0] == x:
@ -88,29 +94,31 @@ def minifyChans(net, listinfo):
return False return False
return listinfo return listinfo
def keepChannels(net, listinfo, mean, sigrelay, relay): def keepChannels(net, listinfo, mean, sigrelay, relay):
listinfo = minifyChans(net, listinfo) listinfo = minifyChans(net, listinfo)
if not listinfo: if not listinfo:
return return
if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels if relay <= main.config["ChanKeep"]["SigSwitch"]: # we can cover all of the channels
coverAll = True coverAll = True
elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels elif relay > main.config["ChanKeep"]["SigSwitch"]: # we cannot cover all of the channels
coverAll = False coverAll = False
if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]: if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
error("Network %s is too big to cover: %i relays required" % (net, sigrelay)) error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
return return
if coverAll: if coverAll:
needed = relay-len(main.network[net].relays.keys()) needed = relay - len(main.network[net].relays.keys())
newNums = modules.provision.provisionMultipleRelays(net, needed) newNums = modules.provision.provisionMultipleRelays(net, needed)
flist = [i[0] for i in listinfo] flist = [i[0] for i in listinfo]
populateChans(net, flist, relay, newNums) populateChans(net, flist, relay, newNums)
else: else:
needed = sigrelay-len(main.network[net].relays.keys()) needed = sigrelay - len(main.network[net].relays.keys())
newNums = modules.provision.provisionMultipleRelays(net, needed) newNums = modules.provision.provisionMultipleRelays(net, needed)
siglist = [i[0] for i in listinfo if int(i[1]) > mean] siglist = [i[0] for i in listinfo if int(i[1]) > mean]
populateChans(net, siglist, sigrelay, newNums) populateChans(net, siglist, sigrelay, newNums)
notifyJoin(net) notifyJoin(net)
def joinSingle(net, channel): def joinSingle(net, channel):
if allRelaysActive(net): if allRelaysActive(net):
chanfree = getChanFree(net, []) chanfree = getChanFree(net, [])
@ -122,14 +130,16 @@ def joinSingle(net, channel):
error("All relays for %s are not active" % net) error("All relays for %s are not active" % net)
return False return False
def nukeNetwork(net):
#purgeRecords(net)
#p = main.g.pipeline()
main.g.delete("analytics.list."+net)
#p.delete("list."+net)
#p.execute()
#def nukeNetwork(net): def nukeNetwork(net):
# purgeRecords(net)
# p = main.g.pipeline()
main.g.delete("analytics.list." + net)
# p.delete("list."+net)
# p.execute()
# def nukeNetwork(net):
# deferToThread(_nukeNetwork, net) # deferToThread(_nukeNetwork, net)
@ -141,7 +151,7 @@ def _initialList(net, num, listinfo, chanlimit):
except TypeError: except TypeError:
warn("Bad LIST data received from %s - %i" % (net, num)) warn("Bad LIST data received from %s - %i" % (net, num))
return return
mean = round(cumul/listlength, 2) mean = round(cumul / listlength, 2)
siglength = 0 siglength = 0
insiglength = 0 insiglength = 0
sigcumul = 0 sigcumul = 0
@ -154,8 +164,8 @@ def _initialList(net, num, listinfo, chanlimit):
insiglength += 1 insiglength += 1
insigcumul += int(i[1]) insigcumul += int(i[1])
sigrelay = ceil(siglength/chanlimit) sigrelay = ceil(siglength / chanlimit)
relay = ceil(listlength/chanlimit) relay = ceil(listlength / chanlimit)
netbase = "list.%s" % net netbase = "list.%s" % net
abase = "analytics.list.%s" % net abase = "analytics.list.%s" % net
p = main.g.pipeline() p = main.g.pipeline()
@ -163,18 +173,18 @@ def _initialList(net, num, listinfo, chanlimit):
p.hset(abase, "total", listlength) p.hset(abase, "total", listlength)
p.hset(abase, "sigtotal", siglength) p.hset(abase, "sigtotal", siglength)
p.hset(abase, "insigtotal", insiglength) p.hset(abase, "insigtotal", insiglength)
p.hset(abase, "sigperc", round(siglength/listlength*100, 2)) p.hset(abase, "sigperc", round(siglength / listlength * 100, 2))
p.hset(abase, "insigperc", round(insiglength/listlength*100, 2)) p.hset(abase, "insigperc", round(insiglength / listlength * 100, 2))
p.hset(abase, "cumul", cumul) p.hset(abase, "cumul", cumul)
p.hset(abase, "sigcumul", sigcumul) p.hset(abase, "sigcumul", sigcumul)
p.hset(abase, "insigcumul", insigcumul) p.hset(abase, "insigcumul", insigcumul)
p.hset(abase, "relay", relay) p.hset(abase, "relay", relay)
p.hset(abase, "sigrelay", sigrelay) p.hset(abase, "sigrelay", sigrelay)
p.hset(abase, "insigrelay", ceil(insiglength/chanlimit)) p.hset(abase, "insigrelay", ceil(insiglength / chanlimit))
# Purge existing records before writing # Purge existing records before writing
#purgeRecords(net) # purgeRecords(net)
#for i in listinfo: # for i in listinfo:
# p.rpush(netbase+"."+i[0], i[1]) # p.rpush(netbase+"."+i[0], i[1])
# p.rpush(netbase+"."+i[0], i[2]) # p.rpush(netbase+"."+i[0], i[2])
# p.sadd(netbase, i[0]) # p.sadd(netbase, i[0])
@ -183,6 +193,6 @@ def _initialList(net, num, listinfo, chanlimit):
debug("List parsing completed on %s" % net) debug("List parsing completed on %s" % net)
keepChannels(net, listinfo, mean, sigrelay, relay) keepChannels(net, listinfo, mean, sigrelay, relay)
def initialList(net, num, listinfo, chanlimit): def initialList(net, num, listinfo, chanlimit):
deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit) deferToThread(_initialList, net, num, deepcopy(listinfo), chanlimit)

4
modules/counters.py
View File

@ -1,6 +1,7 @@
import main import main
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
def event(name, eventType): def event(name, eventType):
if not "local" in main.counters.keys(): if not "local" in main.counters.keys():
main.counters["local"] = {} main.counters["local"] = {}
@ -18,6 +19,7 @@ def event(name, eventType):
main.counters["global"][eventType] += 1 main.counters["global"][eventType] += 1
main.runningSample += 1 main.runningSample += 1
def getEvents(name=None): def getEvents(name=None):
if name == None: if name == None:
if "global" in main.counters.keys(): if "global" in main.counters.keys():
@ -30,10 +32,12 @@ def getEvents(name=None):
else: else:
return None return None
def takeSample(): def takeSample():
main.lastMinuteSample = main.runningSample main.lastMinuteSample = main.runningSample
main.runningSample = 0 main.runningSample = 0
def setupCounterLoop(): def setupCounterLoop():
lc = LoopingCall(takeSample) lc = LoopingCall(takeSample)
lc.start(60) lc.start(60)

45
modules/monitor.py
View File

@ -8,9 +8,25 @@ from modules import userinfo
from modules import regproc from modules import regproc
from utils.dedup import dedup from utils.dedup import dedup
order = ["type", "net", "num", "channel", "msg", "nick", order = [
"ident", "host", "mtype", "user", "mode", "modearg", "type",
"realname", "server", "status", "time"] "net",
"num",
"channel",
"msg",
"nick",
"ident",
"host",
"mtype",
"user",
"mode",
"modearg",
"realname",
"server",
"status",
"time",
]
def parsemeta(numName, c): def parsemeta(numName, c):
if not "channel" in c.keys(): if not "channel" in c.keys():
@ -21,10 +37,16 @@ def parsemeta(numName, c):
if c["type"] in ["msg", "notice", "action", "topic", "mode"]: if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
if "muser" in c.keys(): if "muser" in c.keys():
userinfo.editUser(c["net"], c["muser"]) userinfo.editUser(c["net"], c["muser"])
#if c["type"] == "mode": # if c["type"] == "mode":
# userinfo.updateMode(c) # userinfo.updateMode(c)
elif c["type"] == "nick": elif c["type"] == "nick":
userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) userinfo.renameUser(
c["net"],
c["nick"],
c["muser"],
c["user"],
c["user"] + "!" + c["ident"] + "@" + c["host"],
)
elif c["type"] == "kick": elif c["type"] == "kick":
userinfo.editUser(c["net"], c["muser"]) userinfo.editUser(c["net"], c["muser"])
userinfo.delUserByNick(c["net"], c["channel"], c["user"]) userinfo.delUserByNick(c["net"], c["channel"], c["user"])
@ -37,9 +59,16 @@ def parsemeta(numName, c):
if "mtype" in c.keys(): if "mtype" in c.keys():
if c["mtype"] == "nick": if c["mtype"] == "nick":
userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) userinfo.renameUser(
c["net"],
c["nick"],
c["muser"],
c["user"],
c["user"] + "!" + c["ident"] + "@" + c["host"],
)
def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
if dedup(numName, c): if dedup(numName, c):
return return
@ -48,7 +77,7 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe
if "muser" in c.keys(): if "muser" in c.keys():
del c["muser"] del c["muser"]
sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order sortedKeys = {k: c[k] for k in order if k in c} # Sort dict keys according to order
sortedKeys["src"] = "irc" sortedKeys["src"] = "irc"
sendLogstashNotification(sortedKeys) sendLogstashNotification(sortedKeys)
sendRelayNotification(sortedKeys) sendRelayNotification(sortedKeys)

15
modules/network.py
View File

@ -10,6 +10,7 @@ import main
from utils.logging.log import * from utils.logging.log import *
from utils.get import getRelay from utils.get import getRelay
class Network: class Network:
def __init__(self, net, host, port, security, auth): def __init__(self, net, host, port, security, auth):
self.net = net self.net = net
@ -37,20 +38,20 @@ class Network:
"enabled": main.config["ConnectOnCreate"], "enabled": main.config["ConnectOnCreate"],
"net": self.net, "net": self.net,
"id": num, "id": num,
"registered": registered "registered": registered,
} }
password = alias.generate_password() password = alias.generate_password()
if not num in main.alias.keys(): if not num in main.alias.keys():
main.alias[num] = alias.generate_alias() main.alias[num] = alias.generate_alias()
main.saveConf("alias") main.saveConf("alias")
self.aliases[num] = {"password": password} self.aliases[num] = {"password": password}
#if main.config["ConnectOnCreate"]: -- Done in provision # if main.config["ConnectOnCreate"]: -- Done in provision
# self.start_bot(num) # self.start_bot(num)
return num, main.alias[num]["nick"] return num, main.alias[num]["nick"]
def killAliases(self, aliasList): def killAliases(self, aliasList):
for i in aliasList: for i in aliasList:
name = self.net+str(i) name = self.net + str(i)
if name in main.ReactorPool.keys(): if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys(): if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying() main.FactoryPool[name].stopTrying()
@ -63,7 +64,7 @@ class Network:
def delete_relay(self, id): def delete_relay(self, id):
del self.relays[id] del self.relays[id]
del self.aliases[id] del self.aliases[id]
#del main.alias[id] - Aliases are global per num, so don't delete them! # del main.alias[id] - Aliases are global per num, so don't delete them!
self.killAliases([id]) self.killAliases([id])
def seppuku(self): def seppuku(self):
@ -74,11 +75,11 @@ class Network:
def start_bot(self, num): def start_bot(self, num):
# a single name is given to relays in the backend # a single name is given to relays in the backend
# e.g. freenode1 for the first relay on freenode network # e.g. freenode1 for the first relay on freenode network
keyFN = main.certPath+main.config["Key"] keyFN = main.certPath + main.config["Key"]
certFN = main.certPath+main.config["Certificate"] certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")) contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace"))
bot = IRCBotFactory(self.net, num) bot = IRCBotFactory(self.net, num)
#host, port = self.relays[num]["host"], self.relays[num]["port"] # host, port = self.relays[num]["host"], self.relays[num]["port"]
host, port = getRelay(num) host, port = getRelay(num)
rct = reactor.connectSSL(host, port, bot, contextFactory) rct = reactor.connectSSL(host, port, bot, contextFactory)
name = self.net + str(num) name = self.net + str(num)

35
modules/provision.py
View File

@ -4,6 +4,7 @@ from utils.logging.log import *
from twisted.internet import reactor from twisted.internet import reactor
import modules.regproc import modules.regproc
def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security, auth, password): def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host, port, security, auth, password):
commands = {} commands = {}
stage2commands = {} stage2commands = {}
@ -17,7 +18,7 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host,
commands["controlpanel"].append("Set Ident %s %s" % (user, ident)) commands["controlpanel"].append("Set Ident %s %s" % (user, ident))
commands["controlpanel"].append("Set RealName %s %s" % (user, realname)) commands["controlpanel"].append("Set RealName %s %s" % (user, realname))
if security == "ssl": if security == "ssl":
commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me commands["controlpanel"].append("SetNetwork TrustAllCerts %s %s true" % (user, network)) # Don't judge me
commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port)) commands["controlpanel"].append("AddServer %s %s %s +%s" % (user, network, host, port))
elif security == "plain": elif security == "plain":
commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port)) commands["controlpanel"].append("AddServer %s %s %s %s" % (user, network, host, port))
@ -29,9 +30,9 @@ def provisionUserNetworkData(num, nick, altnick, ident, realname, network, host,
inst = modules.regproc.selectInst(network) inst = modules.regproc.selectInst(network)
if "setmode" in inst.keys(): if "setmode" in inst.keys():
stage2commands["status"].append("LoadMod perform") stage2commands["status"].append("LoadMod perform")
#stage2commands["perform"].append("add mode %nick% +"+inst["setmode"]) # stage2commands["perform"].append("add mode %nick% +"+inst["setmode"])
deliverRelayCommands(num, commands, deliverRelayCommands(num, commands, stage2=[[user + "/" + network, stage2commands]])
stage2=[[user+"/"+network, stage2commands]])
def provisionAuthenticationData(num, nick, network, security, auth, password): def provisionAuthenticationData(num, nick, network, security, auth, password):
commands = {} commands = {}
@ -48,23 +49,28 @@ def provisionAuthenticationData(num, nick, network, security, auth, password):
commands["nickserv"].append("Set %s" % password) commands["nickserv"].append("Set %s" % password)
inst = modules.regproc.selectInst(network) inst = modules.regproc.selectInst(network)
if "setmode" in inst.keys(): if "setmode" in inst.keys():
#commands["status"].append("LoadMod perform") # commands["status"].append("LoadMod perform")
commands["perform"] = ["add mode %nick% +"+inst["setmode"]] commands["perform"] = ["add mode %nick% +" + inst["setmode"]]
deliverRelayCommands(num, commands, user=user+"/"+network) deliverRelayCommands(num, commands, user=user + "/" + network)
def provisionRelay(num, network): # provision user and network data def provisionRelay(num, network): # provision user and network data
aliasObj = main.alias[num] aliasObj = main.alias[num]
alias = aliasObj["nick"] alias = aliasObj["nick"]
provisionUserNetworkData(num, *aliasObj.values(), network, provisionUserNetworkData(
main.network[network].host, num,
main.network[network].port, *aliasObj.values(),
main.network[network].security, network,
main.network[network].auth, main.network[network].host,
main.network[network].aliases[num]["password"]) main.network[network].port,
main.network[network].security,
main.network[network].auth,
main.network[network].aliases[num]["password"]
)
if main.config["ConnectOnCreate"]: if main.config["ConnectOnCreate"]:
reactor.callLater(10, main.network[network].start_bot, num) reactor.callLater(10, main.network[network].start_bot, num)
def provisionMultipleRelays(net, relaysNeeded): def provisionMultipleRelays(net, relaysNeeded):
numsProvisioned = [] numsProvisioned = []
for i in range(relaysNeeded): for i in range(relaysNeeded):
@ -73,4 +79,3 @@ def provisionMultipleRelays(net, relaysNeeded):
provisionRelay(num, net) provisionRelay(num, net)
main.saveConf("network") main.saveConf("network")
return numsProvisioned return numsProvisioned

36
modules/regproc.py
View File

@ -4,6 +4,7 @@ from utils.logging.log import *
from utils.logging.debug import * from utils.logging.debug import *
from copy import deepcopy from copy import deepcopy
def needToRegister(net): def needToRegister(net):
inst = selectInst(net) inst = selectInst(net)
if "register" in inst.keys(): if "register" in inst.keys():
@ -12,6 +13,7 @@ def needToRegister(net):
else: else:
return False return False
def selectInst(net): def selectInst(net):
if net in main.irc.keys(): if net in main.irc.keys():
inst = deepcopy(main.irc[net]) inst = deepcopy(main.irc[net])
@ -22,11 +24,12 @@ def selectInst(net):
inst = main.irc["_"] inst = main.irc["_"]
return inst return inst
def substitute(net, num, token=None): def substitute(net, num, token=None):
inst = selectInst(net) inst = selectInst(net)
alias = main.alias[num] alias = main.alias[num]
nickname = alias["nick"] nickname = alias["nick"]
username = nickname+"/"+net username = nickname + "/" + net
password = main.network[net].aliases[num]["password"] password = main.network[net].aliases[num]["password"]
inst["email"] = inst["email"].replace("{nickname}", nickname) inst["email"] = inst["email"].replace("{nickname}", nickname)
for i in inst.keys(): for i in inst.keys():
@ -39,29 +42,32 @@ def substitute(net, num, token=None):
inst[i] = inst[i].replace("{token}", token) inst[i] = inst[i].replace("{token}", token)
return inst return inst
def registerAccount(net, num): def registerAccount(net, num):
debug("Attempting to register: %s - %i" % (net, num)) debug("Attempting to register: %s - %i" % (net, num))
sinst = substitute(net, num) sinst = substitute(net, num)
if not sinst["register"]: if not sinst["register"]:
error("Cannot register for %s: function disabled" % (net)) error("Cannot register for %s: function disabled" % (net))
return False return False
name = net+str(num) name = net + str(num)
main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"]) main.IRCPool[name].msg(sinst["entity"], sinst["registermsg"])
def confirmAccount(net, num, token): def confirmAccount(net, num, token):
sinst = substitute(net, num, token=token) sinst = substitute(net, num, token=token)
name = net+str(num) name = net + str(num)
main.IRCPool[name].msg(sinst["entity"], sinst["confirm"]) main.IRCPool[name].msg(sinst["entity"], sinst["confirm"])
enableAuthentication(net, num) enableAuthentication(net, num)
def confirmRegistration(net, num, negativepass=None): def confirmRegistration(net, num, negativepass=None):
obj = main.network[net] obj = main.network[net]
name = net+str(num) name = net + str(num)
if name in main.IRCPool.keys(): if name in main.IRCPool.keys():
if not negativepass == None: if not negativepass == None:
main.IRCPool[name].regPing(negativepass=negativepass) main.IRCPool[name].regPing(negativepass=negativepass)
return return
debug("Relay authenticated: %s - %i" %(net, num)) debug("Relay authenticated: %s - %i" % (net, num))
main.IRCPool[name].authenticated = True main.IRCPool[name].authenticated = True
main.IRCPool[name].recheckList() main.IRCPool[name].recheckList()
if obj.relays[num]["registered"]: if obj.relays[num]["registered"]:
@ -75,21 +81,23 @@ def confirmRegistration(net, num, negativepass=None):
obj.relays[num]["registered"] = True obj.relays[num]["registered"] = True
main.saveConf("network") main.saveConf("network")
def enableAuthentication(net, num): def enableAuthentication(net, num):
obj = main.network[net] obj = main.network[net]
nick = main.alias[num]["nick"] nick = main.alias[num]["nick"]
security = obj.security security = obj.security
auth = obj.auth auth = obj.auth
password = obj.aliases[num]["password"] password = obj.aliases[num]["password"]
uname = main.alias[num]["nick"]+"/"+net uname = main.alias[num]["nick"] + "/" + net
provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth provision.provisionAuthenticationData(num, nick, net, security, auth, password) # Set up for auth
main.IRCPool[net+str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"]+"status", "Jump") main.IRCPool[net + str(num)].msg(main.config["Tweaks"]["ZNC"]["Prefix"] + "status", "Jump")
if selectInst(net)["check"] == False: if selectInst(net)["check"] == False:
confirmRegistration(net, num) confirmRegistration(net, num)
def registerTest(c): def registerTest(c):
sinst = substitute(c["net"], c["num"]) sinst = substitute(c["net"], c["num"])
name = c["net"]+str(c["num"]) name = c["net"] + str(c["num"])
if sinst["check"] == False: if sinst["check"] == False:
return return
if "msg" in c.keys() and not c["msg"] == None: if "msg" in c.keys() and not c["msg"] == None:
@ -98,19 +106,19 @@ def registerTest(c):
if not main.IRCPool[name]._negativePass == True: if not main.IRCPool[name]._negativePass == True:
if c["type"] == "query" and c["nick"] == sinst["entity"]: if c["type"] == "query" and c["nick"] == sinst["entity"]:
if sinst["checknegativemsg"] in c["msg"]: if sinst["checknegativemsg"] in c["msg"]:
confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back
return return
if sinst["checkendnegative"] in c["msg"]: if sinst["checkendnegative"] in c["msg"]:
confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back
return return
if sinst["ping"]: if sinst["ping"]:
if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]: if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]:
confirmRegistration(c["net"], c["num"]) confirmRegistration(c["net"], c["num"])
return return
if sinst["checktype"] == "msg": if sinst["checktype"] == "msg":
if sinst["checkmsg"] in c["msg"]: if sinst["checkmsg"] in c["msg"]:
confirmRegistration(c["net"], c["num"]) confirmRegistration(c["net"], c["num"])
return return
elif sinst["checktype"] == "mode": elif sinst["checktype"] == "mode":
if c["type"] == "self": if c["type"] == "self":
if c["mtype"] == "mode": if c["mtype"] == "mode":

102
modules/userinfo.py
View File

@ -6,12 +6,14 @@ from utils.logging.log import *
from utils.logging.debug import debug, trace from utils.logging.debug import debug, trace
from utils.parsing import parsen from utils.parsing import parsen
def getWhoSingle(name, query): def getWhoSingle(name, query):
result = main.r.sscan("live.who."+name, 0, query, count=999999) result = main.r.sscan("live.who." + name, 0, query, count=999999)
if result[1] == []: if result[1] == []:
return None return None
return (i.decode() for i in result[1]) return (i.decode() for i in result[1])
def getWho(query): def getWho(query):
result = {} result = {}
for i in main.network.keys(): for i in main.network.keys():
@ -20,20 +22,23 @@ def getWho(query):
result[i] = f result[i] = f
return result return result
def getChansSingle(name, nick): def getChansSingle(name, nick):
nick = ["live.chan."+name+"."+i for i in nick] nick = ["live.chan." + name + "." + i for i in nick]
result = main.r.sinter(*nick) result = main.r.sinter(*nick)
if len(result) == 0: if len(result) == 0:
return None return None
return (i.decode() for i in result) return (i.decode() for i in result)
def getChanList(name, nick): def getChanList(name, nick):
chanspace = "live.chan."+name+"."+nick chanspace = "live.chan." + name + "." + nick
result = main.r.smembers(chanspace) result = main.r.smembers(chanspace)
if len(result) == 0: if len(result) == 0:
return None return None
return (i.decode() for i in result) return (i.decode() for i in result)
def getChans(nick): def getChans(nick):
result = {} result = {}
for i in main.network.keys(): for i in main.network.keys():
@ -42,13 +47,15 @@ def getChans(nick):
result[i] = f result[i] = f
return result return result
def getUsersSingle(name, nick): def getUsersSingle(name, nick):
nick = ("live.who."+name+"."+i for i in nick) nick = ("live.who." + name + "." + i for i in nick)
result = main.r.sinter(*nick) result = main.r.sinter(*nick)
if len(result) == 0: if len(result) == 0:
return None return None
return (i.decode() for i in result) return (i.decode() for i in result)
def getUsers(nick): def getUsers(nick):
result = {} result = {}
for i in main.network.keys(): for i in main.network.keys():
@ -57,8 +64,10 @@ def getUsers(nick):
result[i] = f result[i] = f
return result return result
def getNumWhoEntries(name): def getNumWhoEntries(name):
return main.r.scard("live.who."+name) return main.r.scard("live.who." + name)
def getNumTotalWhoEntries(): def getNumTotalWhoEntries():
total = 0 total = 0
@ -66,6 +75,7 @@ def getNumTotalWhoEntries():
total += getNumWhoEntries(i) total += getNumWhoEntries(i)
return total return total
def getNamespace(name, channel, nick): def getNamespace(name, channel, nick):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
namespace = "live.who.%s.%s" % (name, channel) namespace = "live.who.%s.%s" % (name, channel)
@ -73,35 +83,40 @@ def getNamespace(name, channel, nick):
mapspace = "live.map.%s" % name mapspace = "live.map.%s" % name
return (gnamespace, namespace, chanspace, mapspace) return (gnamespace, namespace, chanspace, mapspace)
def _initialUsers(name, channel, users): def _initialUsers(name, channel, users):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
mapspace = "live.map.%s" % name mapspace = "live.map.%s" % name
p = main.r.pipeline() p = main.r.pipeline()
for i in users: for i in users:
user = i[0]+"!"+i[1]+"@"+i[2] user = i[0] + "!" + i[1] + "@" + i[2]
p.hset(mapspace, i[0], user) p.hset(mapspace, i[0], user)
p.sadd(gnamespace, user) p.sadd(gnamespace, user)
p.execute() p.execute()
def initialUsers(name, channel, users): def initialUsers(name, channel, users):
trace("Initialising WHO records for %s on %s" % (channel, name)) trace("Initialising WHO records for %s on %s" % (channel, name))
d = deferToThread(_initialUsers, name, channel, users) d = deferToThread(_initialUsers, name, channel, users)
#d.addCallback(testCallback) # d.addCallback(testCallback)
def _initialNames(name, channel, names): def _initialNames(name, channel, names):
namespace = "live.who.%s.%s" % (name, channel) namespace = "live.who.%s.%s" % (name, channel)
p = main.r.pipeline() p = main.r.pipeline()
for mode, nick in names: for mode, nick in names:
p.sadd(namespace, nick) p.sadd(namespace, nick)
p.sadd("live.chan."+name+"."+nick, channel) p.sadd("live.chan." + name + "." + nick, channel)
if mode: if mode:
p.hset("live.prefix."+name+"."+channel, nick, mode) p.hset("live.prefix." + name + "." + channel, nick, mode)
p.execute() p.execute()
def initialNames(name, channel, names): def initialNames(name, channel, names):
trace("Initialising NAMES records for %s on %s" % (channel, name)) trace("Initialising NAMES records for %s on %s" % (channel, name))
d = deferToThread(_initialNames, name, channel, names) d = deferToThread(_initialNames, name, channel, names)
#d.addCallback(testCallback) # d.addCallback(testCallback)
def editUser(name, user): def editUser(name, user):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
@ -109,9 +124,10 @@ def editUser(name, user):
parsed = parsen(user) parsed = parsen(user)
p = main.r.pipeline() p = main.r.pipeline()
p.sadd(gnamespace, user) p.sadd(gnamespace, user)
p.hset(mapspace, parsed[0], user) # add nick -> user mapping p.hset(mapspace, parsed[0], user) # add nick -> user mapping
p.execute() p.execute()
def addUser(name, channel, nick, user): def addUser(name, channel, nick, user):
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick) gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline() p = main.r.pipeline()
@ -121,40 +137,43 @@ def addUser(name, channel, nick, user):
p.hset(mapspace, nick, user) p.hset(mapspace, nick, user)
p.execute() p.execute()
def delUser(name, channel, nick, user): def delUser(name, channel, nick, user):
gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick) gnamespace, namespace, chanspace, mapspace = getNamespace(name, channel, nick)
p = main.r.pipeline() p = main.r.pipeline()
channels = main.r.smembers(chanspace) channels = main.r.smembers(chanspace)
p.srem(namespace, nick) p.srem(namespace, nick)
if channels == {channel.encode()}: # can we only see them on this channel? if channels == {channel.encode()}: # can we only see them on this channel?
p.delete(chanspace) # remove channel tracking entry p.delete(chanspace) # remove channel tracking entry
p.hdel("live.prefix."+name+"."+channel, nick) # remove prefix tracking entry p.hdel("live.prefix." + name + "." + channel, nick) # remove prefix tracking entry
p.hdel(mapspace, nick) # remove nick mapping entry p.hdel(mapspace, nick) # remove nick mapping entry
if user: if user:
p.srem(gnamespace, user) # remove global userinfo entry p.srem(gnamespace, user) # remove global userinfo entry
else: else:
warn("Attempt to delete nonexistent user: %s" % user) warn("Attempt to delete nonexistent user: %s" % user)
else: else:
p.srem(chanspace, channel) # keep up - remove the channel from their list p.srem(chanspace, channel) # keep up - remove the channel from their list
p.execute() p.execute()
def escape(text): def escape(text):
chars = ["[", "]", "^", "-", "*", "?"] chars = ["[", "]", "^", "-", "*", "?"]
text = text.replace("\\", "\\\\") text = text.replace("\\", "\\\\")
for i in chars: for i in chars:
text = text.replace(i, "\\"+i) text = text.replace(i, "\\" + i)
return text return text
def getUserByNick(name, nick): def getUserByNick(name, nick):
gnamespace = "live.who.%s" % name # "nick": "nick!ident@host" gnamespace = "live.who.%s" % name # "nick": "nick!ident@host"
mapspace = "live.map.%s" % name mapspace = "live.map.%s" % name
if main.r.hexists(mapspace, nick): if main.r.hexists(mapspace, nick):
return main.r.hget(mapspace, nick) return main.r.hget(mapspace, nick)
else: else:
warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace)) warn("Entry doesn't exist: %s on %s - attempting auxiliary lookup" % (nick, mapspace))
#return False # return False
# legacy code below - remove when map is reliable # legacy code below - remove when map is reliable
usermatch = main.r.sscan(gnamespace, match=escape(nick)+"!*", count=999999999) usermatch = main.r.sscan(gnamespace, match=escape(nick) + "!*", count=999999999)
if usermatch[1] == []: if usermatch[1] == []:
warn("No matches found for user query: %s on %s" % (nick, name)) warn("No matches found for user query: %s on %s" % (nick, name))
return False return False
@ -166,6 +185,7 @@ def getUserByNick(name, nick):
warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace)) warn("Auxiliary lookup failed: %s on %s" % (nick, gnamespace))
return False return False
def renameUser(name, oldnick, olduser, newnick, newuser): def renameUser(name, oldnick, olduser, newnick, newuser):
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, oldnick) chanspace = "live.chan.%s.%s" % (name, oldnick)
@ -176,40 +196,43 @@ def renameUser(name, oldnick, olduser, newnick, newuser):
p.sadd(gnamespace, newuser) p.sadd(gnamespace, newuser)
for i in main.r.smembers(chanspace): for i in main.r.smembers(chanspace):
i = i.decode() i = i.decode()
p.srem("live.who."+name+"."+i, oldnick) p.srem("live.who." + name + "." + i, oldnick)
p.sadd("live.who."+name+"."+i, newnick) p.sadd("live.who." + name + "." + i, newnick)
p.hdel(mapspace, oldnick) p.hdel(mapspace, oldnick)
p.hset(mapspace, newnick, newuser) p.hset(mapspace, newnick, newuser)
if main.r.exists("live.prefix."+name+"."+i): # if there's a prefix entry for the channel if main.r.exists("live.prefix." + name + "." + i): # if there's a prefix entry for the channel
if main.r.hexists("live.prefix."+name+"."+i, oldnick): # if the old nick is in it if main.r.hexists("live.prefix." + name + "." + i, oldnick): # if the old nick is in it
mode = main.r.hget("live.prefix."+name+"."+i, oldnick) # retrieve old modes mode = main.r.hget("live.prefix." + name + "." + i, oldnick) # retrieve old modes
p.hset("live.prefix."+name+"."+i, newnick, mode) # set old modes to new nickname p.hset("live.prefix." + name + "." + i, newnick, mode) # set old modes to new nickname
if main.r.exists(chanspace): if main.r.exists(chanspace):
p.rename(chanspace, newchanspace) p.rename(chanspace, newchanspace)
else: else:
warn("Key doesn't exist: %s" % chanspace) warn("Key doesn't exist: %s" % chanspace)
p.execute() p.execute()
def delUserByNick(name, channel, nick): # kick
def delUserByNick(name, channel, nick): # kick
user = getUserByNick(name, nick) user = getUserByNick(name, nick)
if not user: if not user:
return return
delUser(name, channel, nick, user) delUser(name, channel, nick, user)
def delUserByNetwork(name, nick, user): # quit
def delUserByNetwork(name, nick, user): # quit
gnamespace = "live.who.%s" % name gnamespace = "live.who.%s" % name
chanspace = "live.chan.%s.%s" % (name, nick) chanspace = "live.chan.%s.%s" % (name, nick)
mapspace = "live.chan.%s" % name mapspace = "live.chan.%s" % name
p = main.r.pipeline() p = main.r.pipeline()
p.srem(gnamespace, user) p.srem(gnamespace, user)
for i in main.r.smembers(chanspace): for i in main.r.smembers(chanspace):
p.srem("live.who."+name+"."+i.decode(), nick) p.srem("live.who." + name + "." + i.decode(), nick)
p.hdel("live.prefix."+name+"."+i.decode(), nick) p.hdel("live.prefix." + name + "." + i.decode(), nick)
p.delete(chanspace) p.delete(chanspace)
p.hdel(mapspace, nick) p.hdel(mapspace, nick)
p.execute() p.execute()
def _delChannels(net, channels): def _delChannels(net, channels):
gnamespace = "live.who.%s" % net gnamespace = "live.who.%s" % net
mapspace = "live.map.%s" % net mapspace = "live.map.%s" % net
@ -218,23 +241,24 @@ def _delChannels(net, channels):
namespace = "live.who.%s.%s" % (net, channel) namespace = "live.who.%s.%s" % (net, channel)
for i in main.r.smembers(namespace): for i in main.r.smembers(namespace):
nick = i.decode() nick = i.decode()
#user = getUserByNick(net, nick) -- far too many function calls # user = getUserByNick(net, nick) -- far too many function calls
user = main.r.hget(mapspace, nick) user = main.r.hget(mapspace, nick)
if not user: if not user:
warn("User lookup failed: %s on %s" % (nick, net)) warn("User lookup failed: %s on %s" % (nick, net))
if main.r.smembers("live.chan."+net+"."+nick) == {channel.encode()}: if main.r.smembers("live.chan." + net + "." + nick) == {channel.encode()}:
if user: if user:
p.srem(gnamespace, user) p.srem(gnamespace, user)
p.delete("live.chan."+net+"."+nick) p.delete("live.chan." + net + "." + nick)
p.hdel(mapspace, nick) # remove map entry p.hdel(mapspace, nick) # remove map entry
else: else:
p.srem("live.chan."+net+"."+nick, channel) p.srem("live.chan." + net + "." + nick, channel)
p.delete(namespace) p.delete(namespace)
p.delete("live.prefix."+net+"."+channel) p.delete("live.prefix." + net + "." + channel)
p.execute() p.execute()
def delChannels(net, channels): # we have left a channel
def delChannels(net, channels): # we have left a channel
trace("Purging channel %s for %s" % (", ".join(channels), net)) trace("Purging channel %s for %s" % (", ".join(channels), net))
d = deferToThread(_delChannels, net, channels) d = deferToThread(_delChannels, net, channels)
#d.addCallback(testCallback) # d.addCallback(testCallback)

1
requirements.txt
View File

@ -1,3 +1,4 @@
pre-commit
twisted twisted
pyOpenSSL pyOpenSSL
redis redis

59
threshold
View File

@ -3,19 +3,23 @@ from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.ssl import DefaultOpenSSLContextFactory
import sys import sys
from signal import signal, SIGINT from signal import signal, SIGINT
#from twisted.python import log
#from sys import stdout # from twisted.python import log
#log.startLogging(stdout) # from sys import stdout
from sys import stdout, stderr # Import again because we want to override # log.startLogging(stdout)
from codecs import getwriter # fix printing odd shit to the terminal from sys import stdout, stderr # Import again because we want to override
from codecs import getwriter # fix printing odd shit to the terminal
stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know
stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using
# unicode quit messages for no reason # unicode quit messages for no reason
import main import main
main.initMain() main.initMain()
from utils.cleanup import handler from utils.cleanup import handler
signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine
if "--debug" in sys.argv: # yes really signal(SIGINT, handler) # Handle Ctrl-C and run the cleanup routine
if "--debug" in sys.argv: # yes really
main.config["Debug"] = True main.config["Debug"] = True
if "--trace" in sys.argv: if "--trace" in sys.argv:
main.config["Trace"] = True main.config["Trace"] = True
@ -24,25 +28,54 @@ from utils.loaders.command_loader import loadCommands
from core.server import Server, ServerFactory from core.server import Server, ServerFactory
from core.relay import Relay, RelayFactory from core.relay import Relay, RelayFactory
import modules.counters import modules.counters
loadCommands() loadCommands()
import core.logstash import core.logstash
core.logstash.init_logstash() core.logstash.init_logstash()
if __name__ == "__main__": if __name__ == "__main__":
listener = ServerFactory() listener = ServerFactory()
if main.config["Listener"]["UseSSL"] == True: if main.config["Listener"]["UseSSL"] == True:
reactor.listenSSL(main.config["Listener"]["Port"], listener, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["Listener"]["Address"]) reactor.listenSSL(
main.config["Listener"]["Port"],
listener,
DefaultOpenSSLContextFactory(
main.certPath + main.config["Key"],
main.certPath + main.config["Certificate"],
),
interface=main.config["Listener"]["Address"],
)
log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]))
else: else:
reactor.listenTCP(main.config["Listener"]["Port"], listener, interface=main.config["Listener"]["Address"]) reactor.listenTCP(
main.config["Listener"]["Port"],
listener,
interface=main.config["Listener"]["Address"],
)
log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"])) log("Threshold running on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]))
if main.config["RelayAPI"]["Enabled"]: if main.config["RelayAPI"]["Enabled"]:
relay = RelayFactory() relay = RelayFactory()
if main.config["RelayAPI"]["UseSSL"] == True: if main.config["RelayAPI"]["UseSSL"] == True:
reactor.listenSSL(main.config["RelayAPI"]["Port"], relay, DefaultOpenSSLContextFactory(main.certPath+main.config["Key"], main.certPath+main.config["Certificate"]), interface=main.config["RelayAPI"]["Address"]) reactor.listenSSL(
log("Threshold relay running with SSL on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) main.config["RelayAPI"]["Port"],
relay,
DefaultOpenSSLContextFactory(
main.certPath + main.config["Key"],
main.certPath + main.config["Certificate"],
),
interface=main.config["RelayAPI"]["Address"],
)
log(
"Threshold relay running with SSL on %s:%s"
% (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])
)
else: else:
reactor.listenTCP(main.config["RelayAPI"]["Port"], relay, interface=main.config["RelayAPI"]["Address"]) reactor.listenTCP(
main.config["RelayAPI"]["Port"],
relay,
interface=main.config["RelayAPI"]["Address"],
)
log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])) log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]))
for net in main.network.keys(): for net in main.network.keys():
main.network[net].start_bots() main.network[net].start_bots()

6
utils/cleanup.py
View File

@ -1,15 +1,17 @@
import main import main
from twisted.internet import reactor from twisted.internet import reactor
from utils.logging.debug import debug from utils.logging.debug import debug
from utils.logging.log import * from utils.logging.log import *
import sys import sys
def handler(sig, frame): def handler(sig, frame):
log("Received SIGINT, cleaning up") log("Received SIGINT, cleaning up")
cleanup() cleanup()
def cleanup(): def cleanup():
debug("Flushing Redis database") debug("Flushing Redis database")
main.r.flushdb() main.r.flushdb()
reactor.stop() reactor.stop()
#sys.exit(1) # sys.exit(1)

7
utils/dedup.py
View File

@ -5,20 +5,21 @@ from json import dumps
import main import main
from utils.logging.debug import debug from utils.logging.debug import debug
def dedup(numName, b): def dedup(numName, b):
c = deepcopy(b) c = deepcopy(b)
if "time" in c.keys(): if "time" in c.keys():
del c["time"] del c["time"]
c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]] c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]]
castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
del c["approxtime"] del c["approxtime"]
isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
if isDuplicate: if isDuplicate:
debug("Duplicate: %s" % (c)) debug("Duplicate: %s" % (c))
return True return True
if numName in main.lastEvents.keys(): if numName in main.lastEvents.keys():
main.lastEvents[numName].insert(0, castHash) main.lastEvents[numName].insert(0, castHash)
main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]] main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]]
else: else:
main.lastEvents[numName] = [castHash] main.lastEvents[numName] = [castHash]
return False return False

1
utils/get.py
View File

@ -1,5 +1,6 @@
import main import main
def getRelay(num): def getRelay(num):
host = main.config["Relay"]["Host"].replace("x", str(num)) host = main.config["Relay"]["Host"].replace("x", str(num))
port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2))) port = int(str(main.config["Relay"]["Port"]).replace("x", str(num).zfill(2)))

5
utils/loaders/command_loader.py
View File

@ -6,13 +6,14 @@ import commands
from main import CommandMap from main import CommandMap
def loadCommands(allowDup=False): def loadCommands(allowDup=False):
for filename in listdir("commands"): for filename in listdir("commands"):
if filename.endswith(".py") and filename != "__init__.py": if filename.endswith(".py") and filename != "__init__.py":
commandName = filename[0:-3] commandName = filename[0:-3]
className = commandName.capitalize()+"Command" className = commandName.capitalize() + "Command"
try: try:
module = __import__('commands.%s' % commandName) module = __import__("commands.%s" % commandName)
if not commandName in CommandMap: if not commandName in CommandMap:
CommandMap[commandName] = getattr(getattr(module, commandName), className) CommandMap[commandName] = getattr(getattr(module, commandName), className)
debug("Registered command: %s" % commandName) debug("Registered command: %s" % commandName)

11
utils/loaders/single_loader.py
View File

@ -8,16 +8,17 @@ import commands
from main import CommandMap from main import CommandMap
def loadSingle(commandName): def loadSingle(commandName):
if commandName+".py" in listdir("commands"): if commandName + ".py" in listdir("commands"):
className = commandName.capitalize()+"Command" className = commandName.capitalize() + "Command"
try: try:
if commandName in CommandMap.keys(): if commandName in CommandMap.keys():
reload(sys.modules["commands."+commandName]) reload(sys.modules["commands." + commandName])
CommandMap[commandName] = getattr(sys.modules["commands."+commandName], className) CommandMap[commandName] = getattr(sys.modules["commands." + commandName], className)
debug("Reloaded command: %s" % commandName) debug("Reloaded command: %s" % commandName)
return "RELOAD" return "RELOAD"
module = __import__('commands.%s' % commandName) module = __import__("commands.%s" % commandName)
CommandMap[commandName] = getattr(getattr(module, commandName), className) CommandMap[commandName] = getattr(getattr(module, commandName), className)
debug("Registered command: %s" % commandName) debug("Registered command: %s" % commandName)
return True return True

2
utils/logging/debug.py
View File

@ -1,10 +1,12 @@
import main import main
# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main # we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main
# to read the setting # to read the setting
def debug(*data): def debug(*data):
if main.config["Debug"]: if main.config["Debug"]:
print("[DEBUG]", *data) print("[DEBUG]", *data)
def trace(*data): def trace(*data):
if main.config["Trace"]: if main.config["Trace"]:
print("[TRACE]", *data) print("[TRACE]", *data)

2
utils/logging/log.py
View File

@ -1,8 +1,10 @@
def log(*data): def log(*data):
print("[LOG]", *data) print("[LOG]", *data)
def warn(*data): def warn(*data):
print("[WARNING]", *data) print("[WARNING]", *data)
def error(*data): def error(*data):
print("[ERROR]", *data) print("[ERROR]", *data)

7
utils/logging/send.py
View File

@ -1,29 +1,36 @@
import main import main
def sendData(addr, data): def sendData(addr, data):
main.connections[addr].send(data) main.connections[addr].send(data)
def sendWithPrefix(addr, data, prefix): def sendWithPrefix(addr, data, prefix):
toSend = "" toSend = ""
for i in data.split("\n"): for i in data.split("\n"):
toSend += prefix + " " + i + "\n" toSend += prefix + " " + i + "\n"
sendData(addr, toSend) sendData(addr, toSend)
def sendSuccess(addr, data): def sendSuccess(addr, data):
sendWithPrefix(addr, data, "[y]") sendWithPrefix(addr, data, "[y]")
def sendFailure(addr, data): def sendFailure(addr, data):
sendWithPrefix(addr, data, "[n]") sendWithPrefix(addr, data, "[n]")
def sendInfo(addr, data): def sendInfo(addr, data):
sendWithPrefix(addr, data, "[i]") sendWithPrefix(addr, data, "[i]")
def sendAll(data): def sendAll(data):
for i in main.connections: for i in main.connections:
if main.connections[i].authed: if main.connections[i].authed:
main.connections[i].send(data) main.connections[i].send(data)
return return
def incorrectUsage(addr, mode): def incorrectUsage(addr, mode):
if mode == None: if mode == None:
sendFailure(addr, "Incorrect usage") sendFailure(addr, "Incorrect usage")