Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
pull from: druid
merge into: Pathogen:master
Conversation 1 Commits 263 Files Changed 128 +52 -29
11 changed files with 52 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4ecb37b179 - Show all commits

2
commands/admall.py
View File

@ -1,5 +1,5 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
class AdmallCommand:

4
commands/email.py
View File

@ -74,7 +74,9 @@ class EmailCommand:
elif length == 3:
if spl[1] == "list":
if spl[2] == "domain":
filtered = {f"{k}:{k2}":v2 for k,v in main.irc.items() for k2,v2 in v.items() if k2 == "domains"}
filtered = {
f"{k}:{k2}": v2 for k, v in main.irc.items() for k2, v2 in v.items() if k2 == "domains"
}
info(dump(filtered))
return
else:

8
core/bot.py
View File

@ -35,13 +35,7 @@ from utils.parsing import parsen
from twisted.internet.ssl import DefaultOpenSSLContextFactory
def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
keyFN = main.certPath + main.config["Key"]
certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace"))
bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
host, port = getRelay(num)
rct = reactor.connectSSL(host, port, bot, contextFactory)
from utils.deliver_relay_commands import deliverRelayCommands
# Copied from the Twisted source so we can fix a bug

5
core/relay.py
View File

@ -114,7 +114,10 @@ class Relay(Protocol):
def handleHello(self, parsed):
if parsed["key"] in main.tokens.keys():
if parsed["hello"] == main.tokens[parsed["key"]]["hello"] and main.tokens[parsed["key"]]["usage"] == "relay":
if (
parsed["hello"] == main.tokens[parsed["key"]]["hello"]
and main.tokens[parsed["key"]]["usage"] == "relay"
):
self.sendMsg({"type": "hello", "hello": main.tokens[parsed["key"]]["counter"]})
self.authed = True
else:

4
main.py
View File

@ -106,5 +106,7 @@ def initConf():
def initMain():
global r, g
initConf()
r = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]) # Ephemeral - flushed on quit
r = StrictRedis(
unix_socket_path=config["RedisSocket"], db=config["RedisDBEphemeral"]
) # Ephemeral - flushed on quit
g = StrictRedis(unix_socket_path=config["RedisSocket"], db=config["RedisDBPersistent"]) # Persistent

4
modules/network.py
View File

@ -77,7 +77,9 @@ class Network:
# e.g. freenode1 for the first relay on freenode network
keyFN = main.certPath + main.config["Key"]
certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace"))
contextFactory = DefaultOpenSSLContextFactory(
keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace")
)
bot = IRCBotFactory(self.net, num)
# host, port = self.relays[num]["host"], self.relays[num]["port"]
host, port = getRelay(num)

6
modules/provision.py
View File

@ -1,11 +1,13 @@
import main
from core.bot import deliverRelayCommands
from utils.deliver_relay_commands import deliverRelayCommands
from utils.logging.log import *
from twisted.internet import reactor
import modules.regproc
def provisionUserNetworkData(num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password):
def provisionUserNetworkData(
num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password
):
print("nick", nick)
print("altnick", altnick)
print("emails", emails)

11
modules/regproc.py
View File

@ -5,6 +5,7 @@ from utils.logging.debug import *
from copy import deepcopy
from random import choice
def needToRegister(net):
# Check if the network does not support authentication
networkObj = main.network[net]
@ -51,7 +52,7 @@ def substitute(net, num, token=None):
nickname = alias["nick"]
username = nickname + "/" + net
password = main.network[net].aliases[num]["password"]
#inst["email"] = inst["email"].replace("{nickname}", nickname)
# inst["email"] = inst["email"].replace("{nickname}", nickname)
for i in inst.keys():
if not isinstance(inst[i], str):
continue
@ -129,10 +130,14 @@ def registerTest(c):
if not main.IRCPool[name]._negativePass == True:
if c["type"] == "query" and c["nick"] == sinst["entity"]:
if sinst["checknegativemsg"] in c["msg"]:
confirmRegistration(c["net"], c["num"], negativepass=False) # Not passed negative check, report back
confirmRegistration(
c["net"], c["num"], negativepass=False
) # Not passed negative check, report back
return
if sinst["checkendnegative"] in c["msg"]:
confirmRegistration(c["net"], c["num"], negativepass=True) # Passed the negative check, report back
confirmRegistration(
c["net"], c["num"], negativepass=True
) # Passed the negative check, report back
return
if sinst["ping"]:
if sinst["checkmsg2"] in c["msg"] and c["nick"] == sinst["entity"]:

10
threshold
View File

@ -46,7 +46,10 @@ if __name__ == "__main__":
),
interface=main.config["Listener"]["Address"],
)
log("Threshold running with SSL on %s:%s" % (main.config["Listener"]["Address"], main.config["Listener"]["Port"]))
log(
"Threshold running with SSL on %s:%s"
% (main.config["Listener"]["Address"], main.config["Listener"]["Port"])
)
else:
reactor.listenTCP(
main.config["Listener"]["Port"],
@ -76,7 +79,10 @@ if __name__ == "__main__":
relay,
interface=main.config["RelayAPI"]["Address"],
)
log("Threshold relay running on %s:%s" % (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"]))
log(
"Threshold relay running on %s:%s"
% (main.config["RelayAPI"]["Address"], main.config["RelayAPI"]["Port"])
)
for net in main.network.keys():
main.network[net].start_bots()
modules.counters.setupCounterLoop()

7
utils/deliver_relay_commands.py Normal file
View File

@ -0,0 +1,7 @@
def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
keyFN = main.certPath + main.config["Key"]
certFN = main.certPath + main.config["Certificate"]
contextFactory = DefaultOpenSSLContextFactory(keyFN.encode("utf-8", "replace"), certFN.encode("utf-8", "replace"))
bot = IRCBotFactory(net=None, num=num, relayCommands=relayCommands, user=user, stage2=stage2)
host, port = getRelay(num)
rct = reactor.connectSSL(host, port, bot, contextFactory)

20
utils/loaders/command_loader.py
View File

@ -12,16 +12,16 @@ def loadCommands(allowDup=False):
if filename.endswith(".py") and filename != "__init__.py":
commandName = filename[0:-3]
className = commandName.capitalize() + "Command"
try:
module = __import__("commands.%s" % commandName)
if not commandName in CommandMap:
# try:
module = __import__("commands.%s" % commandName)
if not commandName in CommandMap:
CommandMap[commandName] = getattr(getattr(module, commandName), className)
debug("Registered command: %s" % commandName)
else:
if allowDup:
CommandMap[commandName] = getattr(getattr(module, commandName), className)
debug("Registered command: %s" % commandName)
else:
if allowDup:
CommandMap[commandName] = getattr(getattr(module, commandName), className)
debug("Registered command: %s" % commandName)
error("Duplicate command: %s" % (commandName))
except Exception as err:
error("Exception while loading command %s:\n%s" % (commandName, err))
error("Duplicate command: %s" % (commandName))
# except Exception as err:
# error("Exception while loading command %s:\n%s" % (commandName, err))