Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
12 changed files with 21 additions and 18 deletions
Showing only changes of commit 47312b04d4 - Show all commits

11
.gitignore vendored
View File

@ -2,15 +2,8 @@
*.pem *.pem
*.swp *.swp
__pycache__/ __pycache__/
conf/config.json conf/live/
conf/wholist.json conf/cert/
conf/counters.json
conf/tokens.json
conf/network.dat
conf/alias.json
conf/irc.json
conf/dist.sh
conf/blacklist.json
env/ env/
.idea/ .idea/
.env .env

View File

@ -1 +0,0 @@
{}

View File

@ -1 +0,0 @@
{}

View File

@ -1 +0,0 @@
{}

View File

@ -1 +0,0 @@
{}

View File

@ -6,6 +6,9 @@ services:
build: ./docker build: ./docker
volumes: volumes:
- ${PORTAINER_GIT_DIR}:/code - ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
ports: ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"

View File

@ -6,6 +6,9 @@ services:
build: ./docker build: ./docker
volumes: volumes:
- ${PORTAINER_GIT_DIR}:/code - ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
ports: ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}" - "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}" - "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"

18
main.py
View File

@ -5,13 +5,14 @@ from os.path import exists
from string import digits from string import digits
from redis import StrictRedis from redis import StrictRedis
from os import getenv
# List of errors ZNC can give us # List of errors ZNC can give us
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
configPath = "conf/" configPath = getenv("THRESHOLD_CONFIG_DIR", "conf/live/")
exampleConfigPath = "conf/example/" templateConfigPath = getenv("THRESHOLD_TEMPLATE_DIR", "conf/templates/")
certPath = "cert/" certPath = getenv("THRESHOLD_CERT_DIR", "conf/cert/")
filemap = { filemap = {
# JSON configs # JSON configs
@ -70,6 +71,8 @@ def liveNets():
def saveConf(var): def saveConf(var):
if var in ("help", "aliasdata"):
return # no need to save this
if filemap[var][2] == "json": if filemap[var][2] == "json":
with open(configPath + filemap[var][0], "w") as f: with open(configPath + filemap[var][0], "w") as f:
json.dump(globals()[var], f, indent=4) json.dump(globals()[var], f, indent=4)
@ -83,21 +86,26 @@ def saveConf(var):
def loadConf(var): def loadConf(var):
if filemap[var][2] == "json": if filemap[var][2] == "json":
filename = configPath + filemap[var][0] filename = configPath + filemap[var][0]
# Only take the help from the templates
if var in ("help", "aliasdata"):
filename = templateConfigPath + filemap[var][0]
if not exists(filename): if not exists(filename):
# Load the template config
if var == "config": if var == "config":
filename = exampleConfigPath + filemap[var][0] filename = templateConfigPath + filemap[var][0]
else: else:
# Everything else should be blank
globals()[var] = {} globals()[var] = {}
return return
with open(filename, "r") as f: with open(filename, "r") as f:
globals()[var] = json.load(f) globals()[var] = json.load(f)
return
if var == "alias": if var == "alias":
# This is a workaround to convert all the keys into integers since JSON # This is a workaround to convert all the keys into integers since JSON
# turns them into strings... # turns them into strings...
# Dammit Jason! # Dammit Jason!
global alias global alias
alias = {int(x): y for x, y in alias.items()} alias = {int(x): y for x, y in alias.items()}
elif filemap[var][2] == "pickle": elif filemap[var][2] == "pickle":
try: try:
with open(configPath + filemap[var][0], "rb") as f: with open(configPath + filemap[var][0], "rb") as f: