Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
12 changed files with 21 additions and 18 deletions
Showing only changes of commit 47312b04d4 - Show all commits

11
.gitignore vendored
View File

@ -2,15 +2,8 @@
*.pem
*.swp
__pycache__/
conf/config.json
conf/wholist.json
conf/counters.json
conf/tokens.json
conf/network.dat
conf/alias.json
conf/irc.json
conf/dist.sh
conf/blacklist.json
conf/live/
conf/cert/
env/
.idea/
.env

View File

@ -1 +0,0 @@
{}

View File

@ -1 +0,0 @@
{}

View File

@ -1 +0,0 @@
{}

View File

@ -1 +0,0 @@
{}

View File

@ -6,6 +6,9 @@ services:
build: ./docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"

View File

@ -6,6 +6,9 @@ services:
build: ./docker
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${THRESHOLD_CONFIG_DIR}:/code/conf/live
- ${THRESHOLD_TEMPLATE_DIR}:/code/conf/templates
- ${THRESHOLD_CERT_DIR}:/code/conf/cert
ports:
- "${THRESHOLD_LISTENER_PORT}:${THRESHOLD_LISTENER_PORT}"
- "${THRESHOLD_RELAY_PORT}:${THRESHOLD_RELAY_PORT}"

18
main.py
View File

@ -5,13 +5,14 @@ from os.path import exists
from string import digits
from redis import StrictRedis
from os import getenv
# List of errors ZNC can give us
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]
configPath = "conf/"
exampleConfigPath = "conf/example/"
certPath = "cert/"
configPath = getenv("THRESHOLD_CONFIG_DIR", "conf/live/")
templateConfigPath = getenv("THRESHOLD_TEMPLATE_DIR", "conf/templates/")
certPath = getenv("THRESHOLD_CERT_DIR", "conf/cert/")
filemap = {
# JSON configs
@ -70,6 +71,8 @@ def liveNets():
def saveConf(var):
if var in ("help", "aliasdata"):
return # no need to save this
if filemap[var][2] == "json":
with open(configPath + filemap[var][0], "w") as f:
json.dump(globals()[var], f, indent=4)
@ -83,21 +86,26 @@ def saveConf(var):
def loadConf(var):
if filemap[var][2] == "json":
filename = configPath + filemap[var][0]
# Only take the help from the templates
if var in ("help", "aliasdata"):
filename = templateConfigPath + filemap[var][0]
if not exists(filename):
# Load the template config
if var == "config":
filename = exampleConfigPath + filemap[var][0]
filename = templateConfigPath + filemap[var][0]
else:
# Everything else should be blank
globals()[var] = {}
return
with open(filename, "r") as f:
globals()[var] = json.load(f)
return
if var == "alias":
# This is a workaround to convert all the keys into integers since JSON
# turns them into strings...
# Dammit Jason!
global alias
alias = {int(x): y for x, y in alias.items()}
elif filemap[var][2] == "pickle":
try:
with open(configPath + filemap[var][0], "rb") as f: