Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
29 changed files with 114 additions and 101 deletions
Showing only changes of commit 80c016761f - Show all commits

View File

@ -1,13 +1,17 @@
repos: repos:
- repo: https://github.com/ambv/black - repo: https://github.com/psf/black
rev: 22.3.0 rev: 22.6.0
hooks: hooks:
- id: black - id: black
args: args:
- --line-length=120 - --line-length=120
- repo: https://gitlab.com/pycqa/flake8 - repo: https://github.com/PyCQA/isort
rev: 5.10.1
hooks:
- id: isort
args: ["--profile", "black"]
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1 rev: 4.0.1
hooks: hooks:
- id: flake8 - id: flake8
args: args: [--max-line-length=120]
- "--max-line-length=120"

View File

@ -1,5 +1,6 @@
import main
from yaml import dump from yaml import dump
import main
from modules import alias from modules import alias

View File

@ -1,6 +1,7 @@
import main
from yaml import dump from yaml import dump
import main
class BlacklistCommand: class BlacklistCommand:
def __init__(self, *args): def __init__(self, *args):

View File

@ -1,5 +1,6 @@
from subprocess import PIPE, run
import main import main
from subprocess import run, PIPE
class DistCommand: class DistCommand:

View File

@ -1,6 +1,7 @@
import main
from yaml import dump from yaml import dump
import main
class EmailCommand: class EmailCommand:
def __init__(self, *args): def __init__(self, *args):

View File

@ -1,8 +1,10 @@
import main
from yaml import dump
from modules.network import Network
from string import digits from string import digits
from yaml import dump
import main
from modules.network import Network
class NetworkCommand: class NetworkCommand:
def __init__(self, *args): def __init__(self, *args):

View File

@ -1,6 +1,7 @@
import main
from yaml import dump from yaml import dump
import main
class RelayCommand: class RelayCommand:
def __init__(self, *args): def __init__(self, *args):

View File

@ -1,7 +1,8 @@
from string import digits
import main import main
import modules.counters as count import modules.counters as count
import modules.userinfo as userinfo import modules.userinfo as userinfo
from string import digits
class StatsCommand: class StatsCommand:

View File

@ -1,7 +1,9 @@
import main
from yaml import dump
from uuid import uuid4 from uuid import uuid4
from yaml import dump
import main
class TokenCommand: class TokenCommand:
def __init__(self, *args): def __init__(self, *args):

View File

@ -1,31 +1,25 @@
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.internet import reactor
from twisted.words.protocols.irc import (
numeric_to_symbolic,
lowDequote,
IRCBadMessage,
)
import sys import sys
from random import randint
from copy import deepcopy from copy import deepcopy
from datetime import datetime from datetime import datetime
from random import randint
from modules import userinfo from twisted.internet import reactor
from modules import counters from twisted.internet.defer import Deferred
from modules import monitor from twisted.internet.protocol import ReconnectingClientFactory
from modules import chankeep from twisted.internet.task import LoopingCall
from modules import regproc from twisted.words.protocols.irc import (
IRCBadMessage,
from core.relay import sendRelayNotification IRCClient,
from utils.dedup import dedup lowDequote,
numeric_to_symbolic,
)
import main import main
from utils.logging.log import log, warn, error from core.relay import sendRelayNotification
from modules import chankeep, counters, monitor, regproc, userinfo
from utils.dedup import dedup
from utils.logging.debug import debug from utils.logging.debug import debug
from utils.logging.log import error, log, warn
from utils.logging.send import sendAll from utils.logging.send import sendAll
from utils.parsing import parsen from utils.parsing import parsen

View File

@ -1,7 +1,8 @@
import logstash
import logging import logging
from json import dumps from json import dumps
import logstash
import main import main
logger = None logger = None

View File

@ -1,6 +1,6 @@
import main import main
from utils.logging.log import warn from utils.logging.log import warn
from utils.logging.send import sendSuccess, sendFailure, sendInfo, incorrectUsage from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess
def parseCommand(addr, authed, data): def parseCommand(addr, authed, data):

View File

@ -1,5 +1,7 @@
from twisted.internet.protocol import Protocol, Factory
from json import dumps, loads from json import dumps, loads
from twisted.internet.protocol import Factory, Protocol
import main import main
from utils.logging.log import log, warn from utils.logging.log import log, warn

View File

@ -1,8 +1,8 @@
from twisted.internet.protocol import Protocol, Factory from twisted.internet.protocol import Factory, Protocol
import main
from utils.logging.log import log, warn
import main
from core.parser import parseCommand from core.parser import parseCommand
from utils.logging.log import log, warn
class Server(Protocol): class Server(Protocol):

View File

@ -1,8 +1,9 @@
import json import json
import pickle import pickle
from redis import StrictRedis
from string import digits
from os import urandom from os import urandom
from string import digits
from redis import StrictRedis
# List of errors ZNC can give us # List of errors ZNC can give us
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"] ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]

View File

@ -1,7 +1,8 @@
import main
import random import random
import re import re
import main
def generate_password(): def generate_password():
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)]) return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])

View File

@ -1,11 +1,13 @@
import main
from utils.logging.log import log, warn, error
from utils.logging.debug import debug
from copy import deepcopy from copy import deepcopy
from math import ceil from math import ceil
import modules.provision
from twisted.internet.threads import deferToThread from twisted.internet.threads import deferToThread
import main
import modules.provision
from utils.logging.debug import debug
from utils.logging.log import error, log, warn
def allRelaysActive(net): def allRelaysActive(net):
relayNum = len(main.network[net].relays.keys()) relayNum = len(main.network[net].relays.keys())

View File

@ -1,6 +1,7 @@
import main
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
import main
def event(name, eventType): def event(name, eventType):
if "local" not in main.counters.keys(): if "local" not in main.counters.keys():

View File

@ -1,5 +1,5 @@
from core.relay import sendRelayNotification
from core.logstash import sendLogstashNotification from core.logstash import sendLogstashNotification
from core.relay import sendRelayNotification
from modules import userinfo from modules import userinfo
from utils.dedup import dedup from utils.dedup import dedup

View File

@ -1,13 +1,13 @@
from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.ssl import DefaultOpenSSLContextFactory
import main
from core.bot import IRCBotFactory
from modules import alias from modules import alias
from modules.chankeep import nukeNetwork from modules.chankeep import nukeNetwork
from modules.regproc import needToRegister from modules.regproc import needToRegister
from twisted.internet import reactor
from core.bot import IRCBotFactory
import main
from utils.logging.log import log
from utils.get import getRelay from utils.get import getRelay
from utils.logging.log import log
class Network: class Network:

View File

@ -1,7 +1,8 @@
import main
from utils.deliver_relay_commands import deliverRelayCommands
from twisted.internet import reactor from twisted.internet import reactor
import main
import modules.regproc import modules.regproc
from utils.deliver_relay_commands import deliverRelayCommands
def provisionUserNetworkData( def provisionUserNetworkData(

View File

@ -1,10 +1,11 @@
import main
from modules import provision
from utils.logging.log import error
from utils.logging.debug import debug
from copy import deepcopy from copy import deepcopy
from random import choice from random import choice
import main
from modules import provision
from utils.logging.debug import debug
from utils.logging.log import error
def needToRegister(net): def needToRegister(net):
# Check if the network does not support authentication # Check if the network does not support authentication

View File

@ -1,8 +1,8 @@
from twisted.internet.threads import deferToThread from twisted.internet.threads import deferToThread
import main import main
from utils.logging.log import warn
from utils.logging.debug import trace from utils.logging.debug import trace
from utils.logging.log import warn
from utils.parsing import parsen from utils.parsing import parsen

View File

@ -1,27 +1,20 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys
from codecs import getwriter
from signal import SIGINT, signal
from sys import stderr, stdout
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.ssl import DefaultOpenSSLContextFactory
import sys
from signal import signal, SIGINT
# from twisted.python import log
# from sys import stdout
# log.startLogging(stdout)
from sys import stdout, stderr # Import again because we want to override
from codecs import getwriter # fix printing odd shit to the terminal
import main
from utils.cleanup import handler
from utils.logging.log import log
from utils.loaders.command_loader import loadCommands
from core.server import ServerFactory
from core.relay import RelayFactory
import modules.counters
import core.logstash import core.logstash
import main
import modules.counters
from core.relay import RelayFactory
from core.server import ServerFactory
from utils.cleanup import handler
from utils.loaders.command_loader import loadCommands
from utils.logging.log import log
main.initMain() main.initMain()

View File

@ -1,5 +1,6 @@
import main
from twisted.internet import reactor from twisted.internet import reactor
import main
from utils.logging.debug import debug from utils.logging.debug import debug
from utils.logging.log import log from utils.logging.log import log

View File

@ -1,7 +1,9 @@
from datetime import datetime
from csiphash import siphash24
from copy import deepcopy from copy import deepcopy
from datetime import datetime
from json import dumps from json import dumps
from csiphash import siphash24
import main import main
from utils.logging.debug import debug from utils.logging.debug import debug

View File

@ -1,16 +1,17 @@
import main from datetime import datetime
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from twisted.words.protocols.irc import IRCClient from twisted.words.protocols.irc import IRCClient
from twisted.internet.protocol import ReconnectingClientFactory import main
from utils.parsing import parsen
from utils.logging.log import log, error
from utils.logging.send import sendAll
from modules import userinfo
from datetime import datetime
from core.relay import sendRelayNotification from core.relay import sendRelayNotification
from modules import userinfo
from utils.get import getRelay from utils.get import getRelay
from utils.logging.log import error, log
from utils.logging.send import sendAll
from utils.parsing import parsen
# TODO: strip out non-relay functionality # TODO: strip out non-relay functionality

View File

@ -1,10 +1,9 @@
from os import listdir from os import listdir
from main import CommandMap
from utils.logging.debug import debug from utils.logging.debug import debug
from utils.logging.log import error from utils.logging.log import error
from main import CommandMap
def loadCommands(allowDup=False): def loadCommands(allowDup=False):
for filename in listdir("commands"): for filename in listdir("commands"):

View File

@ -1,10 +1,9 @@
from os import listdir
from importlib import reload
import sys import sys
from importlib import reload
from utils.logging.debug import debug from os import listdir
from main import CommandMap from main import CommandMap
from utils.logging.debug import debug
def loadSingle(commandName): def loadSingle(commandName):