Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
29 changed files with 114 additions and 101 deletions
Showing only changes of commit 80c016761f - Show all commits

View File

@ -1,13 +1,17 @@
repos:
- repo: https://github.com/ambv/black
rev: 22.3.0
- repo: https://github.com/psf/black
rev: 22.6.0
hooks:
- id: black
args:
- --line-length=120
- repo: https://gitlab.com/pycqa/flake8
- repo: https://github.com/PyCQA/isort
rev: 5.10.1
hooks:
- id: isort
args: ["--profile", "black"]
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
hooks:
- id: flake8
args:
- "--max-line-length=120"
- id: flake8
args: [--max-line-length=120]

View File

@ -1,5 +1,6 @@
import main
from yaml import dump
import main
from modules import alias

View File

@ -1,6 +1,7 @@
import main
from yaml import dump
import main
class BlacklistCommand:
def __init__(self, *args):

View File

@ -1,5 +1,6 @@
from subprocess import PIPE, run
import main
from subprocess import run, PIPE
class DistCommand:

View File

@ -1,6 +1,7 @@
import main
from yaml import dump
import main
class EmailCommand:
def __init__(self, *args):

View File

@ -1,8 +1,10 @@
import main
from yaml import dump
from modules.network import Network
from string import digits
from yaml import dump
import main
from modules.network import Network
class NetworkCommand:
def __init__(self, *args):

View File

@ -1,6 +1,7 @@
import main
from yaml import dump
import main
class RelayCommand:
def __init__(self, *args):

View File

@ -1,7 +1,8 @@
from string import digits
import main
import modules.counters as count
import modules.userinfo as userinfo
from string import digits
class StatsCommand:

View File

@ -1,7 +1,9 @@
import main
from yaml import dump
from uuid import uuid4
from yaml import dump
import main
class TokenCommand:
def __init__(self, *args):

View File

@ -1,31 +1,25 @@
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.words.protocols.irc import IRCClient
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.internet import reactor
from twisted.words.protocols.irc import (
numeric_to_symbolic,
lowDequote,
IRCBadMessage,
)
import sys
from random import randint
from copy import deepcopy
from datetime import datetime
from random import randint
from modules import userinfo
from modules import counters
from modules import monitor
from modules import chankeep
from modules import regproc
from core.relay import sendRelayNotification
from utils.dedup import dedup
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.task import LoopingCall
from twisted.words.protocols.irc import (
IRCBadMessage,
IRCClient,
lowDequote,
numeric_to_symbolic,
)
import main
from utils.logging.log import log, warn, error
from core.relay import sendRelayNotification
from modules import chankeep, counters, monitor, regproc, userinfo
from utils.dedup import dedup
from utils.logging.debug import debug
from utils.logging.log import error, log, warn
from utils.logging.send import sendAll
from utils.parsing import parsen

View File

@ -1,7 +1,8 @@
import logstash
import logging
from json import dumps
import logstash
import main
logger = None

View File

@ -1,6 +1,6 @@
import main
from utils.logging.log import warn
from utils.logging.send import sendSuccess, sendFailure, sendInfo, incorrectUsage
from utils.logging.send import incorrectUsage, sendFailure, sendInfo, sendSuccess
def parseCommand(addr, authed, data):

View File

@ -1,5 +1,7 @@
from twisted.internet.protocol import Protocol, Factory
from json import dumps, loads
from twisted.internet.protocol import Factory, Protocol
import main
from utils.logging.log import log, warn

View File

@ -1,8 +1,8 @@
from twisted.internet.protocol import Protocol, Factory
import main
from utils.logging.log import log, warn
from twisted.internet.protocol import Factory, Protocol
import main
from core.parser import parseCommand
from utils.logging.log import log, warn
class Server(Protocol):

View File

@ -1,8 +1,9 @@
import json
import pickle
from redis import StrictRedis
from string import digits
from os import urandom
from string import digits
from redis import StrictRedis
# List of errors ZNC can give us
ZNCErrors = ["Error:", "Unable to load", "does not exist", "doesn't exist"]

View File

@ -1,7 +1,8 @@
import main
import random
import re
import main
def generate_password():
return "".join([chr(random.randint(0, 74) + 48) for i in range(32)])

View File

@ -1,11 +1,13 @@
import main
from utils.logging.log import log, warn, error
from utils.logging.debug import debug
from copy import deepcopy
from math import ceil
import modules.provision
from twisted.internet.threads import deferToThread
import main
import modules.provision
from utils.logging.debug import debug
from utils.logging.log import error, log, warn
def allRelaysActive(net):
relayNum = len(main.network[net].relays.keys())

View File

@ -1,6 +1,7 @@
import main
from twisted.internet.task import LoopingCall
import main
def event(name, eventType):
if "local" not in main.counters.keys():

View File

@ -1,5 +1,5 @@
from core.relay import sendRelayNotification
from core.logstash import sendLogstashNotification
from core.relay import sendRelayNotification
from modules import userinfo
from utils.dedup import dedup

View File

@ -1,13 +1,13 @@
from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory
import main
from core.bot import IRCBotFactory
from modules import alias
from modules.chankeep import nukeNetwork
from modules.regproc import needToRegister
from twisted.internet import reactor
from core.bot import IRCBotFactory
import main
from utils.logging.log import log
from utils.get import getRelay
from utils.logging.log import log
class Network:

View File

@ -1,7 +1,8 @@
import main
from utils.deliver_relay_commands import deliverRelayCommands
from twisted.internet import reactor
import main
import modules.regproc
from utils.deliver_relay_commands import deliverRelayCommands
def provisionUserNetworkData(

View File

@ -1,10 +1,11 @@
import main
from modules import provision
from utils.logging.log import error
from utils.logging.debug import debug
from copy import deepcopy
from random import choice
import main
from modules import provision
from utils.logging.debug import debug
from utils.logging.log import error
def needToRegister(net):
# Check if the network does not support authentication

View File

@ -1,8 +1,8 @@
from twisted.internet.threads import deferToThread
import main
from utils.logging.log import warn
from utils.logging.debug import trace
from utils.logging.log import warn
from utils.parsing import parsen

View File

@ -1,27 +1,20 @@
#!/usr/bin/env python
import sys
from codecs import getwriter
from signal import SIGINT, signal
from sys import stderr, stdout
from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory
import sys
from signal import signal, SIGINT
# from twisted.python import log
# from sys import stdout
# log.startLogging(stdout)
from sys import stdout, stderr # Import again because we want to override
from codecs import getwriter # fix printing odd shit to the terminal
import main
from utils.cleanup import handler
from utils.logging.log import log
from utils.loaders.command_loader import loadCommands
from core.server import ServerFactory
from core.relay import RelayFactory
import modules.counters
import core.logstash
import main
import modules.counters
from core.relay import RelayFactory
from core.server import ServerFactory
from utils.cleanup import handler
from utils.loaders.command_loader import loadCommands
from utils.logging.log import log
main.initMain()

View File

@ -1,5 +1,6 @@
import main
from twisted.internet import reactor
import main
from utils.logging.debug import debug
from utils.logging.log import log

View File

@ -1,7 +1,9 @@
from datetime import datetime
from csiphash import siphash24
from copy import deepcopy
from datetime import datetime
from json import dumps
from csiphash import siphash24
import main
from utils.logging.debug import debug

View File

@ -1,16 +1,17 @@
import main
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from datetime import datetime
from twisted.internet import reactor
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.ssl import DefaultOpenSSLContextFactory
from twisted.words.protocols.irc import IRCClient
from twisted.internet.protocol import ReconnectingClientFactory
from utils.parsing import parsen
from utils.logging.log import log, error
from utils.logging.send import sendAll
from modules import userinfo
from datetime import datetime
import main
from core.relay import sendRelayNotification
from modules import userinfo
from utils.get import getRelay
from utils.logging.log import error, log
from utils.logging.send import sendAll
from utils.parsing import parsen
# TODO: strip out non-relay functionality

View File

@ -1,10 +1,9 @@
from os import listdir
from main import CommandMap
from utils.logging.debug import debug
from utils.logging.log import error
from main import CommandMap
def loadCommands(allowDup=False):
for filename in listdir("commands"):

View File

@ -1,10 +1,9 @@
from os import listdir
from importlib import reload
import sys
from utils.logging.debug import debug
from importlib import reload
from os import listdir
from main import CommandMap
from utils.logging.debug import debug
def loadSingle(commandName):