Reformat code

This commit is contained in:
2022-09-04 21:40:04 +01:00
parent 0f717b987d
commit 6e00f70184
7 changed files with 50 additions and 130 deletions

46
db.py
View File

@@ -1,12 +1,13 @@
from math import ceil
import manticoresearch
import ujson
from manticoresearch.rest import ApiException
from numpy import array_split
from redis import StrictRedis
import util
from schemas.mc_s import schema
import ujson
from numpy import array_split
from math import ceil
configuration = manticoresearch.Configuration(host="http://monolith-db-1:9308")
api_client = manticoresearch.ApiClient(configuration)
@@ -15,6 +16,7 @@ api_instance = manticoresearch.IndexApi(api_client)
log = util.get_logger("db")
r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
def store_message(msg):
"""
Store a message into Manticore
@@ -30,27 +32,21 @@ def store_message(msg):
if schema[key].startswith("string"):
msg[key] = str(value)
body = [
{
"insert": {
"index": "main",
"doc": msg
}
}
]
body = [{"insert": {"index": "main", "doc": msg}}]
body_post = ""
for item in body:
body_post += ujson.dumps(item)
body_post += "\n"
#print(body_post)
# print(body_post)
try:
# Bulk index operations
api_response = api_instance.bulk(body_post, async_req=True)
#print(api_response)
api_instance.bulk(body_post, async_req=True)
# print(api_response)
except ApiException as e:
print("Exception when calling IndexApi->bulk: %s\n" % e)
def store_message_bulk(data):
"""
Store a message into Manticore
@@ -71,42 +67,38 @@ def store_message_bulk(data):
if schema[key].startswith("string"):
msg[key] = str(value)
body = {
"insert": {
"index": "main",
"doc": msg
}
}
body = {"insert": {"index": "main", "doc": msg}}
total.append(body)
body_post = ""
for item in total:
body_post += ujson.dumps(item)
body_post += "\n"
#print(body_post)
# print(body_post)
try:
# Bulk index operations
api_response = api_instance.bulk(body_post, async_req=True)
#print(api_response)
api_instance.bulk(body_post, async_req=True)
# print(api_response)
except ApiException as e:
print("Exception when calling IndexApi->bulk: %s\n" % e)
print("FINISHED PROCESSING SPLIT")
print("BULK FINISH")
def update_schema():
pass
def create_index(api_client):
util_instance = manticoresearch.UtilsApi(api_client)
schema_types = ", ".join([f"{k} {v}" for k,v in schema.items()])
schema_types = ", ".join([f"{k} {v}" for k, v in schema.items()])
create_query = f"create table if not exists main({schema_types}) engine='columnar'"
print("Schema types", create_query)
util_instance.sql(create_query)
create_index(api_client)
update_schema()