Reformat code
This commit is contained in:
46
db.py
46
db.py
@@ -1,12 +1,13 @@
|
||||
from math import ceil
|
||||
|
||||
import manticoresearch
|
||||
import ujson
|
||||
from manticoresearch.rest import ApiException
|
||||
from numpy import array_split
|
||||
from redis import StrictRedis
|
||||
|
||||
import util
|
||||
from schemas.mc_s import schema
|
||||
import ujson
|
||||
from numpy import array_split
|
||||
from math import ceil
|
||||
|
||||
configuration = manticoresearch.Configuration(host="http://monolith-db-1:9308")
|
||||
api_client = manticoresearch.ApiClient(configuration)
|
||||
@@ -15,6 +16,7 @@ api_instance = manticoresearch.IndexApi(api_client)
|
||||
log = util.get_logger("db")
|
||||
r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
|
||||
|
||||
|
||||
def store_message(msg):
|
||||
"""
|
||||
Store a message into Manticore
|
||||
@@ -30,27 +32,21 @@ def store_message(msg):
|
||||
if schema[key].startswith("string"):
|
||||
msg[key] = str(value)
|
||||
|
||||
body = [
|
||||
{
|
||||
"insert": {
|
||||
"index": "main",
|
||||
"doc": msg
|
||||
}
|
||||
}
|
||||
]
|
||||
body = [{"insert": {"index": "main", "doc": msg}}]
|
||||
body_post = ""
|
||||
for item in body:
|
||||
body_post += ujson.dumps(item)
|
||||
body_post += "\n"
|
||||
|
||||
#print(body_post)
|
||||
# print(body_post)
|
||||
try:
|
||||
# Bulk index operations
|
||||
api_response = api_instance.bulk(body_post, async_req=True)
|
||||
#print(api_response)
|
||||
api_instance.bulk(body_post, async_req=True)
|
||||
# print(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling IndexApi->bulk: %s\n" % e)
|
||||
|
||||
|
||||
def store_message_bulk(data):
|
||||
"""
|
||||
Store a message into Manticore
|
||||
@@ -71,42 +67,38 @@ def store_message_bulk(data):
|
||||
if schema[key].startswith("string"):
|
||||
msg[key] = str(value)
|
||||
|
||||
body = {
|
||||
"insert": {
|
||||
"index": "main",
|
||||
"doc": msg
|
||||
}
|
||||
}
|
||||
body = {"insert": {"index": "main", "doc": msg}}
|
||||
total.append(body)
|
||||
|
||||
|
||||
body_post = ""
|
||||
for item in total:
|
||||
body_post += ujson.dumps(item)
|
||||
body_post += "\n"
|
||||
|
||||
#print(body_post)
|
||||
# print(body_post)
|
||||
try:
|
||||
# Bulk index operations
|
||||
api_response = api_instance.bulk(body_post, async_req=True)
|
||||
#print(api_response)
|
||||
api_instance.bulk(body_post, async_req=True)
|
||||
# print(api_response)
|
||||
except ApiException as e:
|
||||
print("Exception when calling IndexApi->bulk: %s\n" % e)
|
||||
print("FINISHED PROCESSING SPLIT")
|
||||
|
||||
print("BULK FINISH")
|
||||
|
||||
|
||||
def update_schema():
|
||||
pass
|
||||
|
||||
|
||||
def create_index(api_client):
|
||||
util_instance = manticoresearch.UtilsApi(api_client)
|
||||
schema_types = ", ".join([f"{k} {v}" for k,v in schema.items()])
|
||||
|
||||
schema_types = ", ".join([f"{k} {v}" for k, v in schema.items()])
|
||||
|
||||
create_query = f"create table if not exists main({schema_types}) engine='columnar'"
|
||||
print("Schema types", create_query)
|
||||
util_instance.sql(create_query)
|
||||
|
||||
|
||||
|
||||
create_index(api_client)
|
||||
update_schema()
|
||||
|
||||
Reference in New Issue
Block a user