Implement drug refresh and view
parent
37534b31bf
commit
0488a3e0b2
GPG Key ID:
5ACFCEED46C0904F
@ -0,0 +1,205 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import aiohttp
|
||||
import orjson
|
||||
from glom import glom
|
||||
from pydantic.error_wrappers import ValidationError
|
||||
|
||||
from core.lib import schemas
|
||||
from core.util import logs
|
||||
|
||||
# Return error if the schema for the message type is not found
|
||||
STRICT_VALIDATION = False
|
||||
|
||||
# Raise exception if the conversion schema is not found
|
||||
STRICT_CONVERSION = False
|
||||
|
||||
# TODO: Set them to True when all message types are implemented
|
||||
|
||||
log = logs.get_logger("clients")
|
||||
|
||||
|
||||
class NoSchema(Exception):
|
||||
"""
|
||||
Raised when:
|
||||
- The schema for the message type is not found
|
||||
- The conversion schema is not found
|
||||
- There is no schema library for the client
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchMethod(Exception):
|
||||
"""
|
||||
Client library has no such method.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class GenericAPIError(Exception):
|
||||
"""
|
||||
Generic API error.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def is_camel_case(s):
|
||||
return s != s.lower() and s != s.upper() and "_" not in s
|
||||
|
||||
|
||||
def snake_to_camel(word):
|
||||
if is_camel_case(word):
|
||||
return word
|
||||
return "".join(x.capitalize() or "_" for x in word.split("_"))
|
||||
|
||||
|
||||
DEFAULT_HEADERS = {
|
||||
"accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
class BaseClient(ABC):
|
||||
token = None
|
||||
|
||||
async def __new__(cls, *a, **kw):
|
||||
instance = super().__new__(cls)
|
||||
await instance.__init__(*a, **kw)
|
||||
return instance
|
||||
|
||||
async def __init__(self):
|
||||
"""
|
||||
Initialise the client.
|
||||
:param instance: the database object, e.g. Aggregator
|
||||
"""
|
||||
name = self.__class__.__name__
|
||||
self.name = name.replace("Client", "").lower()
|
||||
# self.instance = instance
|
||||
self.client = None
|
||||
|
||||
await self.connect()
|
||||
|
||||
@abstractmethod
|
||||
async def connect(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def schema(self):
|
||||
"""
|
||||
Get the schema library for the client.
|
||||
"""
|
||||
# Does the schemas library have a library for this client name?
|
||||
if hasattr(schemas, f"{self.name}_s"):
|
||||
schema_instance = getattr(schemas, f"{self.name}_s")
|
||||
else:
|
||||
log.error(f"No schema library for {self.name}")
|
||||
raise Exception(f"No schema library for client {self.name}")
|
||||
|
||||
return schema_instance
|
||||
|
||||
def get_schema(self, method, convert=False):
|
||||
if isinstance(method, str):
|
||||
to_camel = snake_to_camel(method)
|
||||
else:
|
||||
to_camel = snake_to_camel(method.__class__.__name__)
|
||||
if convert:
|
||||
to_camel = f"{to_camel}Schema"
|
||||
|
||||
# if hasattr(self.schema, method):
|
||||
# schema = getattr(self.schema, method)
|
||||
if hasattr(self.schema, to_camel):
|
||||
schema = getattr(self.schema, to_camel)
|
||||
else:
|
||||
raise NoSchema(f"Could not get schema: {to_camel}")
|
||||
return schema
|
||||
|
||||
async def call_method(self, method, *args, **kwargs):
|
||||
"""
|
||||
Call a method with aiohttp.
|
||||
"""
|
||||
if kwargs.get("append_slash", True):
|
||||
path = f"{self.url}/{method}/"
|
||||
else:
|
||||
path = f"{self.url}/{method}"
|
||||
|
||||
http_method = kwargs.get("http_method", "get")
|
||||
|
||||
cast = {
|
||||
"headers": DEFAULT_HEADERS,
|
||||
}
|
||||
|
||||
# Use the token if it's set
|
||||
if self.token is not None:
|
||||
cast["headers"]["Authorization"] = f"Bearer {self.token}"
|
||||
|
||||
if "data" in kwargs:
|
||||
cast["data"] = orjson.dumps(kwargs["data"])
|
||||
|
||||
# Use the method to send a HTTP request
|
||||
async with aiohttp.ClientSession() as session:
|
||||
session_method = getattr(session, http_method)
|
||||
async with session_method(path, **cast) as response:
|
||||
response_json = await response.json()
|
||||
return response_json
|
||||
|
||||
def convert_spec(self, response, method):
|
||||
"""
|
||||
Convert an API response to the requested spec.
|
||||
:raises NoSchema: If the conversion schema is not found
|
||||
"""
|
||||
schema = self.get_schema(method, convert=True)
|
||||
|
||||
# Use glom to convert the response to the schema
|
||||
converted = glom(response, schema)
|
||||
return converted
|
||||
|
||||
def validate_response(self, response, method):
|
||||
schema = self.get_schema(method)
|
||||
# Return a dict of the validated response
|
||||
try:
|
||||
response_valid = schema(**response).dict()
|
||||
except ValidationError as e:
|
||||
log.error(f"Error validating {method} response: {response}")
|
||||
log.error(f"Errors: {e}")
|
||||
raise GenericAPIError("Error validating response")
|
||||
return response_valid
|
||||
|
||||
def method_filter(self, method):
|
||||
"""
|
||||
Return a new method.
|
||||
"""
|
||||
return method
|
||||
|
||||
async def call(self, method, *args, **kwargs):
|
||||
"""
|
||||
Call the exchange API and validate the response
|
||||
:raises NoSchema: If the method is not in the schema mapping
|
||||
:raises ValidationError: If the response cannot be validated
|
||||
"""
|
||||
# try:
|
||||
response = await self.call_method(method, *args, **kwargs)
|
||||
# except (APIError, V20Error) as e:
|
||||
# log.error(f"Error calling method {method}: {e}")
|
||||
# raise GenericAPIError(e)
|
||||
|
||||
if "schema" in kwargs:
|
||||
method = kwargs["schema"]
|
||||
else:
|
||||
method = self.method_filter(method)
|
||||
try:
|
||||
response_valid = self.validate_response(response, method)
|
||||
except NoSchema as e:
|
||||
log.error(f"{e} - {response}")
|
||||
response_valid = response
|
||||
# Convert the response to a format that we can use
|
||||
try:
|
||||
response_converted = self.convert_spec(response_valid, method)
|
||||
except NoSchema as e:
|
||||
log.error(f"{e} - {response}")
|
||||
response_converted = response_valid
|
||||
|
||||
# return (True, response_converted)
|
||||
return response_converted
|
||||
@ -0,0 +1,26 @@
|
||||
from abc import ABC
|
||||
|
||||
from core.models import Source
|
||||
from core.util import logs
|
||||
|
||||
log = logs.get_logger("graphql")
|
||||
|
||||
|
||||
class GraphQLClient(ABC):
|
||||
"""
|
||||
GraphQL API handler.
|
||||
"""
|
||||
|
||||
async def connect(self):
|
||||
try:
|
||||
source = Source.objects.get(name=self.source_name)
|
||||
except Source.DoesNotExist:
|
||||
source = Source(
|
||||
name=self.source_name,
|
||||
type=self.source_type,
|
||||
endpoint=self.source_endpoint,
|
||||
score=self.source_score,
|
||||
)
|
||||
source.save()
|
||||
self.url = source.endpoint
|
||||
self.source = source
|
||||
@ -0,0 +1,297 @@
|
||||
from core.clients.base import BaseClient
|
||||
from core.clients.graphql import GraphQLClient
|
||||
from core.models import SEI, Dosage, Drug, Effect, Entry, Timing
|
||||
|
||||
|
||||
class PsychWikiClient(GraphQLClient, BaseClient):
|
||||
# url = "https://api.psychonautwiki.org"
|
||||
search_limit = 5000
|
||||
source_name = "Psychonaut Wiki GraphQL API"
|
||||
source_type = "DWIKI"
|
||||
source_endpoint = "https://api.psychonautwiki.org"
|
||||
source_score = 75
|
||||
|
||||
async def update_drugs(self):
|
||||
data = await self.get_all_data()
|
||||
self.store_data(data)
|
||||
return len(data)
|
||||
|
||||
def store_data(self, data):
|
||||
"""
|
||||
Store the data in the database.
|
||||
"""
|
||||
for drug in data["substances"]:
|
||||
try:
|
||||
drug_obj = Drug.objects.get(name=drug["name"])
|
||||
except Drug.DoesNotExist:
|
||||
drug_obj = Drug(name=drug["name"])
|
||||
|
||||
if "commonNames" in drug:
|
||||
if drug["commonNames"]:
|
||||
print("common names", drug["commonNames"])
|
||||
drug_obj.common_names = ",".join(drug["commonNames"])
|
||||
if "class" in drug:
|
||||
if drug["class"]:
|
||||
if "psychoactive" in drug["class"]:
|
||||
if drug["class"]["psychoactive"]:
|
||||
drug_obj.drug_class = ",".join(
|
||||
drug["class"]["psychoactive"]
|
||||
)
|
||||
try:
|
||||
entry = Entry.objects.get(source=self.source, url=drug["url"])
|
||||
except Entry.DoesNotExist:
|
||||
entry = Entry.objects.create(source=self.source, url=drug["url"])
|
||||
if not drug_obj.pk:
|
||||
drug_obj.save()
|
||||
if entry not in drug_obj.links.all():
|
||||
drug_obj.links.add(entry)
|
||||
if "roas" in drug:
|
||||
for roa in drug["roas"]:
|
||||
if "name" in roa:
|
||||
roa_name = roa["name"]
|
||||
|
||||
# Parsing dosage information
|
||||
dose = roa["dose"]
|
||||
|
||||
if dose:
|
||||
dosage_data = {
|
||||
"entry": entry,
|
||||
"roa": roa_name,
|
||||
"unit": dose["units"],
|
||||
}
|
||||
# Check and assign dosage values
|
||||
for dose_type in [
|
||||
"threshold",
|
||||
"light",
|
||||
"common",
|
||||
"strong",
|
||||
"heavy",
|
||||
]:
|
||||
if dose_type in dose:
|
||||
if isinstance(dose[dose_type], dict):
|
||||
dosage_data[f"{dose_type}_lower"] = dose[
|
||||
dose_type
|
||||
]["min"]
|
||||
dosage_data[f"{dose_type}_upper"] = dose[
|
||||
dose_type
|
||||
]["max"]
|
||||
else:
|
||||
dosage_data[f"{dose_type}_lower"] = dose[
|
||||
dose_type
|
||||
]
|
||||
dosage_data[f"{dose_type}_upper"] = dose[
|
||||
dose_type
|
||||
]
|
||||
|
||||
# Check if Dosage already exists and is linked to the Drug
|
||||
dosage, created = Dosage.objects.get_or_create(
|
||||
**dosage_data
|
||||
)
|
||||
if created or dosage not in drug_obj.dosages.all():
|
||||
drug_obj.dosages.add(dosage)
|
||||
|
||||
# Parsing timing information
|
||||
timing = roa["duration"]
|
||||
|
||||
print("TIMING", timing)
|
||||
# Check and assign timing values
|
||||
if timing:
|
||||
timing_data = {"entry": entry, "roa": roa_name}
|
||||
for time_type in [
|
||||
"onset",
|
||||
"comeup",
|
||||
"peak",
|
||||
"offset",
|
||||
"total",
|
||||
]:
|
||||
if (
|
||||
time_type in timing
|
||||
and timing[time_type] is not None
|
||||
):
|
||||
unit = timing[time_type].get("units", "hours")
|
||||
|
||||
# Handle case where timing value is a single integer
|
||||
if isinstance(timing[time_type], int):
|
||||
lower = timing[time_type]
|
||||
upper = None
|
||||
else:
|
||||
lower = timing[time_type].get("min")
|
||||
upper = timing[time_type].get("max")
|
||||
|
||||
if unit == "minutes" and lower is not None:
|
||||
lower = lower / 60.0
|
||||
if upper is not None:
|
||||
upper = upper / 60.0
|
||||
|
||||
timing_data[f"{time_type}_lower"] = lower
|
||||
timing_data[f"{time_type}_upper"] = upper
|
||||
timing_data[
|
||||
"unit"
|
||||
] = "HOURS" # Store all times in hours
|
||||
|
||||
# Check if Timing already exists and is linked to the Drug
|
||||
timing_obj, created = Timing.objects.get_or_create(
|
||||
**timing_data
|
||||
)
|
||||
if created or timing_obj not in drug_obj.timings.all():
|
||||
drug_obj.timings.add(timing_obj)
|
||||
if "effects" in drug:
|
||||
# Create or retrieve Effect object linked to the Entry
|
||||
effect_obj, effect_created = Effect.objects.get_or_create(entry=entry)
|
||||
|
||||
for effect in drug["effects"]:
|
||||
# Create or retrieve SEI object
|
||||
sei_obj, sei_created = SEI.objects.get_or_create(
|
||||
name=effect["name"],
|
||||
url=effect.get(
|
||||
"url", ""
|
||||
), # Using .get() to handle missing urls
|
||||
)
|
||||
|
||||
# Link SEI object to Effect if not already linked
|
||||
if (
|
||||
sei_created
|
||||
or sei_obj not in effect_obj.subjective_effects.all()
|
||||
):
|
||||
effect_obj.subjective_effects.add(sei_obj)
|
||||
# Link Effect object to Drug if not already linked
|
||||
if effect_created or effect_obj not in drug_obj.effects.all():
|
||||
drug_obj.effects.add(effect_obj)
|
||||
|
||||
print("SAVING DRUG", drug_obj)
|
||||
drug_obj.save()
|
||||
|
||||
async def get_drugs_list(self):
|
||||
"""
|
||||
Get all drug names from PsychWiki
|
||||
"""
|
||||
body = {"query": "{substances(limit: %i) {name}}" % self.search_limit}
|
||||
result = await self.call(
|
||||
"?",
|
||||
http_method="post",
|
||||
data=body,
|
||||
schema="GetDrugsList",
|
||||
append_slash=False,
|
||||
)
|
||||
print("RESULT", result)
|
||||
|
||||
return result["data"]
|
||||
|
||||
async def get_all_data(self):
|
||||
"""
|
||||
Get all the drug data from PsychWiki (warning - intensive)
|
||||
"""
|
||||
body = {
|
||||
"query": """
|
||||
{
|
||||
substances(limit: %i) {
|
||||
name
|
||||
url
|
||||
featured
|
||||
summary
|
||||
addictionPotential
|
||||
toxicity
|
||||
crossTolerances
|
||||
commonNames
|
||||
class {
|
||||
chemical
|
||||
psychoactive
|
||||
}
|
||||
tolerance {
|
||||
full
|
||||
half
|
||||
zero
|
||||
}
|
||||
roas {
|
||||
name
|
||||
dose {
|
||||
units
|
||||
threshold
|
||||
heavy
|
||||
common {
|
||||
min
|
||||
max
|
||||
}
|
||||
light {
|
||||
min
|
||||
max
|
||||
}
|
||||
strong {
|
||||
min
|
||||
max
|
||||
}
|
||||
}
|
||||
duration {
|
||||
afterglow {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
comeup {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
duration {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
offset {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
onset {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
peak {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
total {
|
||||
min
|
||||
max
|
||||
units
|
||||
}
|
||||
}
|
||||
bioavailability {
|
||||
min
|
||||
max
|
||||
}
|
||||
}
|
||||
effects {
|
||||
name
|
||||
url
|
||||
}
|
||||
images {
|
||||
thumb
|
||||
image
|
||||
}
|
||||
uncertainInteractions {
|
||||
name
|
||||
}
|
||||
unsafeInteractions {
|
||||
name
|
||||
}
|
||||
dangerousInteractions {
|
||||
name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
"""
|
||||
% self.search_limit
|
||||
}
|
||||
result = await self.call(
|
||||
"?",
|
||||
http_method="post",
|
||||
data=body,
|
||||
schema="GetDrugsList",
|
||||
append_slash=False,
|
||||
)
|
||||
|
||||
return result["data"]
|
||||
@ -0,0 +1 @@
|
||||
from core.lib.schemas import psychwiki_s # noqa
|
||||
@ -0,0 +1,10 @@
|
||||
from pydantic import BaseModel, Extra
|
||||
|
||||
|
||||
class MyModel(BaseModel):
|
||||
class Config:
|
||||
extra = Extra.forbid
|
||||
|
||||
|
||||
# class GetDrugsList(MyModel):
|
||||
# ...
|
||||
@ -0,0 +1,41 @@
|
||||
# Generated by Django 4.2.9 on 2024-01-07 14:00
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0005_dosage_roa_timing_roa_alter_experiencedose_roa'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name='entry',
|
||||
old_name='slug',
|
||||
new_name='url',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='sei',
|
||||
name='description',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='sei',
|
||||
name='subtype',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='sei',
|
||||
name='type',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='sei',
|
||||
name='name',
|
||||
field=models.CharField(default='DEFAULT', max_length=255),
|
||||
preserve_default=False,
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='sei',
|
||||
name='url',
|
||||
field=models.CharField(blank=True, max_length=1024, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,53 @@
|
||||
# Generated by Django 4.2.9 on 2024-01-07 14:12
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0006_rename_slug_entry_url_remove_sei_description_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='comeup_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='comeup_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='onset_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='onset_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='peak_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='peak_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='total_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='timing',
|
||||
name='total_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,63 @@
|
||||
# Generated by Django 4.2.9 on 2024-01-07 14:14
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0007_alter_timing_comeup_lower_alter_timing_comeup_upper_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='common_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='common_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='heavy_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='heavy_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='light_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='light_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='strong_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='strong_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='threshold_lower',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='dosage',
|
||||
name='threshold_upper',
|
||||
field=models.FloatField(blank=True, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,23 @@
|
||||
# Generated by Django 4.2.9 on 2024-01-07 14:19
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0008_alter_dosage_common_lower_alter_dosage_common_upper_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='drug',
|
||||
name='common_name',
|
||||
field=models.CharField(blank=True, max_length=1024, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='drug',
|
||||
name='drug_class',
|
||||
field=models.CharField(blank=True, max_length=255, null=True),
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,18 @@
|
||||
import asyncio
|
||||
|
||||
|
||||
def synchronize_async_helper(to_await):
|
||||
async_response = []
|
||||
|
||||
async def run_and_capture_result():
|
||||
r = await to_await
|
||||
async_response.append(r)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
coroutine = run_and_capture_result()
|
||||
loop.run_until_complete(coroutine)
|
||||
return async_response[0]
|
||||
Loading…
Reference in New Issue