XF/pluto
XF
/
pluto
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement async for expensive Nordigen calls

This commit is contained in:
Mark Veidemanis 2022-05-02 16:33:00 +01:00
parent 620adbe123
commit 9f8718a83b
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
2 changed files with 54 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
handler/sinks/nordigen.py
View File

@ -1,5 +1,6 @@
# Twisted/Klein imports # Twisted/Klein imports
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from twisted.internet.defer import inlineCallbacks
# Other library imports # Other library imports
import requests import requests
@ -17,6 +18,7 @@ from lib.serde.nordigen import (
) )
from serde import ValidationError from serde import ValidationError
from hashlib import sha256 from hashlib import sha256
import treq
# Project imports # Project imports
from settings import settings from settings import settings
@ -34,20 +36,28 @@ class Nordigen(util.Base):
self.token = None self.token = None
self.banks = {} self.banks = {}
self.authed = False self.authed = False
self.requisitions = None
# Get the banks from the config and cache them # Get the banks from the config and cache them
self.log.debug("Getting mapped accounts.")
self.get_mapped_accounts() self.get_mapped_accounts()
self.log.debug("Finished getting mapped accounts.")
self.log.debug("Creating loop to get access token.")
self.lc = LoopingCall(self.get_access_token) self.lc = LoopingCall(self.get_access_token)
self.lc.start(int(settings.Nordigen.TokenRefreshSec)) self.lc.start(int(settings.Nordigen.TokenRefreshSec))
self.log.debug("Finished creating loops.")
def __authed__(self): def __authed__(self):
""" """
Called when we have received the access token. Called when we have received the access token.
""" """
self.log.info("Connection authenticated.") self.log.info("Connection authenticated.")
account_infos = self.get_all_account_info() # self.get_requisitions()
d = self.get_all_account_info()
d.addCallback(self.got_all_account_info)
def got_all_account_info(self, account_infos):
# Filter for added accounts since we only do that for TrueLayer # Filter for added accounts since we only do that for TrueLayer
account_infos = { account_infos = {
bank: accounts bank: accounts
@ -64,8 +74,23 @@ class Nordigen(util.Base):
def transaction_loop(self): def transaction_loop(self):
for account_id in self.banks: for account_id in self.banks:
transactions = self.get_transactions(account_id) transactions = self.get_transactions(account_id)
self.sinks.got_transactions("nordigen", account_id, transactions) transactions.addCallback(self.got_transactions, account_id)
def got_transactions(self, transactions, account_id):
self.sinks.got_transactions("nordigen", account_id, transactions)
def generic_deferred(self, response, dest_func):
"""
Generic function to take a treq response and fire a callback with
its content to dest_func.
:param response: a treq response
:param dest_func: function to call with the response data
"""
self.log.debug(f"Generic deferred received: {response}")
content = response.content()
content.addCallback(dest_func)
@inlineCallbacks
def get_access_token(self): def get_access_token(self):
""" """
Get an access token. Get an access token.
@ -81,9 +106,11 @@ class Nordigen(util.Base):
"secret_key": settings.Nordigen.Key, "secret_key": settings.Nordigen.Key,
} }
path = f"{settings.Nordigen.Base}/token/new/" path = f"{settings.Nordigen.Base}/token/new/"
r = requests.post(path, headers=headers, data=dumps(data)) self.log.debug("Getting new access token.")
d = yield treq.post(path, headers=headers, data=data)
content = yield d.content()
try: try:
obj = AccessToken.from_json(r.content) obj = AccessToken.from_json(content)
except ValidationError as err: except ValidationError as err:
self.log.error(f"Validation error: {err}") self.log.error(f"Validation error: {err}")
return return
@ -168,6 +195,7 @@ class Nordigen(util.Base):
return False return False
return link return link
@inlineCallbacks
def get_requisitions(self): def get_requisitions(self):
""" """
Get a list of active accounts. Get a list of active accounts.
@ -177,9 +205,10 @@ class Nordigen(util.Base):
"Authorization": f"Bearer {self.token}", "Authorization": f"Bearer {self.token}",
} }
path = f"{settings.Nordigen.Base}/requisitions" path = f"{settings.Nordigen.Base}/requisitions"
r = requests.get(path, headers=headers) d = yield treq.get(path, headers=headers)
content = yield d.content()
try: try:
obj = Requisitions.from_json(r.content) obj = Requisitions.from_json(content)
except ValidationError as err: except ValidationError as err:
self.log.error(f"Validation error: {err}") self.log.error(f"Validation error: {err}")
return return
@ -209,6 +238,7 @@ class Nordigen(util.Base):
parsed = obj.to_dict() parsed = obj.to_dict()
return parsed return parsed
@inlineCallbacks
def get_accounts(self, requisition): def get_accounts(self, requisition):
""" """
Get a list of accounts for a requisition. Get a list of accounts for a requisition.
@ -218,9 +248,10 @@ class Nordigen(util.Base):
"Authorization": f"Bearer {self.token}", "Authorization": f"Bearer {self.token}",
} }
path = f"{settings.Nordigen.Base}/requisitions/{requisition}/" path = f"{settings.Nordigen.Base}/requisitions/{requisition}/"
r = requests.get(path, headers=headers) d = yield treq.get(path, headers=headers)
content = yield d.content()
try: try:
obj = Agreement.from_json(r.content) obj = Agreement.from_json(content)
except ValidationError as err: except ValidationError as err:
self.log.error(f"Validation error: {err}") self.log.error(f"Validation error: {err}")
return return
@ -236,6 +267,7 @@ class Nordigen(util.Base):
ownernames = loads(settings.Nordigen.OwnerNames) ownernames = loads(settings.Nordigen.OwnerNames)
return ownernames return ownernames
@inlineCallbacks
def get_account(self, account_id): def get_account(self, account_id):
""" """
Get details of an account. Get details of an account.
@ -245,10 +277,10 @@ class Nordigen(util.Base):
"Authorization": f"Bearer {self.token}", "Authorization": f"Bearer {self.token}",
} }
path = f"{settings.Nordigen.Base}/accounts/{account_id}/details/" path = f"{settings.Nordigen.Base}/accounts/{account_id}/details/"
r = requests.get(path, headers=headers) d = yield treq.get(path, headers=headers)
print("GET ACCOUNT", account_id, r.content) content = yield d.content()
try: try:
obj = AccountDetails.from_json(r.content) obj = AccountDetails.from_json(content)
except ValidationError as err: except ValidationError as err:
self.log.error(f"Validation error: {err}") self.log.error(f"Validation error: {err}")
return return
@ -283,7 +315,7 @@ class Nordigen(util.Base):
existing_entry = loads(settings.Nordigen.Maps) existing_entry = loads(settings.Nordigen.Maps)
self.banks = existing_entry self.banks = existing_entry
def map_account(self, account_id): def map_account(self, account_id): # TODO: inlineCallbacks?
""" """
Map an account_id at a bank to an account_name. Map an account_id at a bank to an account_name.
This enables the account for fetching. This enables the account for fetching.
@ -323,18 +355,19 @@ class Nordigen(util.Base):
self.banks = existing_entry self.banks = existing_entry
settings.write() settings.write()
@inlineCallbacks
def get_all_account_info(self): def get_all_account_info(self):
to_return = {} to_return = {}
requisitions = self.get_requisitions() requisitions = yield self.get_requisitions()
if not requisitions: if not requisitions:
self.log.error("Could not get requisitions.") self.log.error("Could not get requisitions.")
return {} return {}
for req in requisitions: for req in requisitions:
if not req["accounts"]: if not req["accounts"]:
continue continue
accounts = self.get_accounts(req["id"]) accounts = yield self.get_accounts(req["id"])
for account_id in accounts: for account_id in accounts:
account_info = self.get_account(account_id) account_info = yield self.get_account(account_id)
if not account_info: if not account_info:
continue continue
if req["institution_id"] in to_return: if req["institution_id"] in to_return:
@ -366,6 +399,7 @@ class Nordigen(util.Base):
transaction["reference"] = transaction["remittanceInformationUnstructured"] transaction["reference"] = transaction["remittanceInformationUnstructured"]
del transaction["remittanceInformationUnstructured"] del transaction["remittanceInformationUnstructured"]
@inlineCallbacks
def get_transactions(self, account_id): def get_transactions(self, account_id):
""" """
Get all transactions for an account. Get all transactions for an account.
@ -378,9 +412,10 @@ class Nordigen(util.Base):
"Authorization": f"Bearer {self.token}", "Authorization": f"Bearer {self.token}",
} }
path = f"{settings.Nordigen.Base}/accounts/{account_id}/transactions/" path = f"{settings.Nordigen.Base}/accounts/{account_id}/transactions/"
r = requests.get(path, headers=headers) d = yield treq.get(path, headers=headers)
content = yield d.content()
try: try:
obj = TXRoot.from_json(r.content) obj = TXRoot.from_json(content)
except ValidationError as err: except ValidationError as err:
self.log.error(f"Validation error: {err}") self.log.error(f"Validation error: {err}")
return return

2
handler/sources/__init__.py
View File

@ -17,8 +17,10 @@ class Sources(util.Base):
self.lbtc = sources.localbitcoins.LBTC() self.lbtc = sources.localbitcoins.LBTC()
def __irc_started__(self): def __irc_started__(self):
self.log.debug("IRC hook called.")
self.agora.setup_loop() self.agora.setup_loop()
self.lbtc.setup_loop() self.lbtc.setup_loop()
self.log.debug("Finished setting up loops.")
def __xmerged__(self): def __xmerged__(self):
""" """