Implement viewing transactions for an account

app/urls.py

@@ -116,4 +116,10 @@ urlpatterns = [
         banks.BanksBalances.as_view(),
         name="balances",
     ),
+    # Transactions
+    path(
+        "banks/<str:type>/transactions/<str:aggregator_id>/<str:account_id>/",
+        banks.BanksTransactions.as_view(),
+        name="transactions",
+    ),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

core/__init__.py

@@ -2,7 +2,6 @@ import os
 
 # import stripe
 os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
-# from redis import StrictRedis
 
 # r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
 

core/clients/aggregator.py

@@ -1,5 +1,7 @@
 from abc import ABC
 
+from core.lib.db import convert, r
+
 
 class AggregatorClient(ABC):
     def store_account_info(self, account_infos):
@@ -40,3 +42,28 @@ class AggregatorClient(ABC):
 
         self.instance.currencies = currencies
         self.instance.save()
+
+    async def process_transactions(self, account_id, transactions):
+        if not transactions:
+            return False
+        transaction_ids = [x["transaction_id"] for x in transactions]
+        new_key_name = f"new.transactions.{self.instance.id}.{self.name}.{account_id}"
+        old_key_name = f"transactions.{self.instance.id}.{self.name}.{account_id}"
+        # for transaction_id in transaction_ids:
+        if not transaction_ids:
+            return
+        r.sadd(new_key_name, *transaction_ids)
+
+        difference = list(r.sdiff(new_key_name, old_key_name))
+
+        difference = convert(difference)
+
+        new_transactions = [
+            x for x in transactions if x["transaction_id"] in difference
+        ]
+
+        # Rename the new key to the old key so we can run the diff again
+        r.rename(new_key_name, old_key_name)
+        for transaction in new_transactions:
+            transaction["subclass"] = self.name
+            # self.tx.transaction(transaction)

core/clients/aggregators/nordigen.py

@@ -1,7 +1,9 @@
 from datetime import timedelta
+from hashlib import sha256
 
 from django.conf import settings
 from django.utils import timezone
+from orjson import dumps
 
 from core.clients.aggregator import AggregatorClient
 from core.clients.base import BaseClient
@@ -125,7 +127,7 @@ class NordigenClient(BaseClient, AggregatorClient):
     async def get_account(self, account_id):
         """
         Get details of an account.
-        :param requisition: requisition ID"""
+        :param account_id: account ID"""
 
         path = f"accounts/{account_id}/details"
         response = await self.call(path, schema="AccountDetails")
@@ -145,6 +147,7 @@ class NordigenClient(BaseClient, AggregatorClient):
                 parsed["recipient"] = "TODO"
         # Let's add the account ID so we can reference it later
         parsed["account_id"] = account_id
+        parsed["aggregator_id"] = str(self.instance.id)
         return parsed
 
     async def get_all_account_info(self, requisition=None, store=False):
@@ -243,3 +246,61 @@ class NordigenClient(BaseClient, AggregatorClient):
                 else:
                     totals[currency] = amount
         return totals
+
+    def normalise_transactions(self, transactions):
+        for transaction in transactions:
+            # Rename ID
+            if "transactionId" in transaction:
+                transaction["transaction_id"] = transaction["transactionId"]
+                del transaction["transactionId"]
+            elif "internalTransactionId" in transaction:
+                transaction["transaction_id"] = transaction["internalTransactionId"]
+                del transaction["internalTransactionId"]
+            else:
+                # No transaction ID. This is a problem for our implementation
+
+                tx_hash = sha256(
+                    dumps(transaction, sort_keys=True).encode("utf8")
+                ).hexdigest()
+                transaction["transaction_id"] = tx_hash
+
+            # Rename timestamp
+
+            if "bookingDateTime" in transaction:
+                transaction["ts"] = transaction["bookingDateTime"]
+                del transaction["bookingDateTime"]
+            elif "bookingDate" in transaction:
+                transaction["ts"] = transaction["bookingDate"]
+                del transaction["bookingDate"]
+
+            transaction["amount"] = float(transaction["transactionAmount"]["amount"])
+            transaction["currency"] = transaction["transactionAmount"]["currency"]
+            del transaction["transactionAmount"]
+
+            if transaction["remittanceInformationUnstructuredArray"]:
+                ref_list = transaction["remittanceInformationUnstructuredArray"]
+                reference = "|".join(ref_list)
+                transaction["reference"] = reference
+                del transaction["remittanceInformationUnstructuredArray"]
+            elif transaction["remittanceInformationUnstructured"]:
+                reference = transaction["remittanceInformationUnstructured"]
+                transaction["reference"] = reference
+                del transaction["remittanceInformationUnstructured"]
+            else:
+                raise Exception(f"No way to get reference: {transaction}")
+
+    async def get_transactions(self, account_id, process=False):
+        """
+        Get all transactions for an account.
+        :param account_id: account to fetch transactions for
+        :return: list of transactions
+        :rtype: dict
+        """
+        path = f"accounts/{account_id}/transactions"
+        response = await self.call(path, schema="Transactions")
+
+        parsed = response["booked"]
+        self.normalise_transactions(parsed)
+        if process:
+            await self.process_transactions(parsed)
+        return parsed

core/lib/db.py

@@ -0,0 +1,150 @@
+from redis import asyncio as aioredis
+
+from core.util import logs
+
+log = logs.get_logger("scheduling")
+
+r = aioredis.from_url("redis://redis:6379", db=0)
+
+
+def convert(data):
+    """
+    Recursively convert a dictionary.
+    """
+    if isinstance(data, bytes):
+        return data.decode("ascii")
+    if isinstance(data, dict):
+        return dict(map(convert, data.items()))
+    if isinstance(data, tuple):
+        return map(convert, data)
+    if isinstance(data, list):
+        return list(map(convert, data))
+    return data
+
+
+def get_refs():
+    """
+    Get all reference IDs for trades.
+    :return: list of trade IDs
+    :rtype: list
+    """
+    references = []
+    ref_keys = r.keys("trade.*.reference")
+    for key in ref_keys:
+        references.append(r.get(key))
+    return convert(references)
+
+
+def tx_to_ref(tx):
+    """
+    Convert a trade ID to a reference.
+    :param tx: trade ID
+    :type tx: string
+    :return: reference
+    :rtype: string
+    """
+    refs = get_refs()
+    for reference in refs:
+        ref_data = convert(r.hgetall(f"trade.{reference}"))
+        if not ref_data:
+            continue
+        if ref_data["id"] == tx:
+            return reference
+
+
+def ref_to_tx(reference):
+    """
+    Convert a reference to a trade ID.
+    :param reference: trade reference
+    :type reference: string
+    :return: trade ID
+    :rtype: string
+    """
+    ref_data = convert(r.hgetall(f"trade.{reference}"))
+    if not ref_data:
+        return False
+    return ref_data["id"]
+
+
+def get_ref_map():
+    """
+    Get all reference IDs for trades.
+    :return: dict of references keyed by TXID
+    :rtype: dict
+    """
+    references = {}
+    ref_keys = r.keys("trade.*.reference")
+    for key in ref_keys:
+        tx = convert(key).split(".")[1]
+        references[tx] = r.get(key)
+    return convert(references)
+
+
+def get_ref(reference):
+    """
+    Get the trade information for a reference.
+    :param reference: trade reference
+    :type reference: string
+    :return: dict of trade information
+    :rtype: dict
+    """
+    ref_data = r.hgetall(f"trade.{reference}")
+    ref_data = convert(ref_data)
+    if "subclass" not in ref_data:
+        ref_data["subclass"] = "agora"
+    if not ref_data:
+        return False
+    return ref_data
+
+
+def get_tx(tx):
+    """
+    Get the transaction information for a transaction ID.
+    :param reference: trade reference
+    :type reference: string
+    :return: dict of trade information
+    :rtype: dict
+    """
+    tx_data = r.hgetall(f"tx.{tx}")
+    tx_data = convert(tx_data)
+    if not tx_data:
+        return False
+    return tx_data
+
+
+def get_subclass(reference):
+    obj = r.hget(f"trade.{reference}", "subclass")
+    subclass = convert(obj)
+    return subclass
+
+
+def del_ref(reference):
+    """
+    Delete a given reference from the Redis database.
+    :param reference: trade reference to delete
+    :type reference: string
+    """
+    tx = ref_to_tx(reference)
+    r.delete(f"trade.{reference}")
+    r.delete(f"trade.{tx}.reference")
+
+
+def cleanup(subclass, references):
+    """
+    Reconcile the internal reference database with a given list of references.
+    Delete all internal references not present in the list and clean up artifacts.
+    :param references: list of references to reconcile against
+    :type references: list
+    """
+    messages = []
+    for tx, reference in get_ref_map().items():
+        if reference not in references:
+            if get_subclass(reference) == subclass:
+                logmessage = (
+                    f"[{reference}] ({subclass}): Archiving trade reference. TX: {tx}"
+                )
+                messages.append(logmessage)
+                log.info(logmessage)
+                r.rename(f"trade.{tx}.reference", f"archive.trade.{tx}.reference")
+                r.rename(f"trade.{reference}", f"archive.trade.{reference}")
+    return messages

core/lib/schemas/nordigen_s.py

@@ -146,3 +146,54 @@ AccountBalancesSchema = {
     "balances": "balances",
     "summary": "summary",
 }
+
+
+class TXCurrencyAmount(BaseModel):
+    amount: str
+    currency: str
+
+
+class TransactionsCurrencyExchange(BaseModel):
+    instructedAmount: TXCurrencyAmount
+    sourceCurrency: str
+    exchangeRate: str
+    unitCurrency: str
+    targetCurrency: str
+
+
+class TXAccount(BaseModel):
+    iban: str
+    bban: str | None
+
+
+class TransactionsNested(BaseModel):
+    transactionId: str | None
+    bookingDate: str | None
+    valueDate: str
+    bookingDateTime: str | None
+    valueDateTime: str | None
+    transactionAmount: TXCurrencyAmount
+    creditorName: str | None
+    creditorAccount: TXAccount | None
+    debtorName: str | None
+    debtorAccount: TXAccount | None
+    remittanceInformationUnstructuredArray: list[str] | None
+    remittanceInformationUnstructured: str | None
+    proprietaryBankTransactionCode: str | None
+    internalTransactionId: str | None
+    currencyExchange: TransactionsCurrencyExchange | None
+
+
+class TransactionsBookedPending(BaseModel):
+    booked: list[TransactionsNested]
+    pending: list[TransactionsNested]
+
+
+class Transactions(BaseModel):
+    transactions: TransactionsBookedPending
+
+
+TransactionsSchema = {
+    "booked": "transactions.booked",
+    "pending": "transactions.pending",
+}

core/templates/partials/banks-currencies-list.html

@@ -21,6 +21,7 @@
       <th>details</th>
       <th>payment</th>
       <th>id</th>
+      <th>actions</th>
 
     </thead>
     {% for account in accounts %}
@@ -42,6 +43,17 @@
             </span>
           </a>
         </td>
+        <td>
+          <a href="{% url 'transactions' type='page' account_id=account.account_id aggregator_id=account.aggregator_id %}"><button
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-table-list"></i>
+              </span>
+            </span>
+          </button>
+          </a>
+        </td>
       </tr>
     {% endfor %}
 

core/templates/partials/banks-transactions-list.html

@@ -0,0 +1,57 @@
+{% load cache %}
+{% load cachalot cache %}
+{% get_last_invalidation 'core.Aggregator' as last %}
+{% include 'mixins/partials/notify.html' %}
+{# cache 600 objects_banks_transactions request.user.id object_list type last #}
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>id</th>
+    <th>ts</th>
+    <th>recipient</th>
+    <th>sender</th>
+    <th>amount</th>
+    <th>currency</th>
+    <th>reference</th>
+  </thead>
+  {% for item in object_list %}
+    <tr class="
+      {% if item.proprietaryBankTransactionCode == 'EXCHANGE' %}has-background-grey-light
+      {% elif item.amount < 0 %}has-background-danger-light
+      {% elif item.amount > 0 %}has-background-success-light
+      {% endif %}">
+      <td>
+        <a
+          class="has-text-grey"
+          onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ item.transaction_id }}/');">
+          <span class="icon" data-tooltip="Copy to clipboard">
+            <i class="fa-solid fa-copy" aria-hidden="true"></i>
+          </span>
+        </a>
+      </td>
+      <td>{{ item.ts }}</td>
+      <td>
+        {{ item.creditorName }}
+        {% for item in item.creditorAccount.values %}
+          {{ item|default_if_none:"—" }}
+        {% endfor %}
+      </td>
+      <td>
+        {{ item.debtorName }}
+        {% for item in item.debtorAccount.values %}
+          {{ item|default_if_none:"—" }}
+        {% endfor %}
+      </td>
+      <td>{{ item.amount }}</td>
+      <td>{{ item.currency }}</td>
+      <td>{{ item.reference }}</td>
+    </tr>
+  {% endfor %}
+
+</table>
+{# endcache #}

core/views/banks.py

@@ -92,3 +92,34 @@ class BanksBalances(LoginRequiredMixin, OTPRequiredMixin, ObjectList):
                     account_balances[k].append(item)
 
         return account_balances
+
+
+class BanksTransactions(LoginRequiredMixin, OTPRequiredMixin, ObjectList):
+    """
+    Get bank transactions.
+    """
+
+    list_template = "partials/banks-transactions-list.html"
+    page_title = "Bank Transactions"
+
+    context_object_name_singular = "transaction"
+    context_object_name = "transactions"
+
+    list_url_name = "transactions"
+    list_url_args = ["type", "account_id", "aggregator_id"]
+
+    def get_queryset(self, **kwargs):
+        aggregator_id = self.kwargs.get("aggregator_id")
+        account_id = self.kwargs.get("account_id")
+        try:
+            aggregator = Aggregator.get_by_id(aggregator_id, self.request.user)
+        except Aggregator.DoesNotExist:
+            context = {
+                "message": "Aggregator does not exist",
+                "class": "danger",
+            }
+            return self.render_to_response(context)
+
+        run = synchronize_async_helper(NordigenClient(aggregator))
+        transactions = synchronize_async_helper(run.get_transactions(account_id))
+        return transactions

requirements.txt

@@ -34,6 +34,5 @@ pyOpenSSL
 Klein
 ConfigObject
 aiohttp[speedups]
-aioredis[hiredis]
 elasticsearch[async]
 uvloop