Change Redis parser class

.gitignore

@@ -1,11 +1,165 @@
-*.pyc
+# ---> Python
-*.swp
+# Byte-compiled / optimized / DLL files
 __pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.python_history
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+# lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+stack.env
+.venv
 env/
-env-glibc/
 venv/
+env-glibc/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+.bash_history
+.vscode/
+core/static/admin
+core/static/debug_toolbar
+
 keys/
 handler/settings.ini
 handler/otp.key
 handler/certs/
-.vscode/

Dockerfile

@@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1
+FROM python:3.10
+ARG OPERATION
+
+RUN useradd -d /code xf
+RUN mkdir -p /code
+RUN chown -R xf:xf /code
+
+RUN mkdir -p /conf/static
+RUN chown -R xf:xf /conf
+
+RUN mkdir /venv
+RUN chown xf:xf /venv
+
+USER xf
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY requirements.txt /code/
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r requirements.txt
+
+# CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini
+
+CMD if [ "$OPERATION" = "uwsgi" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi
+
+# CMD . /venv/bin/activate && uvicorn --reload --reload-include *.html --workers 2 --uds /var/run/socks/app.sock app.asgi:application
+# CMD . /venv/bin/activate && gunicorn -b 0.0.0.0:8000 --reload app.asgi:application -k uvicorn.workers.UvicornWorker

Makefile

@@ -0,0 +1,26 @@
+run:
+	docker-compose --env-file=stack.env up -d
+
+build:
+	docker-compose --env-file=stack.env build
+
+stop:
+	docker-compose --env-file=stack.env down
+
+log:
+	docker-compose --env-file=stack.env logs -f
+
+test:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py test $(MODULES) -v 2"
+
+migrate:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py migrate"
+
+makemigrations:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py makemigrations"
+
+auth:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py createsuperuser"
+
+token:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py addstatictoken m"

app/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for app project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+
+application = get_asgi_application()

app/local_settings.py

@@ -0,0 +1,55 @@
+from os import getenv
+
+trues = ("t", "true", "yes", "y", "1")
+
+# URLs
+DOMAIN = getenv("DOMAIN", "example.com")
+URL = getenv("URL", f"https://{DOMAIN}")
+
+# Access control
+ALLOWED_HOSTS = getenv("ALLOWED_HOSTS", f"127.0.0.1,{DOMAIN}").split(",")
+
+# CSRF
+CSRF_TRUSTED_ORIGINS = getenv("CSRF_TRUSTED_ORIGINS", URL).split(",")
+
+# Stripe
+BILLING_ENABLED = getenv("BILLING_ENABLED", "false").lower() in trues
+
+STRIPE_TEST = getenv("STRIPE_TEST", "true").lower() in trues
+STRIPE_API_KEY_TEST = getenv("STRIPE_API_KEY_TEST", "")
+STRIPE_PUBLIC_API_KEY_TEST = getenv("STRIPE_PUBLIC_API_KEY_TEST", "")
+
+STRIPE_API_KEY_PROD = getenv("STRIPE_API_KEY_PROD", "")
+STRIPE_PUBLIC_API_KEY_PROD = getenv("STRIPE_PUBLIC_API_KEY_PROD", "")
+
+STRIPE_ENDPOINT_SECRET = getenv("STRIPE_ENDPOINT_SECRET", "")
+STATIC_ROOT = getenv("STATIC_ROOT", "")
+SECRET_KEY = getenv("SECRET_KEY", "")
+
+STRIPE_ADMIN_COUPON = getenv("STRIPE_ADMIN_COUPON", "")
+
+REGISTRATION_OPEN = getenv("REGISTRATION_OPEN", "false").lower() in trues
+
+NOTIFY_TOPIC = getenv("NOTIFY_TOPIC", "great-pluto")
+
+ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "elastic")
+ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "changeme")
+ELASTICSEARCH_HOST = getenv("ELASTICSEARCH_HOST", "localhost")
+ELASTICSEARCH_TLS = getenv("ELASTICSEARCH_TLS", "false") in trues
+ELASTICSEARCH_INDEX = getenv("ELASTICSEARCH_INDEX", "pluto")
+ELASTICSEARCH_INDEX_ADS = getenv("ELASTICSEARCH_INDEX_ADS", "ads")
+DEBUG = getenv("DEBUG", "false").lower() in trues
+PROFILER = getenv("PROFILER", "false").lower() in trues
+
+DUMMY = getenv("DUMMY", "false").lower() in trues
+
+if DEBUG:
+    import socket  # only if you haven't already imported this
+
+    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
+        "127.0.0.1",
+        "10.0.2.2",
+    ]
+
+SETTINGS_EXPORT = ["BILLING_ENABLED", "URL"]

app/settings.py

@@ -0,0 +1,230 @@
+"""
+Django settings for app project.
+
+Generated by 'django-admin startproject' using Django 4.0.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.0/ref/settings/
+"""
+
+import os
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+# MOVED TO local_settings.py
+
+ALLOWED_HOSTS = []
+
+# Application definition
+
+INSTALLED_APPS = [
+    "core",
+    "django.contrib.admin",
+    # 'core.apps.LibraryAdminConfig', # our custom OTP'ed admin
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "debug_toolbar",
+    "template_profiler_panel",
+    "django_htmx",
+    "crispy_forms",
+    "crispy_bulma",
+    # "django_tables2",
+    # "django_tables2_bulma_template",
+    "django_otp",
+    "django_otp.plugins.otp_totp",
+    # "django_otp.plugins.otp_email",
+    # 'django_otp.plugins.otp_hotp',
+    "django_otp.plugins.otp_static",
+    "two_factor",
+    "two_factor.plugins.phonenumber",
+    # "two_factor.plugins.email",
+    # "two_factor.plugins.yubikey",
+    # "otp_yubikey",
+    "mixins",
+    "cachalot",
+]
+
+
+# Performance optimisations
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": "unix:///var/run/socks/redis.sock",
+        "OPTIONS": {
+            "db": "10",
+            # "parser_class": "django_redis.cache.RedisCache",
+            "pool_class": "redis.BlockingConnectionPool",
+        },
+    }
+}
+# CACHE_MIDDLEWARE_ALIAS = 'default'
+# CACHE_MIDDLEWARE_SECONDS = '600'
+# CACHE_MIDDLEWARE_KEY_PREFIX = ''
+
+CRISPY_TEMPLATE_PACK = "bulma"
+CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
+DJANGO_TABLES2_TEMPLATE = "django-tables2/bulma.html"
+
+MIDDLEWARE = [
+    "debug_toolbar.middleware.DebugToolbarMiddleware",
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    # 'django.middleware.cache.UpdateCacheMiddleware',
+    "django.middleware.common.CommonMiddleware",
+    # 'django.middleware.cache.FetchFromCacheMiddleware',
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django_otp.middleware.OTPMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "django_htmx.middleware.HtmxMiddleware",
+]
+
+ROOT_URLCONF = "app.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [os.path.join(BASE_DIR, "core/templates")],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+                "core.util.django_settings_export.settings_export",
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = "app.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/4.0/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": "/conf/db.sqlite3",
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {"NAME": f"django.contrib.auth.password_validation.{name}"}
+    for name in [
+        "UserAttributeSimilarityValidator",
+        "MinimumLengthValidator",
+        "CommonPasswordValidator",
+        "NumericPasswordValidator",
+    ]
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/4.0/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/4.0/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+AUTH_USER_MODEL = "core.User"
+
+
+LOGOUT_REDIRECT_URL = "home"
+
+LOGIN_REDIRECT_URL = "home"
+# LOGIN_URL = "/accounts/login/"
+
+# 2FA
+LOGIN_URL = "two_factor:login"
+# LOGIN_REDIRECT_URL = 'two_factor:profile'
+
+# ALLOWED_PAYMENT_METHODS = ["bacs_debit", "card"]
+ALLOWED_PAYMENT_METHODS = ["card"]
+
+REST_FRAMEWORK = {
+    "DEFAULT_PARSER_CLASSES": [
+        "rest_framework.parsers.JSONParser",
+    ]
+}
+
+INTERNAL_IPS = [
+    "127.0.0.1",
+    "10.1.10.11",
+]
+
+DEBUG_TOOLBAR_PANELS = [
+    "template_profiler_panel.panels.template.TemplateProfilerPanel",
+    "debug_toolbar.panels.history.HistoryPanel",
+    "debug_toolbar.panels.versions.VersionsPanel",
+    "debug_toolbar.panels.timer.TimerPanel",
+    "debug_toolbar.panels.settings.SettingsPanel",
+    "debug_toolbar.panels.headers.HeadersPanel",
+    "debug_toolbar.panels.request.RequestPanel",
+    "debug_toolbar.panels.sql.SQLPanel",
+    "debug_toolbar.panels.staticfiles.StaticFilesPanel",
+    "debug_toolbar.panels.templates.TemplatesPanel",
+    "debug_toolbar.panels.cache.CachePanel",
+    "debug_toolbar.panels.signals.SignalsPanel",
+    "debug_toolbar.panels.logging.LoggingPanel",
+    "debug_toolbar.panels.redirects.RedirectsPanel",
+    "debug_toolbar.panels.profiling.ProfilingPanel",
+    "cachalot.panels.CachalotPanel",
+]
+
+from app.local_settings import *  # noqa
+
+if PROFILER:  # noqa - trust me its there
+    import pyroscope
+
+    pyroscope.configure(
+        application_name="neptune",
+        server_address="http://pyroscope:4040",
+        auth_token=os.getenv("PYROSCOPE_AUTH_TOKEN", ""),
+        # tags           = {
+        #     "region":   f'{os.getenv("REGION")}',
+        # }
+    )
+
+
+def show_toolbar(request):
+    return DEBUG  # noqa: from local imports
+
+
+DEBUG_TOOLBAR_CONFIG = {
+    "SHOW_TOOLBAR_CALLBACK": show_toolbar,
+}

app/urls.py

@@ -0,0 +1,312 @@
+"""app URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/4.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.conf import settings
+from django.conf.urls.static import static
+from django.contrib import admin
+from django.contrib.auth.views import LogoutView
+from django.urls import include, path
+from two_factor.urls import urlpatterns as tf_urls
+
+from core.views import (
+    ads,
+    aggregators,
+    banks,
+    base,
+    linkgroups,
+    notifications,
+    payouts,
+    platforms,
+    profit,
+    wallets,
+)
+
+# from core.views.stripe_callbacks import Callback
+
+urlpatterns = [
+    path("__debug__/", include("debug_toolbar.urls")),
+    path("", base.Home.as_view(), name="home"),
+    path("sapp/", admin.site.urls),
+    # 2FA login urls
+    path("", include(tf_urls)),
+    path("accounts/signup/", base.Signup.as_view(), name="signup"),
+    path("accounts/logout/", LogoutView.as_view(), name="logout"),
+    # Notifications
+    path(
+        "notifications/<str:type>/update/",
+        notifications.NotificationsUpdate.as_view(),
+        name="notifications_update",
+    ),
+    # Aggregators
+    path(
+        "aggs/<str:type>/",
+        aggregators.AggregatorList.as_view(),
+        name="aggregators",
+    ),
+    path(
+        "aggs/<str:type>/create/",
+        aggregators.AggregatorCreate.as_view(),
+        name="aggregator_create",
+    ),
+    path(
+        "aggs/<str:type>/update/<str:pk>/",
+        aggregators.AggregatorUpdate.as_view(),
+        name="aggregator_update",
+    ),
+    path(
+        "aggs/<str:type>/delete/<str:pk>/",
+        aggregators.AggregatorDelete.as_view(),
+        name="aggregator_delete",
+    ),
+    # Aggregator Requisitions
+    path(
+        "aggs/<str:type>/info/<str:pk>/",
+        aggregators.ReqsList.as_view(),
+        name="reqs",
+    ),
+    # Aggregator Account link flow
+    path(
+        "aggs/<str:type>/countries/<str:pk>/",
+        aggregators.AggregatorCountriesList.as_view(),
+        name="aggregator_countries",
+    ),
+    path(
+        "aggs/<str:type>/countries/<str:pk>/<str:country>/banks/",
+        aggregators.AggregatorCountryBanksList.as_view(),
+        name="aggregator_country_banks",
+    ),
+    path(
+        "aggs/<str:type>/link/<str:pk>/<str:bank>/",
+        aggregators.AggregatorLinkBank.as_view(),
+        name="aggregator_link",
+    ),
+    # Delete requisition
+    path(
+        "aggs/<str:type>/delete/<str:pk>/<str:req_id>/",
+        aggregators.ReqDelete.as_view(),
+        name="req_delete",
+    ),
+    # Requisition info
+    path(
+        "aggs/<str:type>/info/<str:pk>/<str:req_id>/",
+        aggregators.ReqInfo.as_view(),
+        name="req_info",
+    ),
+    # Request bank fetch
+    path(
+        "ops/bank_fetch/<str:pk>/",
+        aggregators.RequestBankFetch.as_view(),
+        name="bank_fetch",
+    ),
+    path(
+        "ops/bank_fetch/",
+        aggregators.RequestBankFetch.as_view(),
+        name="bank_fetch",
+    ),
+    # Bank details by currency
+    path(
+        "banks/<str:type>/details/",
+        banks.BanksCurrencies.as_view(),
+        name="currencies",
+    ),
+    path(
+        "banks/<str:type>/req/<str:aggregator_id>/<str:req_id>/",
+        banks.BanksRequisitionUpdate.as_view(),
+        name="requisition_update",
+    ),
+    path(
+        "banks/<str:type>/req_delete/<str:pk>/",
+        banks.BanksRequisitionDelete.as_view(),
+        name="requisition_delete",
+    ),
+    # Bank balances
+    path(
+        "banks/<str:type>/balances/",
+        banks.BanksBalances.as_view(),
+        name="balances",
+    ),
+    # Transactions
+    path(
+        "banks/<str:type>/transactions/<str:aggregator_id>/<str:account_id>/",
+        banks.BanksTransactions.as_view(),
+        name="transactions",
+    ),
+    # Platforms
+    path(
+        "platforms/<str:type>/",
+        platforms.PlatformList.as_view(),
+        name="platforms",
+    ),
+    path(
+        "platforms/<str:type>/create/",
+        platforms.PlatformCreate.as_view(),
+        name="platform_create",
+    ),
+    path(
+        "platforms/<str:type>/update/<str:pk>/",
+        platforms.PlatformUpdate.as_view(),
+        name="platform_update",
+    ),
+    path(
+        "platforms/<str:type>/delete/<str:pk>/",
+        platforms.PlatformDelete.as_view(),
+        name="platform_delete",
+    ),
+    # Trades
+    path(
+        "trades/<str:type>/",
+        platforms.PlatformTrades.as_view(),
+        name="trades",
+    ),
+    # Ads
+    path(
+        "ads/<str:type>/",
+        ads.AdList.as_view(),
+        name="ads",
+    ),
+    path(
+        "ads/<str:type>/create/",
+        ads.AdCreate.as_view(),
+        name="ad_create",
+    ),
+    path(
+        "ads/<str:type>/update/<str:pk>/",
+        ads.AdUpdate.as_view(),
+        name="ad_update",
+    ),
+    path(
+        "ads/<str:type>/delete/<str:pk>/",
+        ads.AdDelete.as_view(),
+        name="ad_delete",
+    ),
+    path(
+        "ops/ads/dist/",
+        ads.AdDist.as_view(),
+        name="ad_dist",
+    ),
+    path(
+        "ops/ads/nuke/",
+        ads.AdNuke.as_view(),
+        name="ad_nuke",
+    ),
+    path(
+        "ops/ads/redist/",
+        ads.AdRedist.as_view(),
+        name="ad_redist",
+    ),
+    path(
+        "ops/ads/dedup/",
+        ads.AdDedup.as_view(),
+        name="ad_dedup",
+    ),
+    path(
+        "ops/ads/cheat/",
+        ads.Cheat.as_view(),
+        name="cheat",
+    ),
+    path(
+        "profit/<str:type>/",
+        profit.Profit.as_view(),
+        name="profit",
+    ),
+    # Wallets
+    path(
+        "wallets/<str:type>/",
+        wallets.WalletList.as_view(),
+        name="wallets",
+    ),
+    path(
+        "wallets/<str:type>/create/",
+        wallets.WalletCreate.as_view(),
+        name="wallet_create",
+    ),
+    path(
+        "wallets/<str:type>/update/<str:pk>/",
+        wallets.WalletUpdate.as_view(),
+        name="wallet_update",
+    ),
+    path(
+        "operator_wallets/<str:type>/update/",
+        wallets.OperatorWalletsUpdate.as_view(),
+        name="operator_wallets_update",
+    ),
+    path(
+        "wallets/<str:type>/delete/<str:pk>/",
+        wallets.WalletDelete.as_view(),
+        name="wallet_delete",
+    ),
+    # Payouts
+    path(
+        "payouts/<str:type>/",
+        payouts.PayoutList.as_view(),
+        name="payouts",
+    ),
+    path(
+        "payouts/<str:type>/create/",
+        payouts.PayoutCreate.as_view(),
+        name="payout_create",
+    ),
+    path(
+        "payouts/<str:type>/update/<str:pk>/",
+        payouts.PayoutUpdate.as_view(),
+        name="payout_update",
+    ),
+    path(
+        "payouts/<str:type>/delete/<str:pk>/",
+        payouts.PayoutDelete.as_view(),
+        name="payout_delete",
+    ),
+    path(
+        "payouts/action/delete_all/",
+        payouts.PayoutDeleteAll.as_view(),
+        name="payout_delete_all",
+    ),
+    # Link groups
+    path(
+        "links/<str:type>/",
+        linkgroups.LinkGroupList.as_view(),
+        name="linkgroups",
+    ),
+    path(
+        "links/<str:type>/create/",
+        linkgroups.LinkGroupCreate.as_view(),
+        name="linkgroup_create",
+    ),
+    path(
+        "links/<str:type>/update/<str:pk>/",
+        linkgroups.LinkGroupUpdate.as_view(),
+        name="linkgroup_update",
+    ),
+    path(
+        "links/<str:type>/delete/<str:pk>/",
+        linkgroups.LinkGroupDelete.as_view(),
+        name="linkgroup_delete",
+    ),
+    path(
+        "links/<str:type>/info/<str:pk>/",
+        linkgroups.LinkGroupInfo.as_view(),
+        name="linkgroup_info",
+    ),
+    path(
+        "links/<str:type>/simulate/<str:pk>/",
+        linkgroups.LinkGroupSimulation.as_view(),
+        name="linkgroup_simulate",
+    ),
+    path(
+        "links/<str:type>/withdraw/<str:pk>/",
+        linkgroups.LinkGroupWithdraw.as_view(),
+        name="linkgroup_withdraw",
+    ),
+] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

app/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for app project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+
+application = get_wsgi_application()

core/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# import stripe
+os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+
+# r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
+
+# if settings.STRIPE_TEST:
+#     stripe.api_key = settings.STRIPE_API_KEY_TEST
+# else:
+#     stripe.api_key = settings.STRIPE_API_KEY_PROD

core/admin.py

@@ -0,0 +1,33 @@
+from django.contrib import admin
+from django.contrib.auth.admin import UserAdmin
+
+from .forms import CustomUserCreationForm
+from .models import NotificationSettings, User  # AssetRestriction,; Plan,; Session,
+
+# admin.site.__class__ = OTPAdminSite
+
+# otp_admin_site = OTPAdminSite(OTPAdminSite.name)
+# for model_cls, model_admin in admin.site._registry.items():
+#     otp_admin_site.register(model_cls, model_admin.__class__)
+
+
+# Register your models here.
+class CustomUserAdmin(UserAdmin):
+    # list_filter = ["plans"]
+    model = User
+    add_form = CustomUserCreationForm
+    fieldsets = (
+        *UserAdmin.fieldsets,
+        (
+            "Billing information",
+            {"fields": ("billing_provider_id", "payment_provider_id")},
+        ),
+    )
+
+
+class NotificationSettingsAdmin(admin.ModelAdmin):
+    list_display = ("user", "ntfy_topic", "ntfy_url")
+
+
+admin.site.register(User, CustomUserAdmin)
+admin.site.register(NotificationSettings, NotificationSettingsAdmin)

core/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "core"

core/clients/aggregator.py

@@ -0,0 +1,423 @@
+from abc import ABC
+
+import orjson
+
+from core.clients.platforms.agora import AgoraClient
+from core.lib import notify
+from core.lib.money import money
+from core.util import logs
+
+log = logs.get_logger("aggregator")
+
+
+class AggregatorClient(ABC):
+    def store_account_info(self, account_infos):
+        # account_infos = {
+        #     bank: accounts
+        #     for bank, accounts in account_info.items()
+        #     for account in accounts
+        #     #if account["account_id"] in self.banks
+        # }
+        # For each bank
+        for bank, accounts in account_infos.items():
+            # Iterate the accounts
+            for index, account in enumerate(list(accounts)):
+                if account["ownerName"] is None:
+                    requisition = self.instance.get_requisition(
+                        account["requisition_id"]
+                    )
+                    if requisition is not None:
+                        account["ownerName"] = requisition.owner_name
+                if "account_number" not in account:
+                    account_infos[bank][index]["account_number"] = {}
+                    fields = ["sort_code", "number", "iban"]
+                    for field in fields:
+                        if field in account:
+                            account_infos[bank][index]["account_number"][
+                                field
+                            ] = account[field]
+                            del account_infos[bank][index][field]
+                    # if len(account["account_number"]) == 1:
+                    #     account_infos[bank].remove(account)
+        currencies = [
+            account["currency"]
+            for bank, accounts in account_infos.items()
+            for account in accounts
+        ]
+        for bank, accounts in account_infos.items():
+            if not self.instance.account_info:
+                self.instance.account_info = {}
+            self.instance.account_info[bank] = []
+            for account in accounts:
+                self.instance.account_info[bank].append(account)
+        # self.account_info = account_infos
+        self.currencies = currencies
+
+        self.instance.currencies = currencies
+        self.instance.save()
+
+    async def process_transactions(self, account_id, transactions, req):
+        if not transactions:
+            return False
+
+        if not req:
+            return False
+
+        platforms = self.instance.platforms
+        for transaction in transactions:
+            transaction_id = transaction["transaction_id"]
+            tx_obj = self.instance.get_transaction(
+                account_id,
+                transaction_id,
+            )
+            if tx_obj is None:
+                tx_cast = {
+                    "transaction_id": transaction_id,
+                    "recipient": transaction["creditorName"],
+                    "sender": transaction["debtorName"],
+                    "amount": transaction["amount"],
+                    "currency": transaction["currency"],
+                    "note": transaction["reference"],
+                }
+                tx_obj = self.instance.add_transaction(
+                    req,
+                    account_id,
+                    tx_cast,
+                )
+                # New transaction
+                await notify.sendmsg(
+                    self.instance.user,
+                    f"New transaction: {orjson.dumps(tx_cast)}",
+                    title="New transaction",
+                )
+                await self.transaction(platforms, tx_obj)
+            else:
+                # Transaction exists
+                continue
+        # transaction_ids = [x["transaction_id"] for x in transactions]
+        # new_key_name = f"new.transactions.{self.instance.id}.{self.name}.{account_id}"
+        # old_key_name = f"transactions.{self.instance.id}.{self.name}.{account_id}"
+        # # for transaction_id in transaction_ids:
+        # if not transaction_ids:
+        #     return
+        # await db.r.sadd(new_key_name, *transaction_ids)
+
+        # difference = list(await db.r.sdiff(new_key_name, old_key_name))
+
+        # difference = db.convert(difference)
+
+        # new_transactions = [
+        #     x for x in transactions if x["transaction_id"] in difference
+        # ]
+
+        # # Rename the new key to the old key so we can run the diff again
+        # await db.r.rename(new_key_name, old_key_name)
+        # for transaction in new_transactions:
+        #     transaction["subclass"] = self.name
+        #     # self.tx.transaction(transaction)
+
+    def valid_transaction(self, tx_obj):
+        """
+        Determine if a given transaction object is valid.
+        :param data: a transaction cast
+        :type data: dict
+        :return: whether the transaction is valid
+        :rtype: bool
+        """
+        txid = tx_obj.transaction_id
+        if tx_obj.amount is None:
+            return False
+        if tx_obj.currency is None:
+            return False
+        amount = tx_obj.amount
+        if amount <= 0:
+            log.info(f"Ignoring transaction with negative/zero amount: {txid}")
+            return False
+        return True
+
+    # def extract_reference(self, data):
+    #     """
+    #     Extract a reference from the transaction cast.
+    #     :param data: a transaction cast
+    #     :type data: dict
+    #     :return: the extracted reference or not_set
+    #     :rtype: str
+    #     """
+    #     if "reference" in data:
+    #         return data["reference"]
+    #     elif "meta" in data:
+    #         if "provider_reference" in data["meta"]:
+    #             return data["meta"]["provider_reference"]
+    #     return "not_set"
+
+    # def extract_sender(self, data):
+    #     """
+    #     Extract a sender name from the transaction cast.
+    #     :param data: a transaction cast
+    #     :type data: dict
+    #     :return: the sender name or not_set
+    #     :rtype: str
+    #     """
+    #     if "debtorName" in data:
+    #         return data["debtorName"]
+    #     elif "meta" in data:
+    #         if "debtor_account_name" in data["meta"]:
+    #             return data["meta"]["debtor_account_name"]
+    #     elif "  " in data["reference"]:
+    #         refsplit = data["reference"].split("  ")
+    #         if not len(refsplit) == 2:
+    #             log.error(f"Sender cannot be extracted: {data}")
+    #             return "not_set"
+    #         realname, part2 = data["reference"].split("  ")
+    #         return realname
+
+    #     return "not_set"
+
+    async def reference_partial_check(
+        self, platform, reference, txid, currency, amount
+    ):
+        """
+        Perform a partial check by intersecting all parts of the split of the
+        reference against the existing references, and returning a set of the matches.
+        :param reference: the reference to check
+        :type reference: str
+        :return: matching trade ID string
+        :rtype: str
+        """
+        # Partial reference implementation
+        # Account for silly people not removing the default string
+        # Split the reference into parts
+        ref_split = reference.split(" ")
+        # Get all existing references
+        existing_refs = platform.references
+        # Get all parts of the given reference split that match the existing references
+        # stored_trade_reference = set(existing_refs).intersection(set(ref_split))
+        stored_trade_reference = [x for x in existing_refs if x in ref_split]
+        if len(stored_trade_reference) > 1:
+            message = (
+                f"Multiple references valid for TXID {txid}: {reference}"
+                f"Currency: {currency} | Amount: {amount}"
+            )
+            title = "Error: multiple references valid"
+            await notify.sendmsg(self.instance.user, message, title=title)
+            return False
+        if len(stored_trade_reference) == 0:
+            return None
+        return stored_trade_reference.pop()
+
+    # TODO: pass platform here
+    async def can_alt_lookup(self, platform, amount, currency, reference):
+        amount_usd = await money.to_usd(amount, currency)
+        # Amount is reliable here as it is checked by find_trade,
+        # so no need for stored_trade["amount"]
+        if amount_usd > platform.no_reference_amount_check_max_usd:
+            message = (
+                f"Amount exceeds max for {reference}"
+                f"Currency: {currency} | Amount: {amount}"
+            )
+            title = "Amount exceeds max for {reference}"
+            await notify.sendmsg(self.instance.user, message, title=title)
+            return False
+        return True
+
+    def find_trade(self, platform, txid, currency, amount):
+        """
+        Get a trade reference that matches the given currency and amount.
+        Only works if there is one result.
+        :param txid: Sink transaction ID
+        :param currency: currency
+        :param amount: amount
+        :type txid: string
+        :type currency: string
+        :type amount: int
+        :return: matching trade object or False
+        :rtype: dict or bool
+        """
+        refs = platform.references
+        matching_refs = []
+        # TODO: use get_ref_map in this function instead of calling get_ref multiple
+        # times
+        for ref in refs:
+            stored_trade = platform.get_trade_by_reference(ref)
+            if stored_trade.currency == currency and stored_trade.amount_fiat == amount:
+                matching_refs.append(stored_trade)
+        if len(matching_refs) != 1:
+            log.error(
+                f"Find trade returned multiple results for TXID {txid}: {matching_refs}"
+            )
+            return False
+        return matching_refs[0]
+
+    async def amount_currency_lookup(self, platform, amount, currency, txid, ref):
+        title = f"Checking against amount and currency for TXID {txid}"
+        message = (
+            f"Checking against amount and currency for TXID {txid}"
+            f"Currency: {currency} | Amount: {amount}"
+        )
+        await notify.sendmsg(self.instance.user, message, title=title)
+
+        if not await self.can_alt_lookup(platform, amount, currency, ref):
+            return False
+        stored_trade = self.find_trade(platform, txid, currency, amount)
+        if not stored_trade:
+            title = f"Failed to get reference by amount and currency: {txid}"
+            message = (
+                f"Failed to get reference by amount and currency: {txid}"
+                f"Currency: {currency} | Amount: {amount}"
+            )
+            await notify.sendmsg(self.instance.user, message, title=title)
+            return None
+        return stored_trade
+
+    async def normal_lookup(
+        self, platform, stored_trade_reference, reference, currency, amount
+    ):
+        stored_trade = platform.get_trade_by_reference(stored_trade_reference)
+        if not stored_trade:
+            title = f"No reference in DB for {reference}"
+            message = (
+                f"No reference in DB for {reference}"
+                f"Currency: {currency} | Amount: {amount}"
+            )
+            await notify.sendmsg(self.instance.user, message, title=title)
+            return False
+        # stored_trade["amount"] = float(stored_trade["amount"])  # convert to float
+        return stored_trade
+
+    async def currency_check(self, currency, stored_trade):
+        if not stored_trade.currency == currency:
+            title = "Currency mismatch"
+            message = (
+                f"Currency mismatch, Agora: {stored_trade.currency} "
+                f"/ Sink: {currency}"
+            )
+            await notify.sendmsg(self.instance.user, message, title=title)
+            return False
+        return True
+
+    async def alt_amount_check(self, platform, amount, currency, stored_trade):
+        # If the amount does not match exactly, get the min and max values for our
+        # given acceptable margins for trades
+        min_amount, max_amount = await money.get_acceptable_margins(
+            platform, currency, stored_trade.amount_fiat
+        )
+        log.info(
+            (
+                f"Amount does not match exactly, trying with margins: min: {min_amount}"
+                f" / max: {max_amount}"
+            )
+        )
+        title = "Amount does not match exactly"
+        message = (
+            f"Amount does not match exactly, trying with margins: min: "
+            f"{min_amount} / max: {max_amount}"
+        )
+        await notify.sendmsg(self.instance.user, message, title=title)
+        if not min_amount < amount < max_amount:
+            title = "Amount mismatch - not in margins"
+            message = (
+                f"Amount mismatch - not in margins: {stored_trade.amount_fiat} "
+                f"(min: {min_amount} / max: {max_amount}"
+            )
+            await notify.sendmsg(self.instance.user, message, title=title)
+            return False
+        return True
+
+    async def transaction(self, platforms, tx_obj):
+        """
+        Store details of transaction and post notifications to IRC.
+        Matches it up with data stored in Redis to attempt to reconcile with an Agora
+        trade.
+        :param data: details of transaction
+        :type data: dict
+        """
+        valid = self.valid_transaction(tx_obj)
+        if not valid:
+            return False
+        txid = tx_obj.transaction_id
+        amount = tx_obj.amount
+        currency = tx_obj.currency
+
+        reference = tx_obj.note
+
+        # reference = self.extract_reference(data)
+        # sender = tx_obj.sender
+
+        log.info(f"Transaction processed: {tx_obj}")
+        await notify.sendmsg(
+            self.instance.user,
+            (f"Transaction: {txid} {amount}{currency}: {reference}"),
+            title="Incoming transaction",
+        )
+        for platform in platforms:
+            stored_trade_reference = await self.reference_partial_check(
+                platform, reference, txid, currency, amount
+            )
+            if stored_trade_reference is False:  # can be None though
+                continue
+
+            stored_trade = False
+            looked_up_without_reference = False
+
+            # Normal implementation for when we have a reference
+            if stored_trade_reference:
+                stored_trade = await self.normal_lookup(
+                    platform, stored_trade_reference, reference, currency, amount
+                )
+                # if not stored_trade:
+                #    return
+
+            # Amount/currency lookup implementation for when we have no reference
+            else:
+                if not stored_trade:  # check we don't overwrite the lookup above
+                    stored_trade = await self.amount_currency_lookup(
+                        platform, amount, currency, txid, reference
+                    )
+                    if stored_trade is False:
+                        continue
+                    if stored_trade:
+                        # Note that we have looked it up without reference so we don't
+                        # use +- below
+                        # This might be redundant given the checks in find_trade,
+                        # but better safe than sorry!
+                        looked_up_without_reference = True
+                    else:
+                        continue
+                else:
+                    # Stored trade reference is none, the checks below will do nothing
+                    continue
+
+            # Make sure it was sent in the expected currency
+            if not await self.currency_check(currency, stored_trade):
+                continue
+
+            # Make sure the expected amount was sent
+            if not stored_trade.amount_fiat == amount:
+                if looked_up_without_reference:
+                    continue
+                if not await self.alt_amount_check(
+                    platform, amount, currency, stored_trade
+                ):
+                    continue
+            # platform_buyer = stored_trade["buyer"]
+
+            # Check sender - we don't do anything with this yet
+            # sender_valid = antifraud.check_valid_sender(
+            #     reference, platform, sender, platform_buyer
+            # )
+            # log.info(f"Trade {reference} buyer {platform_buyer}
+            # valid: {sender_valid}")
+            instance = await AgoraClient(platform)
+            rtrn = await instance.release_map_trade(stored_trade, tx_obj)
+            # if trade_released:
+            #    self.ux.notify.notify_complete_trade(amount, currency)
+            # else:
+            #    log.error(f"Cannot release trade {reference}.")
+            #    return
+
+            # rtrn = await platform.release_funds(stored_trade["id"],
+            # stored_trade["reference"])
+            if rtrn:
+                title = "Trade complete"
+                message = f"Trade complete: {amount}{currency}"
+                await notify.sendmsg(self.instance.user, message, title=title)

core/clients/aggregators/nordigen.py

@@ -0,0 +1,336 @@
+from datetime import timedelta
+from hashlib import sha256
+
+import orjson
+from django.conf import settings
+from django.utils import timezone
+
+from core.clients.aggregator import AggregatorClient
+from core.clients.base import BaseClient
+from core.util import logs
+
+log = logs.get_logger("nordigen")
+
+
+class NordigenClient(BaseClient, AggregatorClient):
+    url = "https://ob.nordigen.com/api/v2"
+
+    async def connect(self):
+        now = timezone.now()
+        # Check if access token expires later than now
+        if self.instance.access_token_expires is not None:
+            if self.instance.access_token_expires > now:
+                self.token = self.instance.access_token
+                return
+        await self.get_access_token()
+
+    def method_filter(self, method):
+        new_method = method.replace("/", "_")
+        return new_method
+
+    async def get_access_token(self):
+        """
+        Get the access token for the Nordigen API.
+        """
+        log.debug(f"Getting new access token for {self.instance}")
+        data = {
+            "secret_id": self.instance.secret_id,
+            "secret_key": self.instance.secret_key,
+        }
+
+        response = await self.call("token/new", http_method="post", data=data)
+        access = response["access"]
+        access_expires = response["access_expires"]
+        now = timezone.now()
+        # Offset now by access_expires seconds
+        access_expires = now + timedelta(seconds=access_expires)
+        self.instance.access_token = access
+        self.instance.access_token_expires = access_expires
+        self.instance.save()
+
+        self.token = access
+
+    async def get_requisitions(self):
+        """
+        Get a list of active accounts.
+        """
+        response = await self.call("requisitions")
+        return response["results"]
+
+    async def get_countries(self):
+        """
+        Get a list of countries.
+        """
+        # This function is a stub.
+
+        return ["GB", "SE", "BG"]
+
+    async def get_banks(self, country):
+        """
+        Get a list of supported banks for a country.
+        :param country: country to query
+        :return: list of institutions
+        :rtype: list
+        """
+        if not len(country) == 2:
+            return False
+        path = f"institutions/?country={country}"
+        response = await self.call(path, schema="Institutions", append_slash=False)
+
+        return response
+
+    async def build_link(self, institution_id, redirect=None):
+        """Create a link to access an institution.
+        :param institution_id: ID of the institution
+        """
+
+        data = {
+            "institution_id": institution_id,
+            "redirect": settings.URL,
+        }
+        if redirect:
+            data["redirect"] = redirect
+        response = await self.call(
+            "requisitions", schema="RequisitionsPost", http_method="post", data=data
+        )
+        if "link" in response:
+            return response["link"]
+        return False
+
+    async def delete_requisition(self, requisition_id):
+        """
+        Delete a requisision ID.
+        """
+        path = f"requisitions/{requisition_id}"
+
+        response = await self.call(
+            path, schema="RequisitionDelete", http_method="delete"
+        )
+        return response
+
+    async def get_requisition(self, requisition):
+        """
+        Get a list of accounts for a requisition.
+        :param requisition: requisition ID"""
+        path = f"requisitions/{requisition}"
+        response = await self.call(path, schema="Requisition")
+
+        return response
+
+    # def get_ownernames(self):
+    #     """
+    #     Get list of supplementary owner names.
+    #     """
+    #     ownernames = loads(settings.Nordigen.OwnerNames)
+    #     return ownernames
+
+    async def get_account(self, req_id, account_id):
+        """
+        Get details of an account.
+        :param account_id: account ID"""
+
+        path = f"accounts/{account_id}/details"
+        response = await self.call(path, schema="AccountDetails")
+        if "account" not in response:
+            return False
+        parsed = response["account"]
+        if "iban" in parsed and parsed["currency"] == "GBP":
+            if parsed["iban"]:
+                sort_code = parsed["iban"][-14:-8]
+                account_number = parsed["iban"][-8:]
+                del parsed["iban"]
+                # if "iban" in parsed:
+                #     del parsed["iban"]
+                sort_code = "-".join(list(map("".join, zip(*[iter(sort_code)] * 2))))
+                parsed["sort_code"] = sort_code
+                parsed["number"] = account_number
+        # Let's add the account ID so we can reference it later
+        parsed["account_id"] = account_id
+        parsed["aggregator_id"] = str(self.instance.id)
+        parsed["requisition_id"] = str(req_id)
+        return parsed
+
+    async def get_all_account_info(self, requisition=None, store=False):
+        to_return = {}
+        if not requisition:
+            requisitions = await self.get_requisitions()
+        else:
+            requisitions = [await self.get_requisition(requisition)]
+
+        for req in requisitions:
+            accounts = req["accounts"]
+            for account_id in accounts:
+                account_info = await self.get_account(req["id"], account_id)
+                if not account_info:
+                    continue
+                if req["institution_id"] in to_return:
+                    to_return[req["institution_id"]].append(account_info)
+                else:
+                    to_return[req["institution_id"]] = [account_info]
+
+        if store:
+            if requisition is not None:
+                raise Exception("Cannot store partial data")
+            self.store_account_info(to_return)
+        return to_return
+
+    async def get_balance(self, account_id):
+        """
+        Get the balance and currency of an account.
+        :param account_id: the account ID
+        :return: tuple of (currency, amount)
+        :rtype: tuple
+        """
+
+        path = f"accounts/{account_id}/balances"
+        response = await self.call(path, schema="AccountBalances")
+
+        total = 0
+        currency = None
+        if "balances" not in response:
+            return (False, False)
+        for entry in response["balances"]:
+            if currency:
+                if not currency == entry["balanceAmount"]["currency"]:
+                    return (False, False)
+                if not entry["balanceType"] == "interimAvailable":
+                    continue
+            total += float(entry["balanceAmount"]["amount"])
+            currency = entry["balanceAmount"]["currency"]
+        return (currency, total)
+
+    async def get_all_balances(self):
+        """
+        Get all balances.
+        Keyed by bank.
+        """
+        if self.instance.account_info is None:
+            await self.get_all_account_info(store=True)
+
+        account_balances = {}
+        for bank, accounts in self.instance.account_info.items():
+            if bank not in account_balances:
+                account_balances[bank] = []
+            for account in accounts:
+                account_id = account["account_id"]
+                currency, amount = await self.get_balance(account_id)
+                account_balances[bank].append(
+                    {
+                        "currency": currency,
+                        "balance": amount,
+                        "account_id": account_id,
+                    }
+                )
+        return account_balances
+
+    async def get_total_map(self):
+        """
+        Return a dictionary keyed by currencies with the amounts as values.
+        :return: dict keyed by currency, values are amounts
+        :rtype: dict
+        """
+        if self.instance.account_info is None:
+            await self.get_all_account_info(store=True)
+
+        totals = {}
+        for bank, accounts in self.instance.account_info.items():
+            for account in accounts:
+                account_id = account["account_id"]
+                currency, amount = await self.get_balance(account_id)
+                if not amount:
+                    continue
+                if not currency:
+                    continue
+                if currency in totals:
+                    totals[currency] += amount
+                else:
+                    totals[currency] = amount
+        return totals
+
+    def normalise_transactions(self, transactions, state=None):
+        for transaction in transactions:
+            # Rename ID
+            if transaction["transactionId"]:
+                transaction["transaction_id"] = transaction["transactionId"]
+                del transaction["transactionId"]
+            elif transaction["internalTransactionId"]:
+                transaction["transaction_id"] = transaction["internalTransactionId"]
+                del transaction["internalTransactionId"]
+            else:
+                # No transaction ID. This is a problem for our implementation
+                tx_hash = sha256(
+                    orjson.dumps(transaction, option=orjson.OPT_SORT_KEYS)
+                ).hexdigest()
+                transaction["transaction_id"] = tx_hash
+
+            # Rename timestamp
+
+            if transaction["bookingDateTime"]:
+                transaction["ts"] = transaction["bookingDateTime"]
+                del transaction["bookingDateTime"]
+            elif transaction["bookingDate"]:
+                transaction["ts"] = transaction["bookingDate"]
+                del transaction["bookingDate"]
+            elif transaction["valueDate"]:
+                transaction["ts"] = transaction["valueDate"]
+                del transaction["valueDate"]
+
+            transaction["amount"] = float(transaction["transactionAmount"]["amount"])
+            transaction["currency"] = transaction["transactionAmount"]["currency"]
+            if state:
+                transaction["state"] = state
+            del transaction["transactionAmount"]
+
+            if transaction["remittanceInformationUnstructuredArray"]:
+                ref_list = transaction["remittanceInformationUnstructuredArray"]
+                reference = "|".join(ref_list)
+                transaction["reference"] = reference
+                del transaction["remittanceInformationUnstructuredArray"]
+            elif transaction["remittanceInformationUnstructured"]:
+                reference = transaction["remittanceInformationUnstructured"]
+                transaction["reference"] = reference
+                del transaction["remittanceInformationUnstructured"]
+            else:
+                raise Exception(f"No way to get reference: {transaction}")
+
+    async def get_transactions(
+        self, account_id, req=None, process=False, pending=False
+    ):
+        """
+        Get all transactions for an account.
+        :param account_id: account to fetch transactions for
+        :return: list of transactions
+        :rtype: dict
+        """
+        path = f"accounts/{account_id}/transactions"
+        response = await self.call(path, schema="Transactions")
+
+        if response["status_code"] == 401:
+            log.error(
+                f"Error getting transactions for {account_id}: {response['summary']}"
+            )
+            return []
+
+        source = "booked"
+
+        # If requisition is specified, try to get the object
+        # If present, take the transaction source from there,
+        # pending or booked.
+        if req:
+            requisition = self.instance.get_requisition(req)
+            if requisition:
+                source = requisition.transaction_source
+
+        parsed = response["transactions"][source]
+        self.normalise_transactions(parsed, state=source)
+
+        if process:
+            await self.process_transactions(account_id, parsed, req=req)
+        if pending:
+            if process:
+                raise Exception("Cannot process and get pending")
+            parsed_pending = response["transactions"]["pending"]
+            self.normalise_transactions(parsed_pending, state="pending")
+            parsed_pending.extend(parsed)
+            parsed = parsed_pending
+        return parsed

core/clients/base.py

@@ -0,0 +1,205 @@
+from abc import ABC, abstractmethod
+
+import aiohttp
+import orjson
+from glom import glom
+from pydantic.error_wrappers import ValidationError
+
+from core.lib import schemas
+from core.util import logs
+
+# Return error if the schema for the message type is not found
+STRICT_VALIDATION = False
+
+# Raise exception if the conversion schema is not found
+STRICT_CONVERSION = False
+
+# TODO: Set them to True when all message types are implemented
+
+log = logs.get_logger("clients")
+
+
+class NoSchema(Exception):
+    """
+    Raised when:
+    - The schema for the message type is not found
+    - The conversion schema is not found
+    - There is no schema library for the client
+    """
+
+    pass
+
+
+class NoSuchMethod(Exception):
+    """
+    Client library has no such method.
+    """
+
+    pass
+
+
+class GenericAPIError(Exception):
+    """
+    Generic API error.
+    """
+
+    pass
+
+
+def is_camel_case(s):
+    return s != s.lower() and s != s.upper() and "_" not in s
+
+
+def snake_to_camel(word):
+    if is_camel_case(word):
+        return word
+    return "".join(x.capitalize() or "_" for x in word.split("_"))
+
+
+DEFAULT_HEADERS = {
+    "accept": "application/json",
+    "Content-Type": "application/json",
+}
+
+
+class BaseClient(ABC):
+    token = None
+
+    async def __new__(cls, *a, **kw):
+        instance = super().__new__(cls)
+        await instance.__init__(*a, **kw)
+        return instance
+
+    async def __init__(self, instance):
+        """
+        Initialise the client.
+        :param instance: the database object, e.g. Aggregator
+        """
+        name = self.__class__.__name__
+        self.name = name.replace("Client", "").lower()
+        self.instance = instance
+        self.client = None
+
+        await self.connect()
+
+    @abstractmethod
+    async def connect(self):
+        pass
+
+    @property
+    def schema(self):
+        """
+        Get the schema library for the client.
+        """
+        # Does the schemas library have a library for this client name?
+        if hasattr(schemas, f"{self.name}_s"):
+            schema_instance = getattr(schemas, f"{self.name}_s")
+        else:
+            log.error(f"No schema library for {self.name}")
+            raise Exception(f"No schema library for client {self.name}")
+
+        return schema_instance
+
+    def get_schema(self, method, convert=False):
+        if isinstance(method, str):
+            to_camel = snake_to_camel(method)
+        else:
+            to_camel = snake_to_camel(method.__class__.__name__)
+        if convert:
+            to_camel = f"{to_camel}Schema"
+
+        # if hasattr(self.schema, method):
+        #     schema = getattr(self.schema, method)
+        if hasattr(self.schema, to_camel):
+            schema = getattr(self.schema, to_camel)
+        else:
+            raise NoSchema(f"Could not get schema: {to_camel}")
+        return schema
+
+    async def call_method(self, method, *args, **kwargs):
+        """
+        Call a method with aiohttp.
+        """
+        if kwargs.get("append_slash", True):
+            path = f"{self.url}/{method}/"
+        else:
+            path = f"{self.url}/{method}"
+
+        http_method = kwargs.get("http_method", "get")
+
+        cast = {
+            "headers": DEFAULT_HEADERS,
+        }
+
+        # Use the token if it's set
+        if self.token is not None:
+            cast["headers"]["Authorization"] = f"Bearer {self.token}"
+
+        if "data" in kwargs:
+            cast["data"] = orjson.dumps(kwargs["data"])
+
+        # Use the method to send a HTTP request
+        async with aiohttp.ClientSession() as session:
+            session_method = getattr(session, http_method)
+            async with session_method(path, **cast) as response:
+                response_json = await response.json()
+                return response_json
+
+    def convert_spec(self, response, method):
+        """
+        Convert an API response to the requested spec.
+        :raises NoSchema: If the conversion schema is not found
+        """
+        schema = self.get_schema(method, convert=True)
+
+        # Use glom to convert the response to the schema
+        converted = glom(response, schema)
+        return converted
+
+    def validate_response(self, response, method):
+        schema = self.get_schema(method)
+        # Return a dict of the validated response
+        try:
+            response_valid = schema(**response).dict()
+        except ValidationError as e:
+            log.error(f"Error validating {method} response: {response}")
+            log.error(f"Errors: {e}")
+            raise GenericAPIError("Error validating response")
+        return response_valid
+
+    def method_filter(self, method):
+        """
+        Return a new method.
+        """
+        return method
+
+    async def call(self, method, *args, **kwargs):
+        """
+        Call the exchange API and validate the response
+        :raises NoSchema: If the method is not in the schema mapping
+        :raises ValidationError: If the response cannot be validated
+        """
+        # try:
+        response = await self.call_method(method, *args, **kwargs)
+        # except (APIError, V20Error) as e:
+        #    log.error(f"Error calling method {method}: {e}")
+        #    raise GenericAPIError(e)
+
+        if "schema" in kwargs:
+            method = kwargs["schema"]
+        else:
+            method = self.method_filter(method)
+        try:
+            response_valid = self.validate_response(response, method)
+        except NoSchema as e:
+            log.error(f"{e} - {response}")
+            response_valid = response
+        # Convert the response to a format that we can use
+        try:
+            response_converted = self.convert_spec(response_valid, method)
+        except NoSchema as e:
+            log.error(f"{e} - {response}")
+            response_converted = response_valid
+
+        # return (True, response_converted)
+        return response_converted

core/clients/platforms/agora.py

@@ -1,38 +1,19 @@
-# Twisted/Klein imports
-import sources.local
-
 # Other library imports
 from pyotp import TOTP
 
-# Project imports
+from core.clients.base import BaseClient
-from settings import settings
+from core.clients.platform import LocalPlatformClient
-from twisted.internet.defer import inlineCallbacks
+from core.util import logs
+
+log = logs.get_logger("agora")
 
 
-class Agora(sources.local.Local):
+class AgoraClient(LocalPlatformClient, BaseClient):
     """
     AgoraDesk API handler.
     """
 
-    def __init__(self):
+    async def release_funds(self, contact_id):
-        """
-        Initialise the AgoraDesk API.
-        Initialise the last_dash storage for detecting new trades.
-        """
-        self.platform = "agora"
-        super().__init__()
-
-        # Cache for detecting new trades
-        self.last_dash = set()
-
-        # Cache for detecting new messages
-        self.last_messages = {}
-
-        # Assets that cheat has been run on
-        self.cheat_run_on = []
-
-    @inlineCallbacks
-    def release_funds(self, contact_id):
         """
         Release funds for a contact_id.
         :param contact_id: trade/contact ID
@@ -41,31 +22,25 @@ class Agora(sources.local.Local):
         :rtype: dict
         """
         print("CALLING RELEASE FUNDS", contact_id)
-        if self.sets.Dummy == "1":
+        if self.instance.dummy:
-            self.log.error(
+            log.error(f"Running in dummy mode, not releasing funds for {contact_id}")
-                f"Running in dummy mode, not releasing funds for {contact_id}"
+            return {"message": "OK"}  # Pretend to succeed
-            )
-            return
-        payload = {"tradeId": contact_id, "password": self.sets.Pass}
-        rtrn = yield self.api._api_call(
-            api_method=f"contact_release/{contact_id}",
-            http_method="POST",
-            query_values=payload,
-        )
 
-        # Check if we can withdraw funds
+        rtrn = await self.api.contact_release(
-        yield self.withdraw_funds()
+            contact_id,
+            self.instance.password,
+        )
 
         return rtrn
 
     # TODO: write test before re-enabling adding total_trades
-    @inlineCallbacks
+    async def withdraw_funds(self, checks):
-    def withdraw_funds(self):
         """
         Withdraw excess funds to our XMR wallets.
         """
         print("CALLING WITHDRAW FUNDS")
-        totals_all = yield self.money.get_total()
+        # checks = self.check_all()
+        totals_all = await self.money.get_total()
         if totals_all is False:
             return False
 
@@ -79,7 +54,7 @@ class Agora(sources.local.Local):
             return False
         # total_usd += total_trades_usd
 
-        profit_usd = total_usd - float(settings.Money.BaseUSD)
+        profit_usd = total_usd - self.instance.base_usd
         # Get the XMR -> USD exchange rate
         xmr_usd = self.money.cg.get_price(ids="monero", vs_currencies=["USD"])
 
@@ -93,15 +68,21 @@ class Agora(sources.local.Local):
         if not float(wallet_xmr) > profit_usd_in_xmr:
             # Not enough funds to withdraw
             self.log.error(
-                f"Not enough funds to withdraw {profit_usd_in_xmr}, as wallet only contains {wallet_xmr}"
+                (
+                    f"Not enough funds to withdraw {profit_usd_in_xmr}, "
+                    f"as wallet only contains {wallet_xmr}"
+                )
             )
             self.irc.sendmsg(
-                f"Not enough funds to withdraw {profit_usd_in_xmr}, as wallet only contains {wallet_xmr}"
+                (
+                    f"Not enough funds to withdraw {profit_usd_in_xmr}, "
+                    f"as wallet only contains {wallet_xmr}"
+                )
             )
             self.ux.notify.notify_need_topup(profit_usd_in_xmr)
             return
 
-        if not profit_usd >= float(settings.Money.WithdrawLimit):
+        if not profit_usd >= self.instance.withdrawal_trigger:
             # Not enough profit to withdraw
             return
 
@@ -122,15 +103,19 @@ class Agora(sources.local.Local):
         send_cast = {
             "address": None,
             "amount": half_rounded,
-            "password": settings.Agora.Pass,
+            "password": self.instance.password,
             "otp": otp_code.now(),
         }
+        print("SENDING", send_cast)
 
-        send_cast["address"] = settings.XMR.Wallet1
+        return  # TODO
-        rtrn1 = yield self.api.wallet_send_xmr(**send_cast)
+        # send_cast["address"] = settings.XMR.Wallet1
+        # rtrn1 = await self.api.wallet_send_xmr(**send_cast)
 
-        send_cast["address"] = settings.XMR.Wallet2
+        # send_cast["address"] = settings.XMR.Wallet2
-        rtrn2 = yield self.api.wallet_send_xmr(**send_cast)
+        # rtrn2 = await self.api.wallet_send_xmr(**send_cast)
 
-        self.irc.sendmsg(f"Withdrawal: {rtrn1['success']} | {rtrn2['success']}")
+        # self.irc.sendmsg(f"Withdrawal: {rtrn1['success']} | {rtrn2['success']}")
-        self.ux.notify.notify_withdrawal(half_rounded)
+        # self.ux.notify.notify_withdrawal(half_rounded)
+
+        # await self.successful_withdrawal()

core/clients/platforms/api/agoradesk.py

@@ -1,16 +1,14 @@
 """See https://agoradesk.com/api-docs/v1."""
 # pylint: disable=too-many-lines
 # Large API. Lots of lines can't be avoided.
-import json
 import logging
 from typing import Any, Dict, List, Optional, Union
 
+import aiohttp
 import arrow
-import treq
+import orjson
 
-# Project imports
+from core.util import logs
-import util
-from twisted.internet.defer import inlineCallbacks
 
 __author__ = "marvin8"
 __copyright__ = "(C) 2021 https://codeberg.org/MarvinsCryptoTools/agoradesk_py"
@@ -23,7 +21,7 @@ logging.basicConfig(
 )
 logging.getLogger("requests.packages.urllib3").setLevel(logging.INFO)
 logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO)
-logger = util.get_logger(__name__)
+logger = logs.get_logger(__name__)
 
 URI_API = "https://agoradesk.com/api/v1/"
 
@@ -51,30 +49,7 @@ class AgoraDesk:
 
         logger.debug("creating instance of AgoraDesk API with api_key %s", self.api_key)
 
-    @inlineCallbacks
+    async def _api_call(
-    def callback_api_call(self, response, result):
-        logger.debug(response)
-        try:
-            text = yield response.content()
-        except:  # noqa
-            self.log.error("Error with API call")
-            return
-        try:
-            result["response"] = json.loads(text)
-        except json.decoder.JSONDecodeError:
-            result["success"] = "ERROR"
-            result["message"] = "Error parsing JSON."
-            return result
-        result["status"] = response.code
-        if response.code == 200:
-            result["success"] = True
-            result["message"] = "OK"
-        else:
-            result["message"] = "API ERROR"
-
-        return result
-
-    def _api_call(
         self,
         api_method: str,
         http_method: Optional[str] = "GET",
@@ -88,12 +63,15 @@ class AgoraDesk:
             f"https://codeberg.org/MarvinsCryptoTools/agoradesk_py",
             "Authorization": self.api_key,
         }
+        cast = {
+            "headers": headers,
+        }
 
         logger.debug("API Call URL: %s", api_call_url)
         logger.debug("Headers     : %s", headers)
         logger.debug("HTTP Method : %s", http_method)
         logger.debug("Query Values: %s", query_values)
-        logger.debug("Query Values as Json:\n%s", json.dumps(query_values))
+        logger.debug("Query Values as Json:\n%s", orjson.dumps(query_values))
 
         result: Dict[str, Any] = {
             "success": False,
@@ -105,152 +83,133 @@ class AgoraDesk:
         response = None
         if http_method == "POST":
             if query_values:
-                # response = httpx.post(
+                cast["data"] = orjson.dumps(query_values)
-                #    url=api_call_url,
+            async with aiohttp.ClientSession() as session:
-                #    headers=headers,
+                async with session.post(api_call_url, **cast) as response_raw:
-                #    content=json.dumps(query_values),
+                    response = await response_raw.json()
-                # )
+                    status_code = response_raw.status
-                response = treq.post(
-                    api_call_url,
-                    headers=headers,
-                    data=json.dumps(query_values).encode("ascii"),
-                )
         else:
-                # response = httpx.post(
+            cast["params"] = query_values
-                #    url=api_call_url,
+            async with aiohttp.ClientSession() as session:
-                #    headers=headers,
+                async with session.get(api_call_url, **cast) as response_raw:
-                # )
+                    response = await response_raw.json()
-                response = treq.post(
+                    status_code = response_raw.status
-                    api_call_url,
-                    headers=headers,
-                )
-
-        else:
-            # response = httpx.get(url=api_call_url, headers=headers, params=query_values)
-            response = treq.get(api_call_url, headers=headers, params=query_values)
         if response:
-            response.addCallback(self.callback_api_call, result)
+            logger.debug(response)
-        return response
+            result["status"] = status_code
+            if status_code == 200:
+                result["success"] = True
+                result["message"] = "OK"
+                result["response"] = response
+            else:
+                if "error" in response:
+                    result["error"] = response["error"]
+                result["message"] = "API ERROR"
 
-        # except httpx.ConnectError as error:
+            return result
-        #    result["message"] = str(error)
+        return response
-        #    result["status"] = 600
-        #    result["response"] = {"error": {"message": error}}
-        #    return result
-        # except json.decoder.JSONDecodeError:
-        #    result["message"] = "Not JSON"
-        #    if response:
-        #        result["status"] = response.status_code
-        #        result["response"] = {"error": {"message": response.text}}
-        #    return result
-        # except httpx.ReadTimeout:
-        #   result["message"] = "Read timed out"
-        #   if response:
-        #       result["status"] = response.status_code
-        #       result["response"] = {"error": {"message": response.text}}
-        #   return result
 
     # Account related API Methods
     # ===========================
 
-    def account_info(self, username: str) -> Dict[str, Any]:
+    async def account_info(self, username: str) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserByUsername
         """
-        return self._api_call(api_method=f"account_info/{username}")
+        return await self._api_call(api_method=f"account_info/{username}")
 
-    # def dashboard(self) -> Dict[str, Any]:
+    # async def dashboard(self) -> Dict[str, Any]:
     #    """See Agoradesk API.
 
     #    https://agoradesk.com/api-docs/v1#operation/getUserDashboard
     #    """
-    #    return self._api_call(api_method="dashboard")
+    #    return await self._api_call(api_method="dashboard")
 
-    def dashboard_buyer(self) -> Dict[str, Any]:
+    async def dashboard_buyer(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserDashboardBuyer
         """
-        return self._api_call(api_method="dashboard/buyer")
+        return await self._api_call(api_method="dashboard/buyer")
 
-    def dashboard(self) -> Dict[str, Any]:
+    async def dashboard(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserDashboardSeller
         """
-        return self._api_call(api_method="dashboard/seller")
+        return await self._api_call(api_method="dashboard/seller")
 
-    def dashboard_canceled(self) -> Dict[str, Any]:
+    async def dashboard_canceled(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserDashboardCanceled
         """
-        return self._api_call(api_method="dashboard/canceled")
+        return await self._api_call(api_method="dashboard/canceled")
 
-    def dashboard_closed(self) -> Dict[str, Any]:
+    async def dashboard_closed(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserDashboardClosed
         """
-        return self._api_call(api_method="dashboard/closed")
+        return await self._api_call(api_method="dashboard/closed")
 
-    def dashboard_released(self) -> Dict[str, Any]:
+    async def dashboard_released(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserDashboardReleased
         """
 
-        return self._api_call(api_method="dashboard/released")
+        return await self._api_call(api_method="dashboard/released")
 
-    def logout(self) -> Dict[str, Any]:
+    async def logout(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/logout
         """
 
-        return self._api_call(api_method="logout", http_method="POST")
+        return await self._api_call(api_method="logout", http_method="POST")
 
-    def myself(self) -> Dict[str, Any]:
+    async def myself(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getTokenOwnerUserData
         """
 
-        return self._api_call(api_method="myself")
+        return await self._api_call(api_method="myself")
 
-    def notifications(self) -> Dict[str, Any]:
+    async def notifications(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getUserNotifications
         """
 
-        return self._api_call(api_method="notifications")
+        return await self._api_call(api_method="notifications")
 
-    def notifications_mark_as_read(self, notification_id: str) -> Dict[str, Any]:
+    async def notifications_mark_as_read(self, notification_id: str) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/markNotificationRead
         """
 
-        return self._api_call(
+        return await self._api_call(
             api_method=f"notifications/mark_as_read/{notification_id}",
             http_method="POST",
         )
 
-    def recent_messages(self) -> Dict[str, Any]:
+    async def recent_messages(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getRecemtMessages
         """
 
-        return self._api_call(api_method="recent_messages")
+        return await self._api_call(api_method="recent_messages")
 
     # Trade related API Methods
     # ===========================
 
     #     post/feedback/{username} • Give feedback to a user
-    def feedback(
+    async def feedback(
         self, username: str, feedback: str, msg: Optional[str]
     ) -> Dict[str, Any]:
         """See Agoradesk API.
@@ -261,29 +220,41 @@ class AgoraDesk:
         params = {"feedback": feedback}
         if msg:
             params["msg"] = msg
-        return self._api_call(
+        return await self._api_call(
             api_method=f"feedback/{username}",
             http_method="POST",
             query_values=params,
         )
 
     #     Todo:
-    #     post/trade/contact_release/{trade_id} • Release trade escrow
     #     post/contact_fund/{trade_id} • Fund a trade
     #     post/contact_dispute/{trade_id} • Start a trade dispute
 
     #     post/contact_mark_as_paid/{trade_id} • Mark a trade as paid
-    def contact_mark_as_paid(self, trade_id: str) -> Dict[str, Any]:
+
+    async def contact_release(self, trade_id: str, password: str) -> Dict[str, Any]:
+        """See Agoradesk API documentation.
+
+        https://agoradesk.com/api-docs/v1#operation/releaseEscrow
+        """
+        payload = {"tradeId": trade_id, "password": password}
+        return await self._api_call(
+            api_method=f"contact_release/{trade_id}",
+            http_method="POST",
+            query_values=payload,
+        )
+
+    async def contact_mark_as_paid(self, trade_id: str) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/markPaid
         """
-        return self._api_call(
+        return await self._api_call(
             api_method=f"contact_mark_as_paid/{trade_id}", http_method="POST"
         )
 
     #     post/contact_cancel/{trade_id} • Cancel the trade
-    def contact_cancel(
+    async def contact_cancel(
         self,
         trade_id: str,
     ) -> Dict[str, Any]:
@@ -291,7 +262,7 @@ class AgoraDesk:
 
         https://agoradesk.com/api-docs/v1#operation/cancelTrade
         """
-        return self._api_call(
+        return await self._api_call(
             api_method=f"contact_cancel/{trade_id}",
             http_method="POST",
         )
@@ -300,7 +271,7 @@ class AgoraDesk:
     #     post/contact_escrow/{trade_id} • Enable escrow
 
     #     get/contact_messages/{trade_id} • Get trade messages
-    def contact_messages(
+    async def contact_messages(
         self, trade_id: str, after: Optional[arrow.Arrow] = None
     ) -> Dict[str, Any]:
         """See Agoradesk API.
@@ -318,7 +289,7 @@ class AgoraDesk:
         return reply
 
     #     post/contact_create/{ad_id} • Start a trade
-    def contact_create(
+    async def contact_create(
         self,
         ad_id: str,
         amount: float,
@@ -331,14 +302,14 @@ class AgoraDesk:
         payload: Dict[str, Any] = {"amount": amount}
         if msg:
             payload["msg"] = msg
-        return self._api_call(
+        return await self._api_call(
             api_method=f"contact_create/{ad_id}",
             http_method="POST",
             query_values=payload,
         )
 
     #     get/contact_info/{trade_id} • Get a trade by trade ID
-    def contact_info(self, trade_ids: Union[str, List[str]]) -> Dict[str, Any]:
+    async def contact_info(self, trade_ids: Union[str, List[str]]) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getTradeById and
@@ -353,11 +324,11 @@ class AgoraDesk:
         else:
             params = f"/{trade_ids}"
         api_method += params
-        return self._api_call(api_method=api_method)
+        return await self._api_call(api_method=api_method)
 
     #     Todo: Add image upload functionality
     #     post/contact_message_post/{trade_id} • Send a chat message/attachment
-    def contact_message_post(
+    async def contact_message_post(
         self, trade_id: str, msg: Optional[str] = None
     ) -> Dict[str, Any]:
         """See Agoradesk API.
@@ -365,7 +336,7 @@ class AgoraDesk:
         https://agoradesk.com/api-docs/v1#operation/sendChatMessage
         """
         payload = {"msg": msg}
-        return self._api_call(
+        return await self._api_call(
             api_method=f"contact_message_post/{trade_id}",
             http_method="POST",
             query_values=payload,
@@ -377,7 +348,7 @@ class AgoraDesk:
     # Advertisement related API Methods
     # ================================
 
-    def ad_create(
+    async def ad_create(
         self,
         country_code: str,
         currency: str,
@@ -450,13 +421,13 @@ class AgoraDesk:
         if lon:
             params["lon"] = lon
 
-        return self._api_call(
+        return await self._api_call(
             api_method="ad-create",
             http_method="POST",
             query_values=params,
         )
 
-    def ad(
+    async def ad(
         self,
         ad_id: str,
         country_code: Optional[str] = None,
@@ -540,34 +511,34 @@ class AgoraDesk:
             params["lat"] = lat
         if lon:
             params["lon"] = lon
-        if visible:
+        if visible is not None:
             params["visible"] = True if visible else False
 
-        return self._api_call(
+        return await self._api_call(
             api_method=f"ad/{ad_id}",
             http_method="POST",
             query_values=params,
         )
 
-    def ad_equation(self, ad_id: str, price_equation: str) -> Dict[str, Any]:
+    async def ad_equation(self, ad_id: str, price_equation: str) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/updateFormula
         """
-        return self._api_call(
+        return await self._api_call(
             api_method=f"ad-equation/{ad_id}",
             http_method="POST",
             query_values={"price_equation": price_equation},
         )
 
-    def ad_delete(self, ad_id: str) -> Dict[str, Any]:
+    async def ad_delete(self, ad_id: str) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/deleteAd
         """
-        return self._api_call(api_method=f"ad-delete/{ad_id}", http_method="POST")
+        return await self._api_call(api_method=f"ad-delete/{ad_id}", http_method="POST")
 
-    def ads(
+    async def ads(
         self,
         country_code: Optional[str] = None,
         currency: Optional[str] = None,
@@ -604,11 +575,11 @@ class AgoraDesk:
             params["page"] = page
 
         if len(params) == 0:
-            return self._api_call(api_method="ads")
+            return await self._api_call(api_method="ads")
 
-        return self._api_call(api_method="ads", query_values=params)
+        return await self._api_call(api_method="ads", query_values=params)
 
-    def ad_get(self, ad_ids: List[str]) -> Dict[str, Any]:
+    async def ad_get(self, ad_ids: List[str]) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getAdById and
@@ -622,9 +593,11 @@ class AgoraDesk:
             api_method += f"/{ids}"
         else:
             params = {"ads": ids}
-        return self._api_call(api_method=api_method, query_values=params)
+        return await self._api_call(api_method=api_method, query_values=params)
 
-    def payment_methods(self, country_code: Optional[str] = None) -> Dict[str, Any]:
+    async def payment_methods(
+        self, country_code: Optional[str] = None
+    ) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/paymentMethods and
@@ -633,28 +606,28 @@ class AgoraDesk:
         api_method = "payment_methods"
         if country_code:
             api_method += f"/{country_code}"
-        return self._api_call(api_method=api_method)
+        return await self._api_call(api_method=api_method)
 
-    def country_codes(self) -> Dict[str, Any]:
+    async def country_codes(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/countryCodes
         """
-        return self._api_call(api_method="countrycodes")
+        return await self._api_call(api_method="countrycodes")
 
-    def currencies(self) -> Dict[str, Any]:
+    async def currencies(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/currencyCodes
         """
-        return self._api_call(api_method="currencies")
+        return await self._api_call(api_method="currencies")
 
-    def equation(self, price_equation: str, currency: str) -> Dict[str, Any]:
+    async def equation(self, price_equation: str, currency: str) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/priceFormula
         """
-        return self._api_call(
+        return await self._api_call(
             api_method="equation",
             http_method="POST",
             query_values={
@@ -666,7 +639,7 @@ class AgoraDesk:
     # Public ad search related API Methods
     # ====================================
 
-    def _generic_online(
+    async def _generic_online(
         self,
         direction: str,
         main_currency: str,
@@ -685,7 +658,7 @@ class AgoraDesk:
             add_to_api_method += f"/{payment_method}"
 
         params = self._generic_search_parameters(amount, page)
-        return self._api_call(
+        return await self._api_call(
             api_method=f"{direction}-{main_currency}-online/"
             f"{exchange_currency}{add_to_api_method}",
             query_values=params,
@@ -702,7 +675,8 @@ class AgoraDesk:
             params = {"page": f"{page}"}
         return params
 
-    def buy_monero_online(
+    #
+    async def buy_monero_online(
         self,
         currency_code: str,
         country_code: Optional[str] = None,
@@ -720,7 +694,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_online(
+        return await self._generic_online(
             direction="buy",
             main_currency="monero",
             exchange_currency=currency_code,
@@ -730,7 +704,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def buy_bitcoins_online(
+    async def buy_bitcoins_online(
         self,
         currency_code: str,
         country_code: Optional[str] = None,
@@ -748,7 +722,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_online(
+        return await self._generic_online(
             direction="buy",
             main_currency="bitcoins",
             exchange_currency=currency_code,
@@ -758,7 +732,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def sell_monero_online(
+    async def sell_monero_online(
         self,
         currency_code: str,
         country_code: Optional[str] = None,
@@ -776,7 +750,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_online(
+        return await self._generic_online(
             direction="sell",
             main_currency="monero",
             exchange_currency=currency_code,
@@ -786,7 +760,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def sell_bitcoins_online(
+    async def sell_bitcoins_online(
         self,
         currency_code: str,
         country_code: Optional[str] = None,
@@ -804,7 +778,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_online(
+        return await self._generic_online(
             direction="sell",
             main_currency="bitcoins",
             exchange_currency=currency_code,
@@ -814,7 +788,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def _generic_cash(
+    async def _generic_cash(
         self,
         direction: str,
         main_currency: str,
@@ -829,13 +803,13 @@ class AgoraDesk:
 
         params = self._generic_search_parameters(amount, page)
 
-        return self._api_call(
+        return await self._api_call(
             api_method=f"{direction}-{main_currency}-with-cash/"
             f"{exchange_currency}/{country_code}/{lat}/{lon}",
             query_values=params,
         )
 
-    def buy_monero_with_cash(
+    async def buy_monero_with_cash(
         self,
         currency_code: str,
         country_code: str,
@@ -851,7 +825,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_cash(
+        return await self._generic_cash(
             direction="buy",
             main_currency="monero",
             exchange_currency=currency_code,
@@ -862,7 +836,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def buy_bitcoins_with_cash(
+    async def buy_bitcoins_with_cash(
         self,
         currency_code: str,
         country_code: str,
@@ -878,7 +852,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_cash(
+        return await self._generic_cash(
             direction="buy",
             main_currency="bitcoins",
             exchange_currency=currency_code,
@@ -889,7 +863,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def sell_monero_with_cash(
+    async def sell_monero_with_cash(
         self,
         currency_code: str,
         country_code: str,
@@ -905,7 +879,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_cash(
+        return await self._generic_cash(
             direction="sell",
             main_currency="monero",
             exchange_currency=currency_code,
@@ -916,7 +890,7 @@ class AgoraDesk:
             page=page,
         )
 
-    def sell_bitcoins_with_cash(
+    async def sell_bitcoins_with_cash(
         self,
         currency_code: str,
         country_code: str,
@@ -932,7 +906,7 @@ class AgoraDesk:
 
         # pylint: disable=too-many-arguments
 
-        return self._generic_cash(
+        return await self._generic_cash(
             direction="sell",
             main_currency="bitcoins",
             exchange_currency=currency_code,
@@ -946,7 +920,7 @@ class AgoraDesk:
     # Statistics related API Methods
     # ==============================
 
-    def moneroaverage(
+    async def moneroaverage(
         self, currency: Optional[str] = "ticker-all-currencies"
     ) -> Dict[str, Any]:
         """See Agoradesk API.
@@ -954,68 +928,68 @@ class AgoraDesk:
         https://agoradesk.com/api-docs/v1#operation/getXmrTicker and
         https://agoradesk.com/api-docs/v1#operation/getXmrTickerByCurrencyCode
         """
-        return self._api_call(api_method=f"moneroaverage/{currency}")
+        return await self._api_call(api_method=f"moneroaverage/{currency}")
 
     # Wallet related API Methods
     # ===========================
 
-    def wallet(self) -> Dict[str, Any]:
+    async def wallet(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getBtcWallet
         """
-        return self._api_call(api_method="wallet")
+        return await self._api_call(api_method="wallet")
 
-    def wallet_balance(self) -> Dict[str, Any]:
+    async def wallet_balance(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getBtcWalletBalance
         """
-        return self._api_call(api_method="wallet-balance")
+        return await self._api_call(api_method="wallet-balance")
 
-    def wallet_xmr(self) -> Dict[str, Any]:
+    async def wallet_xmr(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getXmrWallet
         """
-        return self._api_call(api_method="wallet/XMR")
+        return await self._api_call(api_method="wallet/XMR")
 
-    def wallet_balance_xmr(self) -> Dict[str, Any]:
+    async def wallet_balance_xmr(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getXmrWalletBalance
         """
-        return self._api_call(api_method="wallet-balance/XMR")
+        return await self._api_call(api_method="wallet-balance/XMR")
 
-    def wallet_addr(self) -> Dict[str, Any]:
+    async def wallet_addr(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getBtcAddress
         """
-        return self._api_call(api_method="wallet-addr")
+        return await self._api_call(api_method="wallet-addr")
 
-    def wallet_addr_xmr(self) -> Dict[str, Any]:
+    async def wallet_addr_xmr(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getXMRAddress
         """
-        return self._api_call(api_method="wallet-addr/XMR")
+        return await self._api_call(api_method="wallet-addr/XMR")
 
-    def fees(self) -> Dict[str, Any]:
+    async def fees(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getBtcFee
         """
-        return self._api_call(api_method="fees")
+        return await self._api_call(api_method="fees")
 
-    def fees_xmr(self) -> Dict[str, Any]:
+    async def fees_xmr(self) -> Dict[str, Any]:
         """See Agoradesk API.
 
         https://agoradesk.com/api-docs/v1#operation/getXmrFee
         """
-        return self._api_call(api_method="fees/XMR")
+        return await self._api_call(api_method="fees/XMR")
 
-    def wallet_send(
+    async def wallet_send(
         self,
         address: str,
         amount: float,
@@ -1038,11 +1012,11 @@ class AgoraDesk:
         if otp:
             params["otp"] = otp
 
-        return self._api_call(
+        return await self._api_call(
             api_method="wallet-send", http_method="POST", query_values=params
         )
 
-    def wallet_send_xmr(
+    async def wallet_send_xmr(
         self,
         address: str,
         amount: float,
@@ -1065,8 +1039,9 @@ class AgoraDesk:
         if otp:
             params["otp"] = otp
 
-        return self._api_call(
+        response = await self._api_call(
             api_method="wallet-send/XMR",
             http_method="POST",
             query_values=params,
         )
+        return response

core/forms.py

@@ -0,0 +1,350 @@
+from django import forms
+from django.contrib.auth.forms import UserCreationForm
+from django.core.exceptions import FieldDoesNotExist
+from django.forms import ModelForm
+from mixins.restrictions import RestrictedFormMixin
+
+from .models import (
+    Ad,
+    Aggregator,
+    Asset,
+    LinkGroup,
+    NotificationSettings,
+    OperatorWallets,
+    Payout,
+    Platform,
+    Provider,
+    Requisition,
+    User,
+    Wallet,
+)
+
+# flake8: noqa: E501
+
+
+class NewUserForm(UserCreationForm):
+    email = forms.EmailField(required=True)
+
+    class Meta:
+        model = User
+        fields = (
+            "username",
+            "email",
+            "first_name",
+            "last_name",
+            "password1",
+            "password2",
+        )
+
+    def save(self, commit=True):
+        user = super(NewUserForm, self).save(commit=False)
+        user.email = self.cleaned_data["email"]
+        if commit:
+            user.save()
+        return user
+
+
+class CustomUserCreationForm(UserCreationForm):
+    class Meta:
+        model = User
+        fields = "__all__"
+
+
+class NotificationSettingsForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = NotificationSettings
+        fields = (
+            "ntfy_topic",
+            "ntfy_url",
+        )
+        help_texts = {
+            "ntfy_topic": "The topic to send notifications to.",
+            "ntfy_url": "Custom NTFY server. Leave blank to use the default server.",
+        }
+
+
+class AggregatorForm(RestrictedFormMixin, ModelForm):
+    def __init__(self, *args, **kwargs):
+        super(AggregatorForm, self).__init__(*args, **kwargs)
+        self.fields["secret_id"].label = "Secret ID"
+
+    class Meta:
+        model = Aggregator
+        fields = (
+            "name",
+            "service",
+            "secret_id",
+            "secret_key",
+            "poll_interval",
+            "link_group",
+            "enabled",
+        )
+        help_texts = {
+            "name": "The name of the aggregator connection.",
+            "service": "The aggregator service to use.",
+            "secret_id": "The secret ID for the aggregator service.",
+            "secret_key": "The secret key for the aggregator service.",
+            "poll_interval": "The interval in seconds to poll the aggregator service.",
+            "link_group": "The link group to use for this aggregator connection.",
+            "enabled": "Whether or not the aggregator connection is enabled.",
+        }
+
+
+class PlatformForm(RestrictedFormMixin, ModelForm):
+    def __init__(self, *args, **kwargs):
+        super(PlatformForm, self).__init__(*args, **kwargs)
+        upper = ["usd", "otp"]
+        for field in self.fields:
+            for up in upper:
+                if self.fields[field].label:
+                    if up in self.fields[field].label:
+                        self.fields[field].label = self.fields[field].label.replace(
+                            up, up.upper()
+                        )
+
+    class Meta:
+        model = Platform
+        fields = (
+            "name",
+            "service",
+            "token",
+            "password",
+            "otp_token",
+            "username",
+            "send",
+            "cheat",
+            "dummy",
+            "cheat_interval_seconds",
+            "margin",
+            "max_margin",
+            "min_margin",
+            "min_trade_size_usd",
+            "max_trade_size_usd",
+            "accept_within_usd",
+            "no_reference_amount_check_max_usd",
+            "base_usd",
+            "withdrawal_trigger",
+            "payees",
+            "link_group",
+            "enabled",
+        )
+        help_texts = {
+            "name": "The name of the platform connection.",
+            "service": "The platform service to use.",
+            "token": "The JWT auth token.",
+            "password": "Account password",
+            "otp_token": "The OTP secret key.",
+            "username": "Account username",
+            "send": "Whether or not to send messages on new trades.",
+            "cheat": "Whether or not to run the Autoprice cheat.",
+            "dummy": "When enabled, the trade escrow feature will be disabled.",
+            "cheat_interval_seconds": "The interval in seconds to run the Autoprice cheat.",
+            "margin": "The current margin. Only valid for initial ads post. Autoprice will override this.",
+            "max_margin": "The maximum margin to use.",
+            "min_margin": "The minimum margin to use.",
+            "min_trade_size_usd": "The minimum trade size in USD.",
+            "max_trade_size_usd": "The maximum trade size in USD.",
+            "accept_within_usd": "When a trade is wrong by less than this amount, it will be accepted.",
+            "no_reference_amount_check_max_usd": "When ticked, when no reference was found and a trade is higher than this amount, we will not accept payment even if it is the only one with this amount.",
+            "base_usd": "The amount in USD to keep in the platform.",
+            "withdrawal_trigger": "The amount above the base USD to trigger a withdrawal.",
+            "payees": "The wallet addresses to send profit concerning this platform to.",
+            "link_group": "The link group to use for this platform.",
+            "enabled": "Whether or not the platform connection is enabled.",
+        }
+
+    payees = forms.ModelMultipleChoiceField(
+        queryset=Wallet.objects.all(),
+        widget=forms.CheckboxSelectMultiple,
+        help_text=Meta.help_texts["payees"],
+        required=False,
+    )
+
+
+class AdForm(RestrictedFormMixin, ModelForm):
+    def __init__(self, *args, **kwargs):
+        super(AdForm, self).__init__(*args, **kwargs)
+
+    class Meta:
+        model = Ad
+        fields = (
+            "name",
+            "text",
+            "payment_details",
+            "payment_details_real",
+            "payment_method_details",
+            "dist_list",
+            "asset_list",
+            "provider_list",
+            # "platforms",
+            # "aggregators",
+            "require_feedback_score",
+            "account_whitelist",
+            "send_reference",
+            "visible",
+            "link_group",
+            "enabled",
+        )
+        help_texts = {
+            "name": "The name of the ad.",
+            "text": "The content of the ad.",
+            "payment_details": "Shown before a user opens a trade.",
+            "payment_details_real": "Shown after a user opens a trade.",
+            "payment_method_details": "Shown in the list",
+            "dist_list": "Currency and country, space separated, one pair per line.",
+            "asset_list": "List of assets to distribute ads for.",
+            "provider_list": "List of providers to distribute ads for.",
+            # "platforms": "Enabled platforms for this ad",
+            # "aggregators": "Enabled aggregators for this ad",
+            "require_feedback_score": "Mminimum feedback score for users. Set to 0 to disable.",
+            "account_whitelist": "List of account IDs to use, one per line.",
+            "send_reference": "Whether or not to send the reference on new trades.",
+            "visible": "Whether or not this ad is visible.",
+            "link_group": "The link group to use for this ad.",
+            "enabled": "Whether or not this ad is enabled.",
+        }
+
+    asset_list = forms.ModelMultipleChoiceField(
+        queryset=Asset.objects.all(),
+        widget=forms.CheckboxSelectMultiple,
+        help_text=Meta.help_texts["asset_list"],
+        required=True,
+    )
+    provider_list = forms.ModelMultipleChoiceField(
+        queryset=Provider.objects.all(),
+        widget=forms.CheckboxSelectMultiple,
+        help_text=Meta.help_texts["provider_list"],
+        required=True,
+    )
+    # platforms = forms.ModelMultipleChoiceField(
+    #     queryset=Platform.objects.all(),
+    #     widget=forms.CheckboxSelectMultiple,
+    #     help_text=Meta.help_texts["platforms"],
+    #     required=True,
+    # )
+    # aggregators = forms.ModelMultipleChoiceField(
+    #     queryset=Aggregator.objects.all(),
+    #     widget=forms.CheckboxSelectMultiple,
+    #     help_text=Meta.help_texts["aggregators"],
+    #     required=True,
+    # )
+
+
+class RequisitionForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = Requisition
+        fields = (
+            "payment_details",
+            "owner_name",
+            "transaction_source",
+            "payees",
+        )
+
+        help_texts = {
+            "payment_details": "Shown once a user opens a trade.",
+            "owner_name": "Owner name to send with payment details if not provided by aggregator.",
+            "transaction_source": "Whether to check pending or booked transactions.",
+            "payees": "The wallet addresses to send profit concerning this requisition to.",
+        }
+
+    payees = forms.ModelMultipleChoiceField(
+        queryset=Wallet.objects.all(),
+        widget=forms.CheckboxSelectMultiple,
+        help_text=Meta.help_texts["payees"],
+        required=False,
+    )
+
+
+class WalletForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = Wallet
+        fields = (
+            "name",
+            "address",
+        )
+
+        help_texts = {
+            "name": "The name of the wallet.",
+            "address": "The XMR address to send funds to.",
+        }
+
+
+class LinkGroupForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = LinkGroup
+        fields = (
+            "name",
+            "platform_owner_cut_percentage",
+            "requisition_owner_cut_percentage",
+            "operator_cut_percentage",
+            "enabled",
+        )
+
+        help_texts = {
+            "name": "The name of the link group.",
+            "platform_owner_cut_percentage": "The percentage of the total profit of this group to give to the platform owners.",
+            "requisition_owner_cut_percentage": "The percentage of the total profit of this group to give to the requisition owners.",
+            "operator_cut_percentage": "The percentage of the total profit of this group to give to the operator.",
+            "enabled": "Whether or not this link group is enabled.",
+        }
+
+    def clean(self):
+        cleaned_data = super(LinkGroupForm, self).clean()
+        platform_owner_cut_percentage = cleaned_data.get(
+            "platform_owner_cut_percentage"
+        )
+        requisition_owner_cut_percentage = cleaned_data.get(
+            "requisition_owner_cut_percentage"
+        )
+        operator_cut_percentage = cleaned_data.get("operator_cut_percentage")
+        total_sum = (
+            platform_owner_cut_percentage
+            + requisition_owner_cut_percentage
+            + operator_cut_percentage
+        )
+        if total_sum != 100:
+            self.add_error(
+                "platform_owner_cut_percentage",
+                f"The sum of the percentages must be 100, not {total_sum}.",
+            )
+            self.add_error(
+                "requisition_owner_cut_percentage",
+                f"The sum of the percentages must be 100, not {total_sum}.",
+            )
+            self.add_error(
+                "operator_cut_percentage",
+                f"The sum of the percentages must be 100, not {total_sum}.",
+            )
+            return
+        return cleaned_data
+
+
+class OperatorWalletsForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = OperatorWallets
+        fields = ("payees",)
+        help_texts = {
+            "payees": "Wallets to designate as payees for this operator.",
+        }
+
+    payees = forms.ModelMultipleChoiceField(
+        queryset=Wallet.objects.all(),
+        widget=forms.CheckboxSelectMultiple,
+        help_text=Meta.help_texts["payees"],
+        required=False,
+    )
+
+
+class PayoutForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = Payout
+        fields = (
+            "wallet",
+            "amount",
+            "description",
+        )
+        help_texts = {
+            "wallet": "The wallet the payment was sent to.",
+            "amount": "The amount of the payment.",
+            "description": "The description of the payment.",
+        }

core/lib/antifraud.py

@@ -0,0 +1,115 @@
+# Project imports
+# from core.lib import db  # , notify
+from core.util import logs
+
+log = logs.get_logger("antifraud")
+
+
+class AntiFraud(object):
+    async def add_bank_sender(self, platform_buyer, bank_sender):
+        """
+        Add the bank senders into Redis.
+        :param platform: name of the platform - freeform
+        :param platform_buyer: the username of the buyer on the platform
+        :param bank_sender: the sender name from the bank
+        """
+        # key = f"namemap.{platform}.{platform_buyer}"
+        # await db.r.sadd(key, bank_sender)
+        # TODO
+
+    async def get_previous_senders(self, platform, platform_buyer):
+        """
+        Get all the previous bank sender names for the given buyer on the platform.
+        :param platform: name of the platform - freeform
+        :param platform_buyer: the username of the buyer on the platform
+        :return: set of previous buyers
+        :rtype: set
+        """
+        # key = f"namemap.{platform}.{platform_buyer}"
+        # senders = await db.r.smembers(key)
+        # if not senders:
+        #     return None
+        # senders = db.convert(senders)
+        # return senders
+        # TODO
+
+    async def check_valid_sender(
+        self, reference, platform, bank_sender, platform_buyer
+    ):
+        """
+        Check that either:
+        * The platform buyer has never had a recognised transaction before
+        * The bank sender name matches a previous transaction from the platform buyer
+        :param reference: the trade reference
+        :param platform: name of the platform - freeform
+        :param bank_sender: the sender of the bank transaction
+        :param platform_buyer: the username of the buyer on the platform
+        :return: whether the sender is valid
+        :rtype: bool
+        """
+        # senders = await self.get_previous_senders(platform, platform_buyer)
+        # if senders is None:  # no senders yet, assume it's valid
+        #     return True
+        # if platform_buyer in senders:
+        #     return True
+        # self.ux.notify.notify_sender_name_mismatch(
+        #     reference, platform_buyer, bank_sender
+        # )
+        # # title = "Sender name mismatch"
+        # # message = (
+        # #     f"Sender name mismatch for {reference}:\n"
+        # #     f"Platform buyer: {platform_buyer}"
+        # #     f"Bank sender: {bank_sender}"
+        # # )
+        # # await notify.sendmsg(self.instance.) # TODO
+        # return False
+        # TODO
+
+    async def check_tx_sender(self, tx, reference):
+        """
+        Check whether the sender of a given transaction is authorised based on the
+        previous transactions of the username that originated the trade reference.
+        :param tx: the transaction ID
+        :param reference: the trade reference
+        """
+        # stored_trade = await db.get_ref(reference)
+        # if not stored_trade:
+        #     return None
+        # stored_tx = await db.get_tx(tx)
+        # if not stored_tx:
+        #     return None
+        # bank_sender = stored_tx["sender"]
+        # platform_buyer = stored_trade["buyer"]
+        # platform = stored_trade["subclass"]
+        # is_allowed = await self.check_valid_sender(
+        #     reference, platform, bank_sender, platform_buyer
+        # )
+        # if is_allowed is True:
+        #     return True
+        # return False
+        # TODO
+
+    # def user_verification_successful(self, uid):
+    #     """
+    #     A user has successfully completed verification.
+    #     """
+    #     self.log.info(f"User has completed verification: {uid}")
+    #     trade_list = self.markets.find_trades_by_uid(uid)
+    #     for platform, trade_id, reference, currency in trade_list:
+    #         self.markets.send_bank_details(platform, currency, trade_id)
+    #         self.markets.send_reference(platform, trade_id, reference)
+
+    # def send_verification_url(self, platform, uid, trade_id):
+    #     send_setting, post_message = self.markets.get_send_settings(platform)
+    #     if send_setting == "1":
+    #         auth_url = self.ux.verify.create_applicant_and_get_link(uid)
+    #         if platform == "lbtc":
+    #             auth_url = auth_url.replace("https://", "")  # hack
+    #         post_message(
+    #             trade_id,
+    #             f"Hi! To continue the trade, please complete the verification form:
+    # {auth_url}",
+    #         )
+
+
+antifraud = AntiFraud()

core/lib/elastic.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+
+from django.conf import settings
+from elasticsearch import Elasticsearch
+
+from core.util import logs
+
+log = logs.get_logger(__name__)
+
+client = None
+
+
+def initialise_elasticsearch():
+    """
+    Initialise the Elasticsearch client.
+    """
+    auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
+    client = Elasticsearch(
+        settings.ELASTICSEARCH_HOST, http_auth=auth, verify_certs=False
+    )
+    return client
+
+
+def store_msg(index, msg):
+    global client
+    if not client:
+        client = initialise_elasticsearch()
+    if "ts" not in msg:
+        msg["ts"] = datetime.utcnow().isoformat()
+    result = client.index(index=index, body=msg)
+    if not result["result"] == "created":
+        log.error(f"Indexing of '{msg}' failed: {result}")

core/lib/money.py

@@ -0,0 +1,631 @@
+# Twisted imports
+import asyncio
+import logging
+from datetime import datetime
+
+import urllib3
+from aiocoingecko import AsyncCoinGeckoAPISession
+from django.conf import settings
+from elasticsearch import AsyncElasticsearch
+from forex_python.converter import CurrencyRates
+
+# Other library imports
+from core.models import Aggregator, OperatorWallets, Platform
+from core.util import logs
+
+# TODO: secure ES traffic properly
+urllib3.disable_warnings()
+
+tracer = logging.getLogger("opensearch")
+tracer.setLevel(logging.CRITICAL)
+tracer = logging.getLogger("elastic_transport.transport")
+tracer.setLevel(logging.CRITICAL)
+
+log = logs.get_logger("money")
+
+
+class Money(object):
+    """
+    Generic class for handling money-related matters that aren't Revolut or Agora.
+    """
+
+    def __init__(self):
+        """
+        Initialise the Money object.
+        Set the logger.
+        Initialise the CoinGecko API.
+        """
+        self.cr = CurrencyRates()
+        self.cg = AsyncCoinGeckoAPISession()
+        auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
+        client = AsyncElasticsearch(
+            settings.ELASTICSEARCH_HOST, http_auth=auth, verify_certs=False
+        )
+        self.es = client
+
+    async def check_all(self, user=None, link_group=None, nordigen=None, agora=None):
+        """
+        Run all the balance checks that output into ES in another thread.
+        """
+        if not all([nordigen, agora]):
+            raise Exception
+        if not any([user, link_group]):
+            raise Exception
+
+        # I hate circular dependencies
+        self.nordigen = nordigen
+        self.agora = agora
+
+        cast = {}
+        if user is not None:
+            cast["user"] = user
+        if link_group is not None:
+            cast["link_group"] = link_group
+
+        aggregators = Aggregator.objects.filter(enabled=True, **cast)
+        platforms = Platform.objects.filter(enabled=True, **cast)
+
+        total = await self.get_total(aggregators, platforms, trades=True)
+
+        return total
+
+    # def setup_loops(self):
+    #     """
+    #     Set up the LoopingCalls to get the balance so we have data in ES.
+    #     """
+    #     if settings.ES.Enabled == "1" or settings.Logstash.Enabled == "1":
+    #         self.lc_es_checks = LoopingCall(self.run_checks_in_thread)
+    #         delay = int(settings.ES.RefreshSec)
+    #         self.lc_es_checks.start(delay)
+    #         if settings.ES.Enabled == "1":
+    #             self.agora.es = self.es
+    #             self.lbtc.es = self.es
+
+    async def write_to_es(self, msgtype, cast):
+        cast["type"] = "money"
+        cast["ts"] = str(datetime.now().isoformat())
+        cast["xtype"] = msgtype
+        # cast["user_id"] = self.instance.user.id
+        # cast["platform_id"] = self.instance.id
+        try:
+            await self.es.index(index=settings.ELASTICSEARCH_INDEX, body=cast)
+        except RuntimeError:
+            log.warning("Could not write to ES")
+
+    async def lookup_rates(self, platform, ads, rates=None):
+        """
+        Lookup the rates for a list of public ads.
+        """
+        if not rates:
+            rates = await self.cg.get_price(
+                ids=["monero", "bitcoin"],
+                vs_currencies=self.markets.get_all_currencies(platform),
+            )
+            # Set the price based on the asset
+        for ad in ads:
+            if ad[4] == "XMR":
+                coin = "monero"
+            elif ad[4] == "BTC":
+                coin = "bitcoin"  # No s here
+            currency = ad[5]
+            base_currency_price = rates[coin][currency.lower()]
+            price = float(ad[2])
+            rate = round(price / base_currency_price, 2)
+            ad.append(rate)
+        # TODO: sort?
+        return sorted(ads, key=lambda x: x[2])
+
+    async def get_rates_all(self):
+        """
+        Get all rates that pair with USD.
+        :return: dictionary of USD/XXX rates
+        :rtype: dict
+        """
+        rates = self.cr.get_rates("USD")
+        return rates
+
+    # TODO: pass platform
+    async def get_acceptable_margins(self, platform, currency, amount):
+        """
+        Get the minimum and maximum amounts we would accept a trade for.
+        :param currency: currency code
+        :param amount: amount
+        :return: (min, max)
+        :rtype: tuple
+        """
+        rates = await self.get_rates_all()
+        if currency == "USD":
+            min_amount = amount - platform.accept_within_usd
+            max_amount = amount + platform.accept_within_usd
+            return (min_amount, max_amount)
+        amount_usd = amount / rates[currency]
+        min_usd = amount_usd - platform.accept_within_usd
+        max_usd = amount_usd + platform.accept_within_usd
+        min_local = min_usd * rates[currency]
+        max_local = max_usd * rates[currency]
+        return (min_local, max_local)
+
+    async def get_minmax(self, min_usd, max_usd, asset, currency):
+        rates = await self.get_rates_all()
+        if currency not in rates and not currency == "USD":
+            log.error(f"Can't create ad without rates: {currency}")
+            return (None, None)
+        if currency == "USD":
+            min_amount = min_usd
+            max_amount = max_usd
+        else:
+            min_amount = rates[currency] * min_usd
+            max_amount = rates[currency] * max_usd
+
+        return (min_amount, max_amount)
+
+    async def to_usd(self, amount, currency):
+        if currency == "USD":
+            return float(amount)
+        else:
+            rates = await self.get_rates_all()
+            return float(amount) / rates[currency]
+
+    async def multiple_to_usd(self, currency_map, rates=None):
+        """
+        Convert multiple curencies to USD while saving API calls.
+        """
+        if not rates:
+            rates = await self.get_rates_all()
+        cumul = 0
+        for currency, amount in currency_map.items():
+            if currency == "USD":
+                cumul += float(amount)
+            else:
+                cumul += float(amount) / rates[currency]
+        return cumul
+
+    async def get_total_usd(self):
+        """
+        Get total USD in all our accounts, bank and trading.
+        :return: value in USD
+        :rtype float:
+        """
+        total_sinks_usd = await self.sinks.get_total_usd()
+        agora_wallet_xmr = await self.agora.api.wallet_balance_xmr()
+        agora_wallet_btc = await self.agora.api.wallet_balance()
+        # lbtc_wallet_btc = await self.lbtc.api.wallet_balance()
+        if not agora_wallet_xmr["success"]:
+            return False
+        if not agora_wallet_btc["success"]:
+            return False
+        # if not lbtc_wallet_btc["success"]:
+        #     return False
+        if not agora_wallet_xmr["response"]:
+            return False
+        if not agora_wallet_btc["response"]:
+            return False
+        # if not lbtc_wallet_btc["response"]:
+        #     return False
+        total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"]
+        total_btc_agora = agora_wallet_btc["response"]["data"]["total"]["balance"]
+        # total_btc_lbtc = lbtc_wallet_btc["response"]["data"]["total"]["balance"]
+        # Get the XMR -> USD exchange rate
+        xmr_usd = await self.cg.get_price(ids="monero", vs_currencies=["USD"])
+
+        # Get the BTC -> USD exchange rate
+        btc_usd = await self.cg.get_price(ids="bitcoin", vs_currencies=["USD"])
+
+        # Convert the Agora BTC total to USD
+        total_usd_agora_btc = float(total_btc_agora) * btc_usd["bitcoin"]["usd"]
+
+        # Convert the LBTC BTC total to USD
+        # total_usd_lbtc_btc = float(total_btc_lbtc) * btc_usd["bitcoin"]["usd"]
+
+        # Convert the Agora XMR total to USD
+        total_usd_agora_xmr = float(total_xmr_agora) * xmr_usd["monero"]["usd"]
+
+        # Add it all up
+        total_usd_agora = total_usd_agora_xmr + total_usd_agora_btc
+        # total_usd_lbtc = total_usd_lbtc_btc
+        total_usd = total_usd_agora + total_sinks_usd
+        # total_usd_lbtc +
+        cast_es = {
+            "price_usd": total_usd,
+            "total_usd_agora_xmr": total_usd_agora_xmr,
+            "total_usd_agora_btc": total_usd_agora_btc,
+            # "total_usd_lbtc_btc": total_usd_lbtc_btc,
+            "total_xmr_agora": total_xmr_agora,
+            "total_btc_agora": total_btc_agora,
+            # "total_btc_lbtc": total_btc_lbtc,
+            "xmr_usd": xmr_usd["monero"]["usd"],
+            "btc_usd": btc_usd["bitcoin"]["usd"],
+            "total_sinks_usd": total_sinks_usd,
+            "total_usd_agora": total_usd_agora,
+        }
+        await self.write_to_es("get_total_usd", cast_es)
+        return total_usd
+
+    async def gather_total_map(self, aggregators, rates):
+        """
+        Gather the total USD of specified aggregators.
+        """
+        total_run_tasks = [self.nordigen(x) for x in aggregators]
+        total_run = await asyncio.gather(*total_run_tasks)
+
+        total_map_tasks = [x.get_total_map() for x in total_run]
+        total_map = await asyncio.gather(*total_map_tasks)
+
+        to_usd_tasks = [self.multiple_to_usd(x, rates=rates) for x in total_map]
+        total = await asyncio.gather(*to_usd_tasks)
+
+        total = sum(total)
+
+        return total
+
+    async def gather_wallet_balance_xmr(self, platforms):
+        """
+        Gather the total XMR of the specified platforms.
+        """
+        run_tasks = [self.agora(platform) for platform in platforms]
+        run = await asyncio.gather(*run_tasks)
+
+        xmr_tasks = [x.api.wallet_balance_xmr() for x in run]
+        xmr_pre = await asyncio.gather(*xmr_tasks)
+
+        xmr = [float(x["response"]["data"]["total"]["balance"]) for x in xmr_pre]
+        xmr = sum(xmr)
+        return xmr
+
+    async def gather_wallet_balance(self, platforms):
+        """
+        Gather the total BTC of the specified platforms.
+        """
+        run_tasks = [self.agora(platform) for platform in platforms]
+        run = await asyncio.gather(*run_tasks)
+
+        btc_tasks = [x.api.wallet_balance() for x in run]
+        btc_pre = await asyncio.gather(*btc_tasks)
+
+        btc = [float(x["response"]["data"]["total"]["balance"]) for x in btc_pre]
+        btc = sum(btc)
+        return btc
+
+    def gather_base_usd(self, platforms):
+        total = 0
+        for platform in platforms:
+            total += platform.base_usd
+        return total
+
+    def gather_withdrawal_limit(self, platforms):
+        total = 0
+        for platform in platforms:
+            total += platform.withdrawal_trigger
+        return total
+
+    # TODO: possibly refactor this into smaller functions which don't return as much
+    # check if this is all really needed in the corresponding withdraw function
+    async def get_total(self, aggregators, platforms, trades=False):
+        """
+        Get all the values corresponding to the amount of money we hold.
+        :return: ((total SEK, total USD, total GBP),
+            (total XMR USD, total BTC USD),
+            (total XMR, total BTC))
+        :rtype: tuple(tuple(float, float, float),
+            tuple(float, float),
+            tuple(float, float))
+        """
+        rates = await self.get_rates_all()
+        total_sinks_usd = await self.gather_total_map(aggregators, rates=rates)
+        agora_wallet_xmr = await self.gather_wallet_balance_xmr(platforms)
+        agora_wallet_btc = await self.gather_wallet_balance(platforms)
+
+        total_xmr_agora = agora_wallet_xmr
+        total_btc_agora = agora_wallet_btc
+        # Get the XMR -> USD exchange rate
+        async with AsyncCoinGeckoAPISession() as cg:
+            xmr_usd = await cg.get_price(ids="monero", vs_currencies="USD")
+
+            # Get the BTC -> USD exchange rate
+            btc_usd = await cg.get_price(ids="bitcoin", vs_currencies="USD")
+
+        # Convert the Agora XMR total to USD
+        total_usd_agora_xmr = float(total_xmr_agora) * xmr_usd["monero"]["usd"]
+
+        # Convert the Agora BTC total to USD
+        total_usd_agora_btc = float(total_btc_agora) * btc_usd["bitcoin"]["usd"]
+
+        # Add it all up
+        total_usd_agora = total_usd_agora_xmr + total_usd_agora_btc
+        total_usd = total_usd_agora + total_sinks_usd
+
+        # Get aggregate totals and withdrawal limits
+        total_base_usd = self.gather_base_usd(platforms)
+        total_withdrawal_limit = self.gather_withdrawal_limit(platforms)
+
+        # Use those to calculate amount remaining
+        withdraw_threshold = total_base_usd + total_withdrawal_limit
+        remaining = withdraw_threshold - total_usd
+
+        profit = total_usd - total_base_usd
+        profit_in_xmr = profit / xmr_usd["monero"]["usd"]
+
+        # Convert the total USD price to GBP and SEK
+        price_sek = rates["SEK"] * total_usd
+        price_usd = total_usd
+        price_gbp = rates["GBP"] * total_usd
+
+        # Get open trades value
+        if trades:
+            dashboards = await self.gather_dashboards(platforms)
+            cumul_trades = 0
+            for dash in dashboards:
+                cumul_add = await self.open_trades_usd_parse_dash(dash, rates)
+                cumul_trades += cumul_add
+
+            total_with_trades = total_usd + cumul_trades
+            total_remaining = withdraw_threshold - total_with_trades
+
+            total_profit = total_with_trades - total_base_usd
+            total_profit_in_xmr = total_profit / xmr_usd["monero"]["usd"]
+
+        # cast = (
+        #     (
+        #         price_sek,
+        #         price_usd,
+        #         price_gbp,
+        #     ),  # Total prices in our 3 favourite currencies
+        #     (
+        #         total_xmr_usd,
+        #         total_btc_usd,
+        #     ),  # Total USD balance in only Agora
+        #     (total_xmr, total_btc),
+        # )  # Total XMR and BTC balance in Agora
+
+        # TODO
+        cast_es = {
+            "price_sek": price_sek,
+            "price_usd": price_usd,
+            "price_gbp": price_gbp,
+            "total_usd_agora_xmr": total_usd_agora_xmr,
+            "total_usd_agora_btc": total_usd_agora_btc,
+            # "total_usd_lbtc_btc": total_usd_lbtc_btc,
+            "total_xmr_agora": total_xmr_agora,
+            "total_btc_agora": total_btc_agora,
+            # "total_btc_lbtc": total_btc_lbtc,
+            "xmr_usd": xmr_usd["monero"]["usd"],
+            "btc_usd": btc_usd["bitcoin"]["usd"],
+            "total_sinks_usd": total_sinks_usd,
+            "total_usd_agora": total_usd_agora,
+            "total_usd": total_usd,
+            "total_base_usd": total_base_usd,
+            "total_withdrawal_limit": total_withdrawal_limit,
+            "remaining": remaining,
+            "profit": profit,
+            "profit_in_xmr": profit_in_xmr,
+            "withdraw_threshold": withdraw_threshold,
+        }
+        if trades:
+            cast_es["open_trade_value"] = cumul_trades
+            cast_es["total_with_trades"] = total_with_trades
+            cast_es["total_remaining"] = total_remaining
+            cast_es["total_profit"] = total_profit
+            cast_es["total_profit_in_xmr"] = total_profit_in_xmr
+        await self.write_to_es("get_total", cast_es)
+        return cast_es
+
+    async def open_trades_usd_parse_dash(self, dash, rates):
+        cumul_usd = 0
+        cache = {}
+        async with AsyncCoinGeckoAPISession() as cg:
+            for _, contact in dash.items():
+                # We need created at in order to look up the historical prices
+                created_at = contact["data"]["created_at"]
+
+                # Reformat the date how CoinGecko likes
+                # 2022-05-02T11:17:14+00:00
+                if "+" in created_at:
+                    date_split = created_at.split("+")
+                    date_split[1].replace(".", "")
+                    date_split[1].replace(":", "")
+                    created_at = "+".join(date_split)
+                    date_parsed = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S%z")
+                else:
+                    date_parsed = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ")
+
+                date_formatted = date_parsed.strftime("%d-%m-%Y")
+
+                # Get the historical rates for the right asset, extract the price
+                asset = contact["data"]["advertisement"]["asset"]
+
+                if asset == "XMR":
+                    amount_crypto = contact["data"]["amount_xmr"]
+                    if (asset, date_formatted) in cache:
+                        history = cache[(asset, date_formatted)]
+                    else:
+                        history = await cg.get_coin_history_by_id(
+                            coin_id="monero", date=date_formatted
+                        )
+                        if "market_data" not in history:
+                            return False
+                    crypto_usd = float(history["market_data"]["current_price"]["usd"])
+                elif asset == "BTC":
+                    amount_crypto = contact["data"]["amount_btc"]
+                    if (asset, date_formatted) in cache:
+                        history = cache[(asset, date_formatted)]
+                    else:
+                        history = await cg.get_coin_history_by_id(
+                            coin_id="bitcoin", date=date_formatted
+                        )
+                        if "market_data" not in history:
+                            return False
+                    crypto_usd = float(history["market_data"]["current_price"]["usd"])
+                if (asset, date_formatted) not in cache:
+                    cache[(asset, date_formatted)] = history
+                # Convert crypto to USD
+                amount = float(amount_crypto) * crypto_usd
+                # currency = contact["data"]["currency"]
+                if not contact["data"]["is_selling"]:
+                    continue
+                cumul_usd += float(amount)
+                # else:
+                #     rate = rates[currency]
+                #     print("RATE", rate)
+                #     print("AMOUNT", amount)
+                #     amount_usd = float(amount) / rate
+                #     print("AMOUJT USD", amount_usd)
+                #     cumul_usd += amount_usd
+        return cumul_usd
+
+    async def gather_dashboards(self, platforms):
+        dashboards = []
+        for platform in platforms:
+            run = await self.agora(platform)
+            dash = await run.wrap_dashboard()
+            dashboards.append(dash)
+        return dashboards
+
+    # async def get_open_trades_usd(self, rates):
+    #     """
+    #     Get total value of open trades in USD.
+    #     :return: total trade value
+    #     :rtype: float
+    #     """
+    #     dash_agora = await self.agora.wrap_dashboard()
+    #     # dash_lbtc = self.lbtc.wrap_dashboard()
+    #     # dash_lbtc = yield dash_lbtc
+    #     if dash_agora is False:
+    #         return False
+    #     # if dash_lbtc is False:
+    #     #     return False
+
+    #     # rates = await self.get_rates_all()
+    #     cumul_usd_agora = await self.open_trades_usd_parse_dash(
+    #         "agora", dash_agora, rates
+    #     )
+    #     # cumul_usd_lbtc = await self.open_trades_usd_parse_dash("lbtc", dash_lbtc,
+    #     # rates)
+    #     cumul_usd = cumul_usd_agora  # + cumul_usd_lbtc
+
+    #     cast_es = {
+    #         "trades_usd": cumul_usd,
+    #     }
+    #     await self.write_to_es("get_open_trades_usd", cast_es)
+    #     return cumul_usd
+
+    async def get_total_remaining(self):
+        """
+        Check how much profit we need to make in order to withdraw, taking into account
+         open trade value.
+        :return: profit remaining in USD
+        :rtype: float
+        """
+        total_usd = await self.get_total_usd()
+        total_trades_usd = await self.get_open_trades_usd()
+        if not total_usd:
+            return False
+        total_usd += total_trades_usd
+        withdraw_threshold = float(settings.Money.BaseUSD) + float(
+            settings.Money.WithdrawLimit
+        )
+        remaining = withdraw_threshold - total_usd
+
+        cast_es = {
+            "total_remaining_usd": remaining,
+        }
+        await self.write_to_es("get_total_remaining", cast_es)
+        return remaining
+
+    async def get_total_with_trades(self):
+        total_usd = await self.get_total_usd()
+        if not total_usd:
+            return False
+        total_trades_usd = await self.get_open_trades_usd()
+        total_with_trades = total_usd + total_trades_usd
+        cast_es = {
+            "total_with_trades": total_with_trades,
+        }
+        await self.write_to_es("get_total_with_trades", cast_es)
+        return total_with_trades
+
+    def get_pay_list(self, linkgroup, requisitions, platforms, user, profit):
+        pay_list = {}  # Wallet: [(amount, reason), (amount, reason), ...]
+
+        # Get the total amount of money we have
+        total_throughput_platform = 0
+        total_throughput_requisition = 0
+        for requisition in requisitions:
+            total_throughput_requisition += requisition.throughput
+        for platform in platforms:
+            total_throughput_platform += platform.throughput
+
+        cut_platform = profit * (linkgroup.platform_owner_cut_percentage / 100)
+        cut_req = profit * (linkgroup.requisition_owner_cut_percentage / 100)
+        cut_operator = profit * (linkgroup.operator_cut_percentage / 100)
+
+        # Add the operator payment
+        operator_wallets = OperatorWallets.objects.filter(user=user).first()
+        operator_length = len(operator_wallets.payees.all())
+        payment_per_operator = cut_operator / operator_length
+        for wallet in operator_wallets.payees.all():
+            if wallet not in pay_list:
+                pay_list[wallet] = []
+            detail = (
+                f"Operator cut for 1 of {operator_length} operators, total "
+                f"{cut_operator}"
+            )
+            pay_list[wallet].append((payment_per_operator, detail))
+
+        # Add the platform payment
+        for platform in platforms:
+            # Get ratio of platform.throughput to the total platform throughput
+            if total_throughput_platform == 0:
+                ratio = 0
+            else:
+                ratio = platform.throughput / total_throughput_platform
+            platform_payment = cut_platform * ratio
+            payees_length = len(platform.payees.all())
+            if payees_length == 0:
+                payment_per_payee = 0
+            else:
+                payment_per_payee = platform_payment / payees_length
+            for wallet in platform.payees.all():
+                if wallet not in pay_list:
+                    pay_list[wallet] = []
+                detail = (
+                    f"Platform {platform} cut for 1 of {payees_length} payees, "
+                    f"total {cut_platform}"
+                )
+                pay_list[wallet].append((payment_per_payee, detail))
+
+        # Add the requisition payment
+        for requisition in requisitions:
+            # Get ratio of requisition.throughput to the requisition cut
+            if total_throughput_requisition == 0:
+                ratio = 0
+            else:
+                ratio = requisition.throughput / total_throughput_requisition
+            req_payment = cut_req * ratio
+            payees_length = len(requisition.payees.all())
+            if payees_length == 0:
+                payment_per_payee = 0
+            else:
+                payment_per_payee = req_payment / payees_length
+            for wallet in requisition.payees.all():
+                if wallet not in pay_list:
+                    pay_list[wallet] = []
+                detail = (
+                    f"Requisition {requisition} cut for 1 of {payees_length} payees, "
+                    f"total {cut_req}"
+                )
+                pay_list[wallet].append((payment_per_payee, detail))
+
+        return pay_list
+
+    def collapse_pay_list(self, pay_list):
+        """
+        Collapse the pay list into a single dict of wallet: amount.
+        """
+        collapsed = {}
+        for wallet, payments in pay_list.items():
+            collapsed[wallet] = sum([x[0] for x in payments])
+
+        return collapsed
+
+
+money = Money()

core/lib/notify.py

@@ -0,0 +1,39 @@
+import aiohttp
+
+from core.util import logs
+
+NTFY_URL = "https://ntfy.sh"
+
+log = logs.get_logger(__name__)
+
+
+# Actual function to send a message to a topic
+async def raw_sendmsg(msg, title=None, priority=None, tags=None, url=None, topic=None):
+    if url is None:
+        url = NTFY_URL
+    headers = {"Title": "Pluto"}
+    if title:
+        headers["Title"] = title
+    if priority:
+        headers["Priority"] = priority
+    if tags:
+        headers["Tags"] = tags
+    cast = {
+        "headers": headers,
+        "data": msg,
+    }
+    async with aiohttp.ClientSession() as session:
+        await session.post(f"{url}/{topic}", **cast)
+
+
+# Sendmsg helper to send a message to a user's notification settings
+async def sendmsg(user, *args, **kwargs):
+    notification_settings = user.get_notification_settings()
+
+    if notification_settings.ntfy_topic is None:
+        # No topic set, so don't send
+        return
+    else:
+        topic = notification_settings.ntfy_topic
+
+    await raw_sendmsg(*args, **kwargs, url=notification_settings.ntfy_url, topic=topic)

core/lib/schemas/__init__.py

@@ -0,0 +1,2 @@
+from core.lib.schemas import agora_s  # noqa
+from core.lib.schemas import nordigen_s  # noqa

core/lib/schemas/agora_s.py

@@ -0,0 +1,298 @@
+from pydantic import BaseModel, Extra
+
+
+class MyModel(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+
+class ContactDataBuyerSeller(MyModel):
+    username: str
+    name: str
+    feedback_score: int
+    trade_count: str
+    last_online: str
+
+
+class ContactDataAd(MyModel):
+    payment_method: str
+    trade_type: str
+    advertiser: ContactDataBuyerSeller
+    asset: str
+    id: str | None
+    contact_id: str | None
+
+
+class ContactData(MyModel):
+    buyer: ContactDataBuyerSeller
+    seller: ContactDataBuyerSeller
+    amount: str
+    amount_xmr: str | None
+    fee_xmr: str | None
+    amount_btc: str | None
+    fee_btc: str | None
+    advertisement: ContactDataAd
+    contact_id: str
+    currency: str
+    country: str
+    account_info: str
+    price_equation: str
+    is_buying: bool
+    is_selling: bool
+    created_at: str
+    escrowed_at: str
+    funded_at: str
+    canceled_at: str | None
+    closed_at: str | None
+    msg: str
+    released_at: str | None
+    payment_completed_at: str | None
+    disputed_at: str | None
+    arbitrated: bool
+    transfer_to_seller_non_custodial_wallet_transaction_confirmations: str | None
+    transfer_to_buyer_settlement_wallet_transaction_id: str | None
+    transfer_to_buyer_settlement_wallet_transaction_key: str | None
+    buyer_settlement_address: str | None
+    buyer_settlement_fee_level: str | None
+    seller_non_custodial_wallet_mnemonic: str | None
+    transfer_to_seller_non_custodial_wallet_transaction_id: str | None
+
+
+class ContactActions(MyModel):
+    advertisement_public_view: str | None
+    advertisement_url: str | None
+    message_post_url: str | None
+    messages_url: str | None
+    release_url: str | None
+    cancel_url: str | None
+    dispute_url: str | None
+
+
+class Contact(MyModel):
+    data: ContactData
+    actions: ContactActions
+
+
+class DashboardResponseData(MyModel):
+    contact_count: int
+    contact_list: list[Contact]
+
+
+class DashboardResponse(MyModel):
+    data: DashboardResponseData
+
+
+class Dashboard(MyModel):
+    success: bool
+    status: int | None
+    message: str
+    response: DashboardResponse
+
+
+DashboardSchema = {
+    "success": "success",
+    "message": "message",
+    "contact_count": "response.data.contact_count",
+    "contact_list": "response.data.contact_list",
+}
+
+
+class Pagination(MyModel):
+    prev: str | None
+    next: str | None
+    total_elements: int
+    total_pages: int
+    current_page: int
+
+
+class Profile(MyModel):
+    username: str
+    name: str
+    feedback_score: int
+    trade_count: str
+    last_online: str
+    localbitcoins_trade_count: int | None
+    paxful_trade_count: int | None
+
+
+class BuyBitcoinsOnlineAd(MyModel):
+    ad_id: str
+    countrycode: str
+    created_at: str
+    currency: str
+    max_amount: float | None
+    max_amount_available: float
+    min_amount: float | None
+    msg: str
+    online_provider: str
+    require_trusted_by_advertiser: bool
+    verified_email_required: bool
+    temp_price: float
+    track_max_amount: bool
+    trade_type: str
+    trusted_required: bool
+    visible: bool
+    asset: str
+    payment_method_detail: str | None
+    profile: Profile
+    require_feedback_score: int | None
+    first_time_limit_btc: float | None
+    limit_to_fiat_amounts: str | None
+
+
+class Actions(MyModel):
+    public_view: str
+
+
+class BuyBitcoinsOnlineResponseDataAdList(MyModel):
+    data: BuyBitcoinsOnlineAd
+    actions: Actions
+
+
+class BuyBitcoinsOnlineResponseData(MyModel):
+    ad_count: int
+    ad_list: list[BuyBitcoinsOnlineResponseDataAdList]
+
+
+class BuyBitcoinsOnlineResponse(MyModel):
+    data: BuyBitcoinsOnlineResponseData
+    pagination: Pagination | None
+
+
+class BuyBitcoinsOnline(MyModel):
+    success: bool
+    message: str
+    response: BuyBitcoinsOnlineResponse
+    status: int | None
+
+
+class BuyMoneroOnlineAd(MyModel):
+    ad_id: str
+    countrycode: str
+    created_at: str
+    currency: str
+    max_amount: float | None
+    max_amount_available: float
+    min_amount: float | None
+    msg: str
+    online_provider: str
+    require_trusted_by_advertiser: bool
+    verified_email_required: bool
+    temp_price: float
+    track_max_amount: bool
+    trade_type: str
+    trusted_required: bool
+    visible: bool
+    asset: str
+    payment_method_detail: str | None
+    profile: Profile
+    require_feedback_score: int | None
+    first_time_limit_xmr: float | None
+    limit_to_fiat_amounts: str | None
+
+
+class BuyMoneroOnlineAdList(MyModel):
+    data: BuyMoneroOnlineAd
+    actions: Actions
+
+
+class BuyMoneroOnlineResponseData(MyModel):
+    ad_count: int
+    ad_list: list[BuyMoneroOnlineAdList]
+
+
+class BuyMoneroOnlineResponse(MyModel):
+    data: BuyMoneroOnlineResponseData
+    pagination: Pagination | None
+
+
+class BuyMoneroOnline(MyModel):
+    success: bool
+    message: str
+    response: BuyMoneroOnlineResponse
+    status: int | None
+
+
+BuyBitcoinsOnlineSchema = {
+    "success": "success",
+    "message": "message",
+    "ad_count": "response.data.ad_count",
+    "ad_list": "response.data.ad_list",
+    "pagination": "response.pagination",
+}
+
+BuyMoneroOnlineSchema = {
+    "success": "success",
+    "message": "message",
+    "ad_count": "response.data.ad_count",
+    "ad_list": "response.data.ad_list",
+    "pagination": "response.pagination",
+}
+
+
+class AccountInfo(MyModel):
+    ...
+
+
+class AdsResponseDataAd(MyModel):
+    ad_id: str
+    countrycode: str
+    created_at: str
+    currency: str
+    max_amount: float | None
+    max_amount_available: float
+    min_amount: float | None
+    msg: str
+    online_provider: str
+    require_trusted_by_advertiser: bool
+    verified_email_required: bool
+    temp_price: float
+    track_max_amount: bool
+    trade_type: str
+    trusted_required: bool
+    visible: bool
+    asset: str
+    payment_method_detail: str | None
+    require_feedback_score: int | None
+    first_time_limit_xmr: float | None
+    first_time_limit_btc: float | None
+    limit_to_fiat_amounts: str | None
+    account_info: str
+    price_equation: str
+
+
+class AdsActions(MyModel):
+    change_form: str | None
+    html_form: str | None
+    public_view: str | None
+
+
+class AdsResponseDataAdList(MyModel):
+    data: AdsResponseDataAd
+    actions: AdsActions
+
+
+class AdsResponseData(MyModel):
+    ad_count: int
+    ad_list: list[AdsResponseDataAdList]
+
+
+class AdsResponse(MyModel):
+    data: AdsResponseData
+    pagination: Pagination | None
+
+
+class Ads(MyModel):
+    success: bool
+    message: str
+    response: AdsResponse
+    status: int | None
+
+
+AdsSchema = {
+    "success": "success",
+    "message": "message",
+    "ad_count": "response.data.ad_count",
+    "ad_list": "response.data.ad_list",
+    "pagination": "response.pagination",
+}

core/lib/schemas/nordigen_s.py

@@ -0,0 +1,213 @@
+from pydantic import BaseModel, Extra
+
+
+class MyModel(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+
+# TODO: inherit from MyModel
+
+
+class TokenNew(MyModel):
+    access: str
+    access_expires: int
+    refresh: str
+    refresh_expires: int
+
+
+TokenNewSchema = {
+    "access": "access",
+    "access_expires": "access_expires",
+    "refresh": "refresh",
+    "refresh_expires": "refresh_expires",
+}
+
+
+class RequisitionResult(MyModel):
+    id: str
+    created: str
+    redirect: str
+    status: str
+    institution_id: str
+    agreement: str
+    reference: str
+    accounts: list[str]
+    link: str
+    ssn: str | None
+    account_selection: bool
+    redirect_immediate: bool
+
+
+class Requisitions(MyModel):
+    count: int
+    next: str | None
+    previous: str | None
+    results: list[RequisitionResult]
+
+
+RequisitionsSchema = {
+    "count": "count",
+    "next": "next",
+    "previous": "previous",
+    "results": "results",
+}
+
+
+class RequisitionsPost(MyModel):
+    id: str
+    created: str
+    redirect: str
+    status: str
+    institution_id: str
+    agreement: str
+    reference: str
+    accounts: list[str]
+    link: str
+    ssn: str | None
+    account_selection: bool
+    redirect_immediate: bool
+
+
+RequisitionsPostSchema = {
+    "id": "id",
+    "created": "created",
+    "redirect": "redirect",
+    "status": "status",
+    "institution_id": "institution_id",
+    "agreement": "agreement",
+    "reference": "reference",
+    "accounts": "accounts",
+    "link": "link",
+    "ssn": "ssn",
+    "account_selection": "account_selection",
+    "redirect_immediate": "redirect_immediate",
+}
+
+
+class Requisition(MyModel):
+    id: str
+    created: str
+    redirect: str
+    status: str
+    institution_id: str
+    agreement: str
+    reference: str
+    accounts: list[str]
+    link: str
+    ssn: str | None
+    account_selection: bool
+    redirect_immediate: bool
+
+
+RequisitionSchema = {
+    "id": "id",
+    "created": "created",
+    "redirect": "redirect",
+    "status": "status",
+    "institution_id": "institution_id",
+    "agreement": "agreement",
+    "reference": "reference",
+    "accounts": "accounts",
+    "link": "link",
+    "ssn": "ssn",
+    "account_selection": "account_selection",
+    "redirect_immediate": "redirect_immediate",
+}
+
+
+class AccountDetailsNested(MyModel):
+    resourceId: str
+    currency: str
+    ownerName: str | None
+    cashAccountType: str | None
+    status: str | None
+    maskedPan: str | None
+    details: str | None
+    iban: str | None
+    bban: str | None
+    name: str | None
+    product: str | None
+    bic: str | None
+
+
+class AccountDetails(MyModel):
+    account: AccountDetailsNested
+
+
+AccountDetailsSchema = {
+    "account": "account",
+}
+
+
+class AccountBalance(MyModel):
+    balanceAmount: dict[str, str]
+    balanceType: str | None
+    referenceDate: str | None
+
+
+class AccountBalances(MyModel):
+    balances: list[AccountBalance]
+    summary: str | None
+
+
+AccountBalancesSchema = {
+    "balances": "balances",
+    "summary": "summary",
+}
+
+
+class TXCurrencyAmount(MyModel):
+    amount: str
+    currency: str
+
+
+class TransactionsCurrencyExchange(MyModel):
+    instructedAmount: TXCurrencyAmount
+    sourceCurrency: str
+    exchangeRate: str
+    unitCurrency: str
+    targetCurrency: str
+
+
+class TXAccount(MyModel):
+    iban: str | None
+    bban: str | None
+
+
+class TransactionsNested(MyModel):
+    transactionId: str | None
+    bookingDate: str | None
+    valueDate: str | None
+    bookingDateTime: str | None
+    valueDateTime: str | None
+    transactionAmount: TXCurrencyAmount
+    creditorName: str | None
+    creditorAccount: TXAccount | None
+    debtorName: str | None
+    debtorAccount: TXAccount | None
+    remittanceInformationUnstructuredArray: list[str] | None
+    remittanceInformationUnstructured: str | None
+    proprietaryBankTransactionCode: str | None
+    internalTransactionId: str | None
+    currencyExchange: TransactionsCurrencyExchange | None
+    merchantCategoryCode: str | None
+
+
+class TransactionsBookedPending(MyModel):
+    booked: list[TransactionsNested] | None
+    pending: list[TransactionsNested] | None
+
+
+class Transactions(MyModel):
+    transactions: TransactionsBookedPending | None
+    detail: str | None
+    status_code: int | None
+    summary: str | None
+
+
+TransactionsSchema = {
+    "transactions": "transactions",
+    "status_code": "status_code",
+    "summary": "summary",
+}

core/management/commands/polling.py

@@ -0,0 +1,56 @@
+import asyncio
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from django.core.management.base import BaseCommand
+
+# from core.clients.aggregators.nordigen import NordigenClient
+from core.clients.platforms.agora import AgoraClient
+from core.models import Aggregator, Platform
+from core.util import logs
+
+log = logs.get_logger("polling")
+
+INTERVAL = 5
+
+
+async def poll_aggregator(aggregator):
+    pass
+
+
+async def poll_platform(platform):
+    client = await AgoraClient(platform)
+    await client.poll()
+
+
+async def job():
+    platforms = Platform.objects.filter(enabled=True)
+    aggregators = Aggregator.objects.filter(enabled=True)
+
+    tasks = []
+    for platform in platforms:
+        tasks.append(poll_platform(platform))
+
+    for aggregator in aggregators:
+        tasks.append(poll_aggregator(aggregator))
+
+    # Run it all at once
+    await asyncio.gather(*tasks)
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        """
+        Start the polling process.
+        """
+        scheduler = AsyncIOScheduler()
+
+        log.debug(f"Scheduling polling process job every {INTERVAL} seconds")
+        scheduler.add_job(job, "interval", seconds=INTERVAL)
+        scheduler.start()
+        loop = asyncio.get_event_loop()
+        try:
+            loop.run_forever()
+        except (KeyboardInterrupt, SystemExit):
+            log.info("Process terminating")
+        finally:
+            loop.close()

core/management/commands/scheduling.py

@@ -0,0 +1,228 @@
+import asyncio
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from pyotp import TOTP
+
+from core.clients.aggregators.nordigen import NordigenClient
+from core.clients.platforms.agora import AgoraClient
+from core.lib.money import Money
+from core.lib.notify import sendmsg
+from core.models import (
+    INTERVAL_CHOICES,
+    Aggregator,
+    LinkGroup,
+    Payout,
+    Platform,
+    Requisition,
+)
+from core.util import logs
+from core.util.validation import Validation
+
+log = logs.get_logger("scheduling")
+
+INTERVAL_AGGREGATOR = 10
+INTERVAL_WITHDRAWAL = 7200
+
+INTERVALS_PLATFORM = [x[0] for x in INTERVAL_CHOICES]
+
+
+async def withdrawal_job(group=None):
+    money = Money()
+    if group is not None:
+        groups = [group]
+    else:
+        groups = LinkGroup.objects.filter(enabled=True)
+    for group in groups:
+        checks = await money.check_all(
+            link_group=group, nordigen=NordigenClient, agora=AgoraClient
+        )
+
+        if checks["total_remaining"] > 0:
+            # More than 0 remaining, so we can't withdraw
+            await sendmsg(
+                group.user,
+                f"{checks['total_remaining']} left until you can withdraw.",
+                title="Balance update",
+            )
+            continue
+
+        print("CHECKS", checks)
+        aggregators = Aggregator.objects.filter(
+            user=group.user,
+            link_group=group,
+        )
+        platforms = Platform.objects.filter(
+            user=group.user,
+            link_group=group,
+        )
+        requisitions = Requisition.objects.filter(
+            user=group.user,
+            aggregator__in=aggregators,
+        )
+        pay_list = money.get_pay_list(
+            group,
+            requisitions,
+            platforms,
+            group.user,
+            checks["total_profit_in_xmr"],
+        )
+        collapsed = money.collapse_pay_list(pay_list)
+        if any(collapsed.values()):
+            message = ""
+            print("COLLAPSED", collapsed)
+            for wallet, amount in collapsed.items():
+                print("ITER", wallet, amount)
+                message += f"{wallet}: {amount}\n"
+            print("MESSAGE", message)
+            await sendmsg(
+                group.user,
+                message,
+                title="Your withdrawal is ready!",
+            )
+
+            # TODO: UNCOMMENT
+            # COMMENTED FOR TESTING
+            if not checks["total_profit_in_xmr"] >= 0:
+                return
+
+            total_withdrawal = sum(collapsed.values())
+            if checks["total_xmr_agora"] < total_withdrawal:
+                await sendmsg(
+                    group.user,
+                    (
+                        f"Attempting to withdraw {total_withdrawal}, but you only have"
+                        f" {checks['total_xmr_agora']} in your Agora wallet."
+                    ),
+                    title="Withdrawal failed",
+                )
+                continue
+            if group.platforms.count() != 1:
+                raise Exception("You can only have one platform per group")
+
+            platform = group.platforms.first()
+            run = await AgoraClient(platform)
+
+            for wallet, pay_list_iter in pay_list.items():
+                print("WALLET ITER", wallet)
+                if not Validation.is_address("xmr", wallet.address):
+                    print("NOT VALID", wallet.address)
+                    await sendmsg(
+                        group.user,
+                        f"Invalid XMR address: {wallet.address}, ignored",
+                        title="Invalid XMR address",
+                    )
+                    continue
+
+                for amount, reason in pay_list_iter:
+                    await asyncio.sleep(31)
+                    print("ITER", wallet, pay_list_iter)
+                    print("ITER SENT", wallet, amount, reason)
+
+                    # for wallet, amount in collapsed.items():
+                    print("ITER SEND", wallet, amount)
+                    amount_rounded = round(amount, 8)
+                    cast = {
+                        "address": wallet.address,
+                        "amount": amount_rounded,
+                        "password": platform.password,
+                        "otp": TOTP(platform.otp_token).now(),
+                    }
+
+                    print("CAST ADDRESS", cast["address"])
+                    print("CAST AMOUNT", cast["amount"])
+                    print("CAST OTP TRUNCATED BY 2", cast["otp"][-2])
+                    if not settings.DUMMY:
+                        sent = await run.call("wallet_send_xmr", **cast)
+                        print("SENT", sent)
+
+                    payout = Payout.objects.create(  # noqa
+                        user=group.user,
+                        wallet=wallet,
+                        amount=amount_rounded,
+                        description=reason,
+                    )
+
+                    if not settings.DUMMY:
+                        payout.response = sent
+                        payout.save()
+
+
+async def aggregator_job():
+    aggregators = Aggregator.objects.filter(enabled=True)
+    for aggregator in aggregators:
+        open_trade_currencies = aggregator.trades_currencies
+        if aggregator.service == "nordigen":
+            instance = None
+            if aggregator.fetch_accounts is True:
+                aggregator.account_info = {}
+                aggregator.save()
+                instance = await NordigenClient(aggregator)
+                await instance.get_all_account_info(store=True)
+
+            # fetch_tasks = []
+            for bank, accounts in aggregator.account_info.items():
+                for account in accounts:
+                    account_id = account["account_id"]
+                    requisition_id = account["requisition_id"]
+                    if account["currency"] not in open_trade_currencies:
+                        continue  # Next account
+                    # Avoid hammering the API with new access token requests
+                    if instance is None:
+                        instance = await NordigenClient(aggregator)
+                    # task = instance.get_transactions(
+                    #     account_id, req=requisition_id, process=True
+                    # )
+                    await instance.get_transactions(
+                        account_id, req=requisition_id, process=True
+                    )
+            #         fetch_tasks.append(task)
+            # await asyncio.gather(*fetch_tasks)
+        else:
+            raise NotImplementedError(f"No such client library: {aggregator.service}")
+        aggregator.fetch_accounts = False
+        aggregator.save()
+
+
+async def platform_job(interval):
+    if interval == 0:
+        return
+    platforms = Platform.objects.filter(enabled=True, cheat_interval_seconds=interval)
+    for platform in platforms:
+        if platform.service == "agora":
+            if platform.cheat is True:
+                instance = await AgoraClient(platform)
+                await instance.cheat()
+        else:
+            raise NotImplementedError(f"No such client library: {platform.service}")
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        """
+        Start the scheduling process.
+        """
+        scheduler = AsyncIOScheduler()
+
+        log.debug(f"Scheduling {INTERVAL_AGGREGATOR} second aggregator job")
+        scheduler.add_job(aggregator_job, "interval", seconds=INTERVAL_AGGREGATOR)
+
+        log.debug(f"Scheduling {INTERVAL_WITHDRAWAL} second withdrawal job")
+        scheduler.add_job(withdrawal_job, "interval", seconds=INTERVAL_WITHDRAWAL)
+
+        for interval in INTERVALS_PLATFORM:
+            if interval == 0:
+                continue
+            log.debug(f"Scheduling {interval} second platform job")
+            scheduler.add_job(
+                platform_job, "interval", seconds=interval, args=[interval]
+            )
+        scheduler.start()
+        loop = asyncio.get_event_loop()
+        try:
+            loop.run_forever()
+        except (KeyboardInterrupt, SystemExit):
+            log.info("Process terminating")
+        finally:
+            loop.close()

core/migrations/0001_initial.py

@@ -0,0 +1,59 @@
+# Generated by Django 4.1.7 on 2023-03-04 14:11
+
+import uuid
+
+import django.contrib.auth.models
+import django.contrib.auth.validators
+import django.db.models.deletion
+import django.utils.timezone
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('auth', '0012_alter_user_first_name_max_length'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='User',
+            fields=[
+                ('password', models.CharField(max_length=128, verbose_name='password')),
+                ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
+                ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
+                ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')),
+                ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')),
+                ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')),
+                ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')),
+                ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')),
+                ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')),
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('payment_provider_id', models.CharField(blank=True, max_length=255, null=True)),
+                ('billing_provider_id', models.CharField(blank=True, max_length=255, null=True)),
+                ('email', models.EmailField(max_length=254, unique=True)),
+                ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')),
+                ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')),
+            ],
+            options={
+                'verbose_name': 'user',
+                'verbose_name_plural': 'users',
+                'abstract': False,
+            },
+            managers=[
+                ('objects', django.contrib.auth.models.UserManager()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='NotificationSettings',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ntfy_topic', models.CharField(blank=True, max_length=255, null=True)),
+                ('ntfy_url', models.CharField(blank=True, max_length=255, null=True)),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0002_alter_notificationsettings_user_aggregator.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.7 on 2023-03-07 16:54
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='notificationsettings',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.CreateModel(
+            name='Aggregator',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('service', models.CharField(choices=[('nordigen', 'Nordigen')], max_length=255)),
+                ('secret_id', models.CharField(blank=True, max_length=1024, null=True)),
+                ('secret_key', models.CharField(blank=True, max_length=1024, null=True)),
+                ('access_token', models.CharField(blank=True, max_length=1024, null=True)),
+                ('poll_interval', models.IntegerField(default=10)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0003_aggregator_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-07 17:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_alter_notificationsettings_user_aggregator'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='aggregator',
+            name='enabled',
+            field=models.BooleanField(default=True),
+        ),
+    ]

core/migrations/0004_aggregator_access_token_expires.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-08 10:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0003_aggregator_enabled'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='aggregator',
+            name='access_token_expires',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

core/migrations/0005_aggregator_account_info_aggregator_currencies.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.7 on 2023-03-09 11:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0004_aggregator_access_token_expires'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='aggregator',
+            name='account_info',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AddField(
+            model_name='aggregator',
+            name='currencies',
+            field=models.JSONField(default=list),
+        ),
+    ]

core/migrations/0006_aggregator_fetch_accounts.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-09 11:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_aggregator_account_info_aggregator_currencies'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='aggregator',
+            name='fetch_accounts',
+            field=models.BooleanField(default=False),
+        ),
+    ]

core/migrations/0007_alter_aggregator_account_info_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.7 on 2023-03-09 14:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0006_aggregator_fetch_accounts'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='aggregator',
+            name='account_info',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name='aggregator',
+            name='fetch_accounts',
+            field=models.BooleanField(default=True),
+        ),
+    ]

core/migrations/0008_platform.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.1.7 on 2023-03-09 20:50
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0007_alter_aggregator_account_info_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Platform',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('service', models.CharField(choices=[('agora', 'Agora')], max_length=255)),
+                ('token', models.CharField(max_length=1024)),
+                ('password', models.CharField(max_length=1024)),
+                ('otp_token', models.CharField(blank=True, max_length=1024, null=True)),
+                ('username', models.CharField(max_length=255)),
+                ('send', models.BooleanField(default=True)),
+                ('cheat', models.BooleanField(default=False)),
+                ('dummy', models.BooleanField(default=False)),
+                ('cheat_interval_seconds', models.IntegerField(default=600)),
+                ('margin', models.FloatField(default=1.2)),
+                ('max_margin', models.FloatField(default=1.3)),
+                ('min_margin', models.FloatField(default=1.15)),
+                ('min_trade_size_usd', models.FloatField(default=10)),
+                ('max_trade_size_usd', models.FloatField(default=4000)),
+                ('accept_within_usd', models.FloatField(default=1)),
+                ('no_reference_amount_check_max_usd', models.FloatField(default=400)),
+                ('enabled', models.BooleanField(default=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0009_asset_provider_ad.py

@@ -0,0 +1,48 @@
+# Generated by Django 4.1.7 on 2023-03-10 00:41
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0008_platform'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Asset',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('code', models.CharField(max_length=64)),
+                ('name', models.CharField(max_length=255)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Provider',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('code', models.CharField(max_length=64)),
+                ('name', models.CharField(max_length=255)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Ad',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('text', models.TextField()),
+                ('payment_details', models.TextField()),
+                ('payment_details_real', models.TextField()),
+                ('payment_method_details', models.CharField(max_length=255)),
+                ('dist_list', models.TextField()),
+                ('asset_list', models.ManyToManyField(to='core.asset')),
+                ('provider_list', models.ManyToManyField(to='core.provider')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0010_ad_account_map_ad_aggregators_ad_enabled_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.1.7 on 2023-03-10 00:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_asset_provider_ad'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ad',
+            name='account_map',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name='ad',
+            name='aggregators',
+            field=models.ManyToManyField(to='core.aggregator'),
+        ),
+        migrations.AddField(
+            model_name='ad',
+            name='enabled',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='ad',
+            name='platforms',
+            field=models.ManyToManyField(to='core.platform'),
+        ),
+    ]

core/migrations/0011_ad_visible.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-10 02:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0010_ad_account_map_ad_aggregators_ad_enabled_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ad',
+            name='visible',
+            field=models.BooleanField(default=True),
+        ),
+    ]

core/migrations/0012_platform_last_messages.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-10 14:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0011_ad_visible'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='platform',
+            name='last_messages',
+            field=models.JSONField(default=dict),
+        ),
+    ]

core/migrations/0013_platform_platform_ad_ids.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-10 14:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0012_platform_last_messages'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='platform',
+            name='platform_ad_ids',
+            field=models.JSONField(default=dict),
+        ),
+    ]

core/migrations/0014_ad_account_whitelist.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-10 15:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_platform_platform_ad_ids'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ad',
+            name='account_whitelist',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

core/migrations/0015_platform_base_usd_platform_withdrawal_trigger.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.7 on 2023-03-10 15:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0014_ad_account_whitelist'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='platform',
+            name='base_usd',
+            field=models.FloatField(default=2800),
+        ),
+        migrations.AddField(
+            model_name='platform',
+            name='withdrawal_trigger',
+            field=models.FloatField(default=200),
+        ),
+    ]

core/migrations/0016_alter_platform_cheat_interval_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-11 17:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0015_platform_base_usd_platform_withdrawal_trigger'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='platform',
+            name='cheat_interval_seconds',
+            field=models.IntegerField(choices=[(60, 'Every minute'), (300, 'Every 5 minutes'), (600, 'Every 10 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=3600),
+        ),
+    ]

core/migrations/0017_alter_platform_cheat_interval_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-11 17:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0016_alter_platform_cheat_interval_seconds'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='platform',
+            name='cheat_interval_seconds',
+            field=models.IntegerField(choices=[(0, 'Never'), (5, 'Every 5 seconds'), (15, 'Every 15 seconds'), (30, 'Every 30 seconds'), (60, 'Every minute'), (300, 'Every 5 minutes'), (600, 'Every 10 minutes'), (3600, 'Every hour'), (14400, 'Every 4 hours'), (86400, 'Every day')], default=0),
+        ),
+    ]

core/migrations/0018_transaction_trade.py

@@ -0,0 +1,53 @@
+# Generated by Django 4.1.7 on 2023-03-12 12:18
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0017_alter_platform_cheat_interval_seconds'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Transaction',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('account_id', models.CharField(max_length=255)),
+                ('recipient', models.CharField(blank=True, max_length=255, null=True)),
+                ('sender', models.CharField(blank=True, max_length=255, null=True)),
+                ('amount', models.FloatField()),
+                ('currency', models.CharField(max_length=16)),
+                ('note', models.CharField(blank=True, max_length=255, null=True)),
+                ('reconciled', models.BooleanField(default=False)),
+                ('aggregator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.aggregator')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Trade',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('contact_id', models.CharField(max_length=255)),
+                ('reference', models.CharField(max_length=255)),
+                ('buyer', models.CharField(max_length=255)),
+                ('seller', models.CharField(max_length=255)),
+                ('amount_fiat', models.FloatField()),
+                ('currency', models.CharField(max_length=16)),
+                ('amount_crypto', models.FloatField()),
+                ('coin', models.CharField(max_length=16)),
+                ('provider', models.CharField(max_length=255)),
+                ('type', models.CharField(max_length=255)),
+                ('ad_id', models.CharField(max_length=255)),
+                ('status', models.CharField(max_length=255)),
+                ('reconciled', models.BooleanField(default=False)),
+                ('released', models.BooleanField(default=False)),
+                ('release_response', models.JSONField(default=dict)),
+                ('linked', models.ManyToManyField(to='core.transaction')),
+                ('platform', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.platform')),
+            ],
+        ),
+    ]

core/migrations/0019_remove_trade_status_trade_open.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.7 on 2023-03-12 12:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0018_transaction_trade'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='trade',
+            name='status',
+        ),
+        migrations.AddField(
+            model_name='trade',
+            name='open',
+            field=models.BooleanField(default=True),
+        ),
+    ]

core/migrations/0020_rename_coin_trade_asset_remove_trade_seller_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-03-12 12:38
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0019_remove_trade_status_trade_open'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='trade',
+            old_name='coin',
+            new_name='asset',
+        ),
+        migrations.RemoveField(
+            model_name='trade',
+            name='seller',
+        ),
+        migrations.RemoveField(
+            model_name='trade',
+            name='type',
+        ),
+    ]

core/migrations/0021_alter_trade_ad_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-12 12:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0020_rename_coin_trade_asset_remove_trade_seller_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='trade',
+            name='ad_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

core/migrations/0022_transaction_transaction_id_transaction_ts_added.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-03-12 19:21
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0021_alter_trade_ad_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='transaction_id',
+            field=models.CharField(default='NONE', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='transaction',
+            name='ts_added',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+    ]

core/migrations/0023_alter_trade_linked.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-13 09:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0022_transaction_transaction_id_transaction_ts_added'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='trade',
+            name='linked',
+            field=models.ManyToManyField(blank=True, to='core.transaction'),
+        ),
+    ]

core/migrations/0024_ad_send_reference.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-14 09:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0023_alter_trade_linked'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ad',
+            name='send_reference',
+            field=models.BooleanField(default=True),
+        ),
+    ]

core/migrations/0025_requisition.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-03-15 10:14
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0024_ad_send_reference'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Requisition',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('requisition_id', models.CharField(max_length=255)),
+                ('payment_details', models.TextField()),
+                ('transaction_source', models.CharField(choices=[('booked', 'Booked'), ('pending', 'Pending')], max_length=255)),
+                ('aggregator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.aggregator')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0026_alter_requisition_transaction_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-15 10:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0025_requisition'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='requisition',
+            name='transaction_source',
+            field=models.CharField(choices=[('booked', 'Booked'), ('pending', 'Pending')], default='booked', max_length=255),
+        ),
+    ]

core/migrations/0027_alter_requisition_payment_details.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-15 10:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0026_alter_requisition_transaction_source'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='requisition',
+            name='payment_details',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

core/migrations/0028_requisition_throughput_wallet_platform_payees_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.7 on 2023-03-17 18:15
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0027_alter_requisition_payment_details'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='requisition',
+            name='throughput',
+            field=models.FloatField(default=0),
+        ),
+        migrations.CreateModel(
+            name='Wallet',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('address', models.CharField(max_length=255)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='platform',
+            name='payees',
+            field=models.ManyToManyField(blank=True, to='core.wallet'),
+        ),
+        migrations.AddField(
+            model_name='requisition',
+            name='payees',
+            field=models.ManyToManyField(blank=True, to='core.wallet'),
+        ),
+    ]

core/migrations/0029_alter_requisition_id_alter_wallet_id.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.7 on 2023-03-17 18:31
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0028_requisition_throughput_wallet_platform_payees_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='requisition',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='wallet',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+    ]

core/migrations/0030_linkgroup.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.1.7 on 2023-03-18 10:12
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0029_alter_requisition_id_alter_wallet_id'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LinkGroup',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('platform_owner_cut_percentage', models.FloatField(default=0)),
+                ('requisition_owner_cut_percentage', models.FloatField(default=0)),
+                ('operator_cut_percentage', models.FloatField(default=0)),
+                ('enabled', models.BooleanField(default=True)),
+                ('aggregators', models.ManyToManyField(to='core.aggregator')),
+                ('platforms', models.ManyToManyField(to='core.platform')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0031_remove_ad_aggregators_remove_ad_platforms_and_more.py

@@ -0,0 +1,45 @@
+# Generated by Django 4.1.7 on 2023-03-18 10:38
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0030_linkgroup'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='ad',
+            name='aggregators',
+        ),
+        migrations.RemoveField(
+            model_name='ad',
+            name='platforms',
+        ),
+        migrations.RemoveField(
+            model_name='linkgroup',
+            name='aggregators',
+        ),
+        migrations.RemoveField(
+            model_name='linkgroup',
+            name='platforms',
+        ),
+        migrations.AddField(
+            model_name='ad',
+            name='link_group',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='core.linkgroup'),
+        ),
+        migrations.AddField(
+            model_name='aggregator',
+            name='link_group',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='core.linkgroup'),
+        ),
+        migrations.AddField(
+            model_name='platform',
+            name='link_group',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='core.linkgroup'),
+        ),
+    ]

core/migrations/0032_operatorwallets.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.7 on 2023-03-20 09:35
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0031_remove_ad_aggregators_remove_ad_platforms_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='OperatorWallets',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('payees', models.ManyToManyField(blank=True, to='core.wallet')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0033_platform_throughput.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-20 09:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0032_operatorwallets'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='platform',
+            name='throughput',
+            field=models.FloatField(default=0),
+        ),
+    ]

core/migrations/0034_transaction_requisition.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.1.7 on 2023-03-20 13:35
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0033_platform_throughput'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transaction',
+            name='requisition',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.requisition'),
+        ),
+    ]

core/migrations/0035_ad_require_feedback_score.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-04-18 07:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0034_transaction_requisition'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ad',
+            name='require_feedback_score',
+            field=models.IntegerField(default=0),
+        ),
+    ]

core/migrations/0036_requisition_owner_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-04-18 07:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0035_ad_require_feedback_score'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='requisition',
+            name='owner_name',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

core/migrations/0037_payout.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.1.7 on 2023-05-06 10:09
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0036_requisition_owner_name'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Payout',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('amount', models.FloatField()),
+                ('description', models.CharField(blank=True, max_length=255, null=True)),
+                ('ts', models.DateTimeField(auto_now_add=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('wallet', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.wallet')),
+            ],
+        ),
+    ]

core/migrations/0038_payout_response.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-05-06 10:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0037_payout'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='payout',
+            name='response',
+            field=models.JSONField(blank=True, default=dict, null=True),
+        ),
+    ]

core/models.py

@@ -0,0 +1,685 @@
+import uuid
+
+from django.contrib.auth.models import AbstractUser
+from django.db import models
+
+# from core.lib.customers import get_or_create, update_customer_fields
+from core.util import logs
+
+log = logs.get_logger(__name__)
+
+SERVICE_CHOICES = (("nordigen", "Nordigen"),)
+
+PLATFORM_SERVICE_CHOICES = (("agora", "Agora"),)
+
+INTERVAL_CHOICES = (
+    (0, "Never"),
+    (5, "Every 5 seconds"),
+    (15, "Every 15 seconds"),
+    (30, "Every 30 seconds"),
+    (60, "Every minute"),
+    (60 * 5, "Every 5 minutes"),
+    (60 * 10, "Every 10 minutes"),
+    (60 * 60, "Every hour"),
+    (60 * 60 * 4, "Every 4 hours"),
+    (86400, "Every day"),
+)
+
+TRANSACTION_SOURCE_CHOICES = (
+    ("booked", "Booked"),
+    ("pending", "Pending"),
+)
+
+
+class User(AbstractUser):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    payment_provider_id = models.CharField(max_length=255, null=True, blank=True)
+    billing_provider_id = models.CharField(max_length=255, null=True, blank=True)
+    email = models.EmailField(unique=True)
+
+    def get_notification_settings(self):
+        return NotificationSettings.objects.get_or_create(user=self)[0]
+
+
+class NotificationSettings(models.Model):
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    ntfy_topic = models.CharField(max_length=255, null=True, blank=True)
+    ntfy_url = models.CharField(max_length=255, null=True, blank=True)
+
+    def __str__(self):
+        return f"Notification settings for {self.user}"
+
+
+class LinkGroup(models.Model):
+    """
+    A group linking Aggregators, Platforms and defining a percentage split
+    that the owners of each should receive.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    name = models.CharField(max_length=255)
+
+    platform_owner_cut_percentage = models.FloatField(default=0)
+    requisition_owner_cut_percentage = models.FloatField(default=0)
+    operator_cut_percentage = models.FloatField(default=0)
+
+    enabled = models.BooleanField(default=True)
+
+    def __str__(self):
+        return self.name
+
+    def payees(self):
+        payees = {}
+        for platform in self.platform_set.all():
+            for payee in platform.payees.all():
+                if "platform" not in payees:
+                    payees["platform"] = []
+                payees["platform"].append(payee)
+
+        for aggregator in self.aggregator_set.all():
+            agg_reqs = aggregator.requisition_set.all()
+            for req in agg_reqs:
+                for payee in req.payees.all():
+                    if "requisition" not in payees:
+                        payees["requisition"] = []
+                    payees["requisition"].append(payee)
+
+        return payees
+
+    @property
+    def platforms(self):
+        return Platform.objects.filter(link_group=self)
+
+    @property
+    def aggregators(self):
+        return Aggregator.objects.filter(link_group=self)
+
+
+class Aggregator(models.Model):
+    """
+    A connection to an API aggregator to pull transactions from bank accounts.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    name = models.CharField(max_length=255)
+    service = models.CharField(max_length=255, choices=SERVICE_CHOICES)
+    secret_id = models.CharField(max_length=1024, null=True, blank=True)
+    secret_key = models.CharField(max_length=1024, null=True, blank=True)
+    access_token = models.CharField(max_length=1024, null=True, blank=True)
+    access_token_expires = models.DateTimeField(null=True, blank=True)
+    poll_interval = models.IntegerField(default=10)
+
+    account_info = models.JSONField(default=dict)
+    currencies = models.JSONField(default=list)
+
+    fetch_accounts = models.BooleanField(default=True)
+
+    link_group = models.ForeignKey(
+        LinkGroup, on_delete=models.CASCADE, null=True, blank=True
+    )
+
+    enabled = models.BooleanField(default=True)
+
+    def __str__(self):
+        return f"{self.name} ({self.get_service_display()})"
+
+    @classmethod
+    def get_by_id(cls, obj_id, user):
+        return cls.objects.get(id=obj_id, user=user)
+
+    @property
+    def client(self):
+        pass
+
+    @classmethod
+    def get_for_platform(cls, platform):
+        # aggregators = []
+        # linkgroups = LinkGroup.objects.filter(
+        #     platforms=platform,
+        #     enabled=True,
+        # )
+        # for link in linkgroups:
+        #     for aggregator in link.aggregators.all():
+        #         if aggregator not in aggregators:
+        #             aggregators.append(aggregator)
+        platform_link = platform.link_group
+
+        # return aggregators
+        return cls.objects.filter(
+            link_group=platform_link,
+        )
+
+    @property
+    def platforms(self):
+        """
+        Get platforms for this aggregator.
+        Do this by looking up LinkGroups with the aggregator.
+        Then, join them all together.
+        """
+        return Platform.objects.filter(link_group=self.link_group)
+
+    @property
+    def requisitions(self):
+        """
+        Get requisitions for this aggregator.
+        Do this by looking up LinkGroups with the aggregator.
+        Then, join them all together.
+        """
+        return Requisition.objects.filter(
+            aggregator=self,
+        )
+
+    def get_requisition(self, requisition_id):
+        return Requisition.objects.filter(
+            aggregator=self,
+            requisition_id=requisition_id,
+        ).first()
+
+    @classmethod
+    def get_currencies_for_platform(cls, platform):
+        # aggregators = Aggregator.get_for_platform(platform)
+        aggregators = platform.aggregators
+        currencies = set()
+        for aggregator in aggregators:
+            for currency in aggregator.currencies:
+                currencies.add(currency)
+
+        return list(currencies)
+
+    @classmethod
+    def get_account_info_for_platform(cls, platform):
+        # aggregators = Aggregator.get_for_platform(platform)
+        aggregators = platform.aggregators
+        account_info = {}
+        for agg in aggregators:
+            for bank, accounts in agg.account_info.items():
+                if bank not in account_info:
+                    account_info[bank] = []
+                for account in accounts:
+                    account_info[bank].append(account)
+        return account_info
+
+    def add_transaction(self, requisition_id, account_id, tx_data):
+        requisition = Requisition.objects.filter(
+            aggregator=self, requisition_id=requisition_id
+        ).first()
+        # if requisition:
+        #     tx_data["requisition"] = requisition
+        return Transaction.objects.create(
+            aggregator=self,
+            account_id=account_id,
+            reconciled=False,
+            requisition=requisition,
+            **tx_data,
+        )
+
+    def get_transaction(self, account_id, tx_id):
+        transaction = Transaction.objects.filter(
+            account_id=account_id,
+            transaction_id=tx_id,
+        ).first()
+        if not transaction:
+            return None
+        return transaction
+
+    @property
+    def trades(self):
+        """
+        Get all trades for the platforms of this aggregator's link group.
+        """
+        trades = []
+        for platform in self.platforms:
+            platform_trades = platform.trades
+            for trade in platform_trades:
+                trades.append(trade)
+
+        return trades
+
+    @property
+    def trades_currencies(self):
+        """
+        Get all the trade fiat currencies.
+        """
+        currencies = []
+        for trade in self.trades:
+            if trade.currency not in currencies:
+                currencies.append(trade.currency)
+
+        return currencies
+
+
+class Wallet(models.Model):
+    """
+    A wallet for a user.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    name = models.CharField(max_length=255)
+    address = models.CharField(max_length=255)
+
+    def __str__(self):
+        return self.name
+
+
+class Platform(models.Model):
+    """
+    A connection to an arbitrage platform like AgoraDesk.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    name = models.CharField(max_length=255)
+    service = models.CharField(max_length=255, choices=PLATFORM_SERVICE_CHOICES)
+    token = models.CharField(max_length=1024)
+    password = models.CharField(max_length=1024)
+    otp_token = models.CharField(max_length=1024, null=True, blank=True)
+    username = models.CharField(max_length=255)
+
+    send = models.BooleanField(default=True)
+    cheat = models.BooleanField(default=False)
+    dummy = models.BooleanField(default=False)
+    cheat_interval_seconds = models.IntegerField(default=0, choices=INTERVAL_CHOICES)
+
+    margin = models.FloatField(default=1.20)
+    max_margin = models.FloatField(default=1.30)
+    min_margin = models.FloatField(default=1.15)
+    min_trade_size_usd = models.FloatField(default=10)
+    max_trade_size_usd = models.FloatField(default=4000)
+
+    accept_within_usd = models.FloatField(default=1)
+    no_reference_amount_check_max_usd = models.FloatField(default=400)
+
+    last_messages = models.JSONField(default=dict)
+    platform_ad_ids = models.JSONField(default=dict)
+
+    base_usd = models.FloatField(default=2800)
+    withdrawal_trigger = models.FloatField(default=200)
+
+    payees = models.ManyToManyField(Wallet, blank=True)
+
+    link_group = models.ForeignKey(
+        LinkGroup, on_delete=models.CASCADE, null=True, blank=True
+    )
+
+    enabled = models.BooleanField(default=True)
+    throughput = models.FloatField(default=0)
+
+    def __str__(self):
+        return self.name
+
+    def get_ad(self, platform_ad_id):
+        ad_id = self.platform_ad_ids.get(platform_ad_id, None)
+        if not ad_id:
+            return None
+        ad_object = Ad.objects.filter(
+            id=ad_id, user=self.user, link_group=self.link_group, enabled=True
+        ).first()
+        return ad_object
+
+    @classmethod
+    def get_for_user(cls, user):
+        return cls.objects.filter(user=user, enabled=True)
+
+    @property
+    def currencies(self):
+        return Aggregator.get_currencies_for_platform(self)
+
+    @property
+    def account_info(self):
+        return Aggregator.get_account_info_for_platform(self)
+
+    @property
+    def ads(self):
+        """
+        Get all ads linked to this platform.
+        """
+        return Ad.objects.filter(
+            user=self.user, enabled=True, link_group=self.link_group
+        )
+
+    @property
+    def ads_assets(self):
+        """
+        Get all the assets of all the ads.
+        """
+        assets = set()
+
+        for ad in self.ads:
+            for asset in ad.asset_list.all():
+                assets.add(asset.code)
+
+        return list(assets)
+
+    @property
+    def ads_providers(self):
+        """
+        Get all the providers of all the ads.
+        """
+        providers = set()
+
+        for ad in self.ads:
+            for provider in ad.provider_list.all():
+                providers.add(provider.code)
+
+        return list(providers)
+
+    @property
+    def references(self):
+        """
+        Get references of all our trades that are open.
+        """
+        references = []
+        our_trades = Trade.objects.filter(platform=self, open=True)
+        for trade in our_trades:
+            references.append(trade.reference)
+
+        return references
+
+    @property
+    def trade_ids(self):
+        """
+        Get trade IDs of all our trades that are open.
+        """
+        references = []
+        our_trades = Trade.objects.filter(platform=self, open=True)
+        for trade in our_trades:
+            references.append(trade.contact_id)
+
+        return references
+
+    def get_trade_by_reference(self, reference):
+        return Trade.objects.filter(
+            platform=self,
+            open=True,
+            reference=reference,
+        ).first()
+
+    @property
+    def trades(self):
+        """
+        Get all our open trades.
+        """
+        our_trades = Trade.objects.filter(platform=self, open=True)
+        return our_trades
+
+    def contact_id_to_reference(self, contact_id):
+        """
+        Get a reference from a contact_id.
+        """
+        trade = Trade.objects.filter(
+            platform=self, open=True, contact_id=contact_id
+        ).first()
+        if not trade:
+            return None
+        return trade.reference
+
+    def get_trade_by_trade_id(self, trade_id):
+        return Trade.objects.filter(
+            platform=self,
+            open=True,
+            contact_id=trade_id,
+        ).first()
+
+    def new_trade(self, trade_cast):
+        trade = Trade.objects.create(
+            platform=self,
+            **trade_cast,
+        )
+        return trade
+
+    def remove_trades_with_reference_not_in(self, reference_list):
+        """
+        Set trades with reference not in list to open=False.
+        """
+        trades = Trade.objects.filter(platform=self, open=True)
+        messages = []
+        for trade in trades:
+            if trade.reference not in reference_list:
+                trade.open = False
+                trade.save()
+                msg = f"[{trade.reference}]: Archiving ID: {trade.contact_id}"
+                messages.append(msg)
+                log.info(msg)
+        return messages
+
+    @classmethod
+    def get_for_aggregator(cls, aggregator):
+        # platforms = []
+        # linkgroups = LinkGroup.objects.filter(
+        #     aggregators=aggregator,
+        #     enabled=True,
+        # )
+        # for link in linkgroups:
+        #     for platform in link.platforms.all():
+        #         if platform not in platforms:
+        #             platforms.append(platform)
+
+        # return platforms
+
+        aggregator_link = aggregator.link_group
+        return cls.objects.filter(
+            link_group=aggregator_link,
+        )
+
+    @property
+    def aggregators(self):
+        """
+        Get aggregators for this platform.
+        Do this by looking up LinkGroups with the platform.
+        Then, join them all together.
+        """
+
+        return Aggregator.objects.filter(
+            link_group=self.link_group,
+        )
+
+    @property
+    def platforms(self):
+        """
+        Get all platforms in this link group.
+        Do this by looking up LinkGroups with the platform.
+        Then, join them all together.
+        """
+
+        return Platform.objects.filter(
+            link_group=self.link_group,
+        )
+
+    def get_requisition(self, aggregator_id, requisition_id):
+        """
+        Get a Requisition object with the provided values.
+        """
+        requisition = Requisition.objects.filter(
+            aggregator_id=aggregator_id,
+            requisition_id=requisition_id,
+        ).first()
+        return requisition
+
+
+class Asset(models.Model):
+    code = models.CharField(max_length=64)
+    name = models.CharField(max_length=255)
+
+    def __str__(self):
+        return f"{self.name} ({self.code})"
+
+
+class Provider(models.Model):
+    code = models.CharField(max_length=64)
+    name = models.CharField(max_length=255)
+
+    def __str__(self):
+        return f"{self.name} ({self.code})"
+
+
+class Ad(models.Model):
+    """
+    An advert definition
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    name = models.CharField(max_length=255)
+
+    text = models.TextField()
+
+    # Shown when the user opens a trade
+    payment_details = models.TextField()
+
+    # Shown after
+    payment_details_real = models.TextField()
+
+    payment_method_details = models.CharField(max_length=255)
+
+    require_feedback_score = models.IntegerField(default=0)
+
+    dist_list = models.TextField()
+
+    asset_list = models.ManyToManyField(Asset)
+    provider_list = models.ManyToManyField(Provider)
+
+    account_map = models.JSONField(default=dict)
+    account_whitelist = models.TextField(null=True, blank=True)
+
+    send_reference = models.BooleanField(default=True)
+
+    visible = models.BooleanField(default=True)
+    link_group = models.ForeignKey(
+        LinkGroup, on_delete=models.CASCADE, null=True, blank=True
+    )
+    enabled = models.BooleanField(default=True)
+
+    @property
+    def providers(self):
+        return [x.code for x in self.provider_list.all()]
+
+    @property
+    def assets(self):
+        return [x.code for x in self.asset_list.all()]
+
+    @classmethod
+    def get_by_id(cls, ad_id, user):
+        return cls.objects.filter(id=ad_id, user=user, enabled=True).first()
+
+
+class Transaction(models.Model):
+    """
+    A transaction on an aggregator.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    aggregator = models.ForeignKey(Aggregator, on_delete=models.CASCADE)
+    requisition = models.ForeignKey(
+        "core.Requisition", null=True, on_delete=models.CASCADE
+    )
+    account_id = models.CharField(max_length=255)
+    transaction_id = models.CharField(max_length=255)
+
+    ts_added = models.DateTimeField(auto_now_add=True)
+
+    recipient = models.CharField(max_length=255, null=True, blank=True)
+    sender = models.CharField(max_length=255, null=True, blank=True)
+
+    amount = models.FloatField()
+    currency = models.CharField(max_length=16)
+
+    note = models.CharField(max_length=255, null=True, blank=True)
+
+    reconciled = models.BooleanField(default=False)
+
+
+class Trade(models.Model):
+    """
+    A trade on a Platform.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    platform = models.ForeignKey(Platform, on_delete=models.CASCADE)
+
+    contact_id = models.CharField(max_length=255)
+    reference = models.CharField(max_length=255)
+    buyer = models.CharField(max_length=255)
+    amount_fiat = models.FloatField()
+    currency = models.CharField(max_length=16)
+    amount_crypto = models.FloatField()
+    asset = models.CharField(max_length=16)
+    provider = models.CharField(max_length=255)
+    ad_id = models.CharField(max_length=255, null=True, blank=True)
+
+    open = models.BooleanField(default=True)
+
+    linked = models.ManyToManyField(Transaction, blank=True)
+    reconciled = models.BooleanField(default=False)
+
+    released = models.BooleanField(default=False)
+    release_response = models.JSONField(default=dict)
+
+
+class Requisition(models.Model):
+    """
+    A requisition for an Aggregator
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    aggregator = models.ForeignKey(Aggregator, on_delete=models.CASCADE)
+    requisition_id = models.CharField(max_length=255)
+    payment_details = models.TextField(null=True, blank=True)
+
+    owner_name = models.CharField(max_length=255, null=True, blank=True)
+
+    transaction_source = models.CharField(
+        max_length=255, choices=TRANSACTION_SOURCE_CHOICES, default="booked"
+    )
+
+    throughput = models.FloatField(default=0)
+    payees = models.ManyToManyField(Wallet, blank=True)
+
+    def __str__(self):
+        return f"Aggregator: {self.aggregator.name} ID: {self.requisition_id}"
+
+
+class OperatorWallets(models.Model):
+    """
+    A list of wallets to designate as operator wallets for this user.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    payees = models.ManyToManyField(Wallet, blank=True)
+
+
+class Payout(models.Model):
+    """
+    A profit payout.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    wallet = models.ForeignKey(Wallet, on_delete=models.CASCADE)
+    amount = models.FloatField()
+    description = models.CharField(max_length=255, null=True, blank=True)
+    response = models.JSONField(default=dict, null=True, blank=True)
+
+    ts = models.DateTimeField(auto_now_add=True)
+
+
+assets = {
+    "XMR": "Monero",
+    "BTC": "Bitcoin",
+}
+providers = {
+    "REVOLUT": "Revolut",
+    "NATIONAL_BANK": "Bank transfer",
+    "SWISH": "Swish",
+}
+for code, name in assets.items():
+    if not Asset.objects.filter(code=code).exists():
+        Asset.objects.create(code=code, name=name)
+
+for code, name in providers.items():
+    if not Provider.objects.filter(code=code).exists():
+        Provider.objects.create(code=code, name=name)

core/static/django-htmx.js

@@ -0,0 +1,22 @@
+{
+    const data = document.currentScript.dataset;
+    const isDebug = data.debug === "True";
+
+    if (isDebug) {
+        document.addEventListener("htmx:beforeOnLoad", function (event) {
+            const xhr = event.detail.xhr;
+            if (xhr.status == 500 || xhr.status == 404) {
+                // Tell htmx to stop processing this response
+                event.stopPropagation();
+
+                document.children[0].innerHTML = xhr.response;
+
+                // Run Django’s inline script
+                // (1, eval) wtf - see https://stackoverflow.com/questions/9107240/1-evalthis-vs-evalthis-in-javascript
+                (1, eval)(document.scripts[0].innerText);
+                // Need to directly call Django’s onload function since browser won’t
+                window.onload();
+            }
+        });
+    }
+}

core/static/js/remove-me.js

@@ -0,0 +1,27 @@
+(function(){
+    function maybeRemoveMe(elt) {
+        var timing = elt.getAttribute("remove-me") || elt.getAttribute("data-remove-me");
+        if (timing) {
+            setTimeout(function () {
+                elt.parentElement.removeChild(elt);
+            }, htmx.parseInterval(timing));
+        }
+    }
+
+    htmx.defineExtension('remove-me', {
+        onEvent: function (name, evt) {
+            if (name === "htmx:afterProcessNode") {
+                var elt = evt.detail.elt;
+                if (elt.getAttribute) {
+                    maybeRemoveMe(elt);
+                    if (elt.querySelectorAll) {
+                        var children = elt.querySelectorAll("[remove-me], [data-remove-me]");
+                        for (var i = 0; i < children.length; i++) {
+                            maybeRemoveMe(children[i]);
+                        }
+                    }
+                }
+            }
+        }
+    });
+})();

core/static/manifest.webmanifest

@@ -0,0 +1,15 @@
+{
+    "background_color": "white",
+    "description": "Cryptocurrency arbitrage automation",
+    "display": "fullscreen",
+    "icons": [
+      {
+        "src": "/static/logo.png",
+        "sizes": "800x800",
+        "type": "image/png"
+      }
+    ],
+    "name": "Pluto Arbitrage",
+    "short_name": "Pluto",
+    "start_url": "/"
+  }

core/static/modal.js

@@ -0,0 +1,44 @@
+// var modal = document.querySelector('.modal');  // assuming you have only 1
+var modal = document.getElementById("modal");
+var html = document.querySelector('html');
+
+var disableModal = function() {
+    modal.classList.remove('is-active');
+    html.classList.remove('is-clipped');
+    var modal_refresh = document.getElementsByClassName("modal-refresh");
+    for(var i = 0; i < modal_refresh.length; i++) {
+        modal_refresh[i].remove();
+    }
+}
+
+var elements = document.querySelectorAll('.modal-background');
+for(var i = 0; i < elements.length; i++) {
+    elements[i].addEventListener('click', function(e) {
+        // elements[i].preventDefault();
+        disableModal();
+    });
+}
+
+var elements = document.querySelectorAll('.modal-close');
+for(var i = 0; i < elements.length; i++) {
+    elements[i].addEventListener('click', function(e) {
+        // elements[i].preventDefault();
+        disableModal();
+    });
+}
+
+function activateButtons() {
+    var elements = document.querySelectorAll('.modal-close-button');
+    for(var i = 0; i < elements.length; i++) {
+        elements[i].addEventListener('click', function(e) {
+            // elements[i].preventDefault();
+            disableModal();
+        });
+    }
+}
+activateButtons();
+// modal.querySelector('.modal-close-button').addEventListener('click', function(e) {
+//   e.preventDefault();
+//   modal.classList.remove('is-active');
+//   html.classList.remove('is-clipped');
+// });

core/templates/base.html

@@ -0,0 +1,388 @@
+{% load static %}
+{% load cache %}
+
+<!DOCTYPE html>
+<html lang="en-GB">
+  {# cache 600 head request.path_info #}
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Pluto - {{ request.path_info }}</title>
+    <link rel="shortcut icon" href="{%  static 'favicon.ico' %}">
+    <link rel="manifest" href="{% static 'manifest.webmanifest' %}">
+    <link rel="stylesheet" href="{% static 'css/bulma.min.css' %}">
+    <link rel="stylesheet" href="{% static 'css/bulma-tooltip.min.css' %}">
+    <link rel="stylesheet" href="https://site-assets.fontawesome.com/releases/v6.1.1/css/all.css">
+    <link rel="stylesheet" href="{% static 'css/gridstack.min.css' %}">
+    <script src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
+    <script defer src="{% static 'js/hyperscript.min.js' %}" integrity="sha384-6GYN8BDHOJkkru6zcpGOUa//1mn+5iZ/MyT6mq34WFIpuOeLF52kSi721q0SsYF9" crossorigin="anonymous"></script>
+    <script defer src="{% static 'js/remove-me.js' %}" integrity="sha384-6fHcFNoQ8QEI3ZDgw9Z/A6Brk64gF7AnFbLgdrumo8/kBbsKQ/wo7wPegj5WkzuG" crossorigin="anonymous"></script>
+    <script src="{% static 'js/gridstack-all.js' %}"></script>
+    <script defer src="{% static 'js/magnet.min.js' %}"></script>
+    <script>
+      document.addEventListener("restore-scroll", function(event) {
+        var scrollpos = localStorage.getItem('scrollpos');
+        if (scrollpos) {
+          window.scrollTo(0, scrollpos)
+        };
+      });
+
+      document.addEventListener("htmx:beforeSwap", function(event) {
+        localStorage.setItem('scrollpos', window.scrollY);
+
+      });
+    </script>
+    <script>
+      document.addEventListener('DOMContentLoaded', () => {
+
+        // Get all "navbar-burger" elements
+        const $navbarBurgers = Array.prototype.slice.call(document.querySelectorAll('.navbar-burger'), 0);
+
+        // Add a click event on each of them
+        $navbarBurgers.forEach( el => {
+          el.addEventListener('click', () => {
+
+            // Get the target from the "data-target" attribute
+            const target = el.dataset.target;
+            const $target = document.getElementById(target);
+
+            // Toggle the "is-active" class on both the "navbar-burger" and the "navbar-menu"
+            el.classList.toggle('is-active');
+            $target.classList.toggle('is-active');
+
+          });
+        });
+
+      });
+    </script>
+    <style>
+      .icon { border-bottom: 0px !important;}
+      .wrap {
+        word-wrap: break-word;
+      }
+      .nowrap-parent {
+        white-space: nowrap;
+      }
+      .nowrap-child {
+        display: inline-block;
+      }
+      .htmx-indicator{
+        opacity:0;
+        transition: opacity 500ms ease-in;
+      }
+      .htmx-request .htmx-indicator{
+        opacity:1
+      }
+      .htmx-request.htmx-indicator{
+        opacity:1
+      }
+
+      .tooltiptext {
+        visibility: hidden;
+        background-color: black;
+        color: #fff;
+        text-align: center;
+        padding: 5px 0;
+        border-radius: 6px;
+        position: absolute;
+        z-index: 1;
+      }
+
+      .rounded-tooltip:hover .tooltiptext {
+        visibility: visible;
+      }
+
+      .table {
+        background: transparent !important;
+      }
+
+      tr {
+        transition: all 0.2s ease-in-out;
+      }
+
+      tr:hover {
+        cursor:pointer;
+        background-color:rgba(221, 224, 255, 0.3) !important;
+      }
+
+      a.panel-block {
+        transition: all 0.2s ease-in-out;
+      }
+
+      a.panel-block:hover {
+        cursor:pointer;
+        background-color:rgba(221, 224, 255, 0.3) !important;
+      }
+
+      .panel, .box, .modal {
+        background-color:rgba(250, 250, 250, 0.5) !important;
+      }
+      .modal, .modal.box{
+        background-color:rgba(210, 210, 210, 0.9) !important;
+      }
+      .modal-background{
+        background-color:rgba(255, 255, 255, 0.3) !important;
+      }
+
+      .has-background-grey-lighter{
+        background-color:rgba(219, 219, 219, 0.5) !important;
+      }
+      .navbar {
+        background-color:rgba(0, 0, 0, 0.03) !important;
+      }
+
+      .grid-stack-item-content {
+        display: flex !important;
+        flex-direction: column !important;
+        overflow-x: hidden !important;
+        overflow-y: hidden !important;
+      }
+
+      .panel {
+        display: flex !important;
+        flex-direction: column !important;
+        overflow: hidden;
+      }
+
+      .panel-block {
+        overflow-y:auto;
+        overflow-x:auto;
+        min-height: 90%;
+        display: block;
+      }
+
+      .floating-window {
+        /* background-color:rgba(210, 210, 210, 0.6) !important; */
+        display: flex !important;
+        flex-direction: column !important;
+        overflow-x: hidden !important;
+        overflow-y: hidden !important;
+        max-height: 300px;
+        z-index: 9000;
+        position: absolute;
+        top: 50px;
+        left: 50px;
+      }
+
+      .floating-window .panel {
+        background-color:rgba(250, 250, 250, 0.8) !important;
+      }
+
+      .float-right {
+        float: right;
+        padding-right: 5px;
+        padding-left: 5px;
+      }
+
+      .grid-stack-item:hover .ui-resizable-handle {
+        display: block !important;
+      }
+      .ui-resizable-handle {
+        z-index: 39 !important;
+      }
+
+    </style>
+    <!-- Piwik --> {# Yes it's in the source, fight me #}
+    <script type="text/javascript">
+      var _paq = _paq || [];
+      _paq.push(['trackPageView']);
+      _paq.push(['enableLinkTracking']);
+      (function() {
+        _paq.push(['setTrackerUrl',  'https://api-dd242151ac50129c3320f209578a406c.s.zm.is']);
+        _paq.push(['setSiteId', 6]);
+        _paq.push(['setApiToken', 'owVUM8fMHxHtyDoIFdyZxx1TWTNECV5ImmoKI1y5muc']);
+        var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+        g.type='text/javascript'; g.async=true; g.defer=true; g.src='https://c87zpt9a74m181wto33r.s.zm.is/embed.js'; s.parentNode.insertBefore(g,s);
+      })();
+    </script>
+    <!-- End Piwik Code -->
+  </head>
+  {# endcache #}
+  <body>
+
+    {# cache 600 nav request.user.id #}
+    <nav class="navbar" role="navigation" aria-label="main navigation">
+      <div class="navbar-brand">
+        <a class="navbar-item" href="{% url 'home' %}">
+          <img src="{% static 'logo-128.png' %}" alt="logo">
+        </a>
+
+        <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="bar">
+          <span aria-hidden="true"></span>
+          <span aria-hidden="true"></span>
+          <span aria-hidden="true"></span>
+        </a>
+      </div>
+
+      <div id="bar" class="navbar-menu">
+        <div class="navbar-start">
+          <a class="navbar-item" href="{% url 'home' %}">
+            Home
+          </a>
+          {% if user.is_authenticated %}
+
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link" href="{% url 'profit' type='page' %}">
+                Profit
+              </a>
+              <div class="navbar-dropdown">
+                <a class="navbar-item" href="{% url 'payouts' type='page' %}">
+                  Payouts
+                </a>
+              </div>
+            </div>
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link">
+                Banks
+              </a>
+              <div class="navbar-dropdown">
+                <a class="navbar-item" href="{% url 'currencies' type='page' %}">
+                  Currencies
+                </a>
+                <a class="navbar-item" href="{% url 'balances' type='page' %}">
+                  Balances
+                </a>
+              </div>
+            </div>
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link">
+                Platforms
+              </a>
+
+              <div class="navbar-dropdown">
+                <a class="navbar-item" href="{% url 'trades' type='page' %}">
+                  Trades
+                </a>
+                <a class="navbar-item" href="#">
+                  Wallets
+                </a>
+                <a class="navbar-item" href="{% url 'ads' type='page' %}">
+                  Ads
+                </a>
+                <a class="navbar-item" href="{% url 'ads' type='page' %}">
+                  Posted Ads
+                </a>
+                <a class="navbar-item" href="#">
+                  Market Research
+                </a>
+              </div>
+            </div>
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link">
+                Setup
+              </a>
+
+              <div class="navbar-dropdown">
+                <a class="navbar-item" href="{% url 'aggregators' type='page' %}">
+                  Bank Aggregators
+                </a>
+                <a class="navbar-item" href="{% url 'platforms' type='page' %}">
+                  Platform Connections
+                </a>
+                <a class="navbar-item" href="{% url 'wallets' type='page' %}">
+                  Profit Wallets
+                </a>
+                <a class="navbar-item" href="{% url 'linkgroups' type='page' %}">
+                  Link Groups
+                </a>
+              </div>
+            </div>
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link">
+                Account
+              </a>
+
+              <div class="navbar-dropdown">
+                <a class="navbar-item" href="{% url 'two_factor:profile' %}">
+                  Security
+                </a>
+                <a class="navbar-item" href="{% url 'notifications_update' type='page' %}">
+                  Notifications
+                </a>
+                <a class="navbar-item" href="{% url 'operator_wallets_update' type='page' %}">
+                  Operator Wallets
+                </a>
+              </div>
+            </div>
+          {% endif %}
+          {% if settings.BILLING_ENABLED %}
+            {% if user.is_authenticated %}
+              <a class="navbar-item" href="{# url 'billing' #}">
+                Billing
+              </a>
+            {% endif %}
+          {% endif %}
+          <a class="navbar-item add-button">
+            Install
+          </a>
+        </div>
+
+        <div class="navbar-end">
+          <div class="navbar-item">
+            <div class="buttons">
+              {% if not user.is_authenticated %}
+                <a class="button" href="{% url 'signup' %}">
+                  <strong>Sign up</strong>
+                </a>
+                <a class="button" href="{% url 'two_factor:login' %}">
+                  Log in
+                </a>
+              {% endif %}
+
+              {% if user.is_authenticated %}
+                <a class="button" href="{% url 'logout' %}">Logout</a>
+              {% endif %}
+
+            </div>
+          </div>
+        </div>
+      </div>
+    </nav>
+    {# endcache #}
+    <script>
+      let deferredPrompt;
+      const addBtn = document.querySelector('.add-button');
+      addBtn.style.display = 'none';
+      window.addEventListener('beforeinstallprompt', (e) => {
+        // Prevent Chrome 67 and earlier from automatically showing the prompt
+        e.preventDefault();
+        // Stash the event so it can be triggered later.
+        deferredPrompt = e;
+        // Update UI to notify the user they can add to home screen
+        addBtn.style.display = 'block';
+
+        addBtn.addEventListener('click', (e) => {
+          // hide our user interface that shows our A2HS button
+          addBtn.style.display = 'none';
+          // Show the prompt
+          deferredPrompt.prompt();
+          // Wait for the user to respond to the prompt
+          deferredPrompt.userChoice.then((choiceResult) => {
+            if (choiceResult.outcome === 'accepted') {
+              console.log('User accepted the A2HS prompt');
+            } else {
+              console.log('User dismissed the A2HS prompt');
+            }
+            deferredPrompt = null;
+          });
+        });
+      });
+    </script>
+    {% block outer_content %}
+    {% endblock %}
+    <section class="section">
+      <div class="container is-widescreen">
+        {% block content_wrapper %}
+          {% block content %}
+          {% endblock %}
+        {% endblock %}
+        <div id="modals-here">
+        </div>
+        <div id="windows-here">
+        </div>
+        <div id="widgets-here" style="display: none;">
+        </div>
+      </div>
+    </section>
+  </body>
+</html>

core/templates/index.html

@@ -0,0 +1,103 @@
+{% extends "base.html" %}
+{% load static %}
+{% load joinsep %}
+{% block content %}
+  <div class="grid-stack" id="grid-stack-main">
+    <script>
+      var grid = GridStack.init({
+        cellHeight: 20,
+        cellWidth: 50,
+        cellHeightUnit: 'px',
+        auto: true,
+        float: true,
+        draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
+        removable: false,
+        animate: true,
+      });
+      // GridStack.init();
+
+      // a widget is ready to be loaded
+      document.addEventListener('load-widget', function(event) {
+        let containers = htmx.findAll('#widget');
+        for (let x = 0, len = containers.length; x < len; x++) {
+          container = containers[x];
+          // get the scripts, they won't be run on the new element so we need to eval them
+          let widgetelement = container.firstElementChild.cloneNode(true);
+          console.log(widgetelement);
+          var scripts = htmx.findAll(widgetelement, "script");
+          var new_id = widgetelement.id;
+
+          // check if there's an existing element like the one we want to swap
+          let grid_element = htmx.find('#grid-stack-main');
+          let existing_widget = htmx.find(grid_element, "#"+new_id);
+
+          // get the size and position attributes
+          if (existing_widget) {
+            let attrs = existing_widget.getAttributeNames();
+            for (let i = 0, len = attrs.length; i < len; i++) {
+              if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
+                widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
+              }
+            }
+          }
+          // clear the queue element
+          container.outerHTML = "";
+          // container.firstElementChild.outerHTML = "";
+          grid.addWidget(widgetelement);
+
+          // re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
+          htmx.process(widgetelement);
+
+          // update the size of the widget according to its content
+          var added_widget = htmx.find(grid_element, "#"+new_id);
+          var itemContent = htmx.find(added_widget, ".control");
+          var scrollheight = itemContent.scrollHeight+80;
+          var verticalmargin = 0;
+          var cellheight = grid.opts.cellHeight;
+          var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
+          var opts = {
+            h: height,
+          }
+          grid.update(
+            added_widget,
+            opts
+          );
+
+          // run the JS scripts inside the added element again
+          for (var i = 0; i < scripts.length; i++) {
+            eval(scripts[i].innerHTML);
+          }
+        }
+        // clear the containers we just added
+        // for (let x = 0, len = containers.length; x < len; x++) {
+        //   container = containers[x];
+        //   container.inner = "";
+        // }
+        grid.compact();
+      });
+    </script>
+    <!-- <div>
+      <div
+        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        hx-get="{# url 'example' type='widget' #}"
+        hx-target="#widgets-here"
+        hx-trigger="load"
+        hx-swap="afterend"
+        style="display: none;"></div>
+      <div
+        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        hx-get="{# url 'example' type='widget' #}"
+        hx-target="#widgets-here"
+        hx-trigger="load"
+        hx-swap="afterend"
+        style="display: none;"></div>
+      <div
+        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+        hx-get="{# url 'example' type='widget' #}"
+        hx-target="#widgets-here"
+        hx-trigger="load"
+        hx-swap="afterend"
+        style="display: none;"></div>
+    </div> -->
+
+{% endblock %}

core/templates/partials/ad-list.html

@@ -0,0 +1,79 @@
+{% load cache %}
+{% load cachalot cache %}
+{% get_last_invalidation 'core.Ad' as last %}
+{% include 'mixins/partials/notify.html' %}
+{# cache 600 objects_ads request.user.id object_list type last #}
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>id</th>
+    <th>user</th>
+    <th>name</th>
+    <th>enabled</th>
+    <th>actions</th>
+  </thead>
+  {% for item in object_list %}
+    <tr>
+      <td>
+        <a
+          class="has-text-grey"
+          onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ item.id }}');">
+          <span class="icon" data-tooltip="Copy to clipboard">
+            <i class="fa-solid fa-copy" aria-hidden="true"></i>
+          </span>
+        </a>
+      </td>
+      <td>{{ item.user }}</td>
+      <td>{{ item.name }}</td>
+      <td>
+        {% if item.enabled %}
+          <span class="icon">
+            <i class="fa-solid fa-check"></i>
+          </span>
+        {% else %}
+          <span class="icon">
+            <i class="fa-solid fa-xmark"></i>
+          </span>
+        {% endif %}
+      </td>
+      <td>
+        <div class="buttons">
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-get="{% url 'ad_update' type=type pk=item.id %}"
+            hx-trigger="click"
+            hx-target="#{{ type }}s-here"
+            hx-swap="innerHTML"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-pencil"></i>
+              </span>
+            </span>
+          </button>
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-delete="{% url 'ad_delete' type=type pk=item.id %}"
+            hx-trigger="click"
+            hx-target="#modals-here"
+            hx-swap="innerHTML"
+            hx-confirm="Are you sure you wish to delete {{ item.name }}?"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-xmark"></i>
+              </span>
+            </span>
+          </button>
+        </div>
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>
+{# endcache #}

core/templates/partials/aggregator-countries.html

@@ -0,0 +1,36 @@
+{% include 'mixins/partials/notify.html' %}
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>country</th>
+    <th>actions</th>
+  </thead>
+  {% for item in object_list %}
+    <tr>
+      <td> {{ item }}</td>
+      <td>
+        <div class="buttons">
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-get="{% url 'aggregator_country_banks' type=type pk=pk country=item %}"
+            hx-trigger="click"
+            hx-target="#{{ type }}s-here"
+            hx-swap="innerHTML"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-eye"></i>
+              </span>
+            </span>
+          </button>
+        </div>
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>

core/templates/partials/aggregator-country-banks.html

@@ -0,0 +1,38 @@
+{% include 'mixins/partials/notify.html' %}
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>name</th>
+    <th>logo</th>
+    <th>actions</th>
+  </thead>
+  {% for item in object_list %}
+    <tr>
+      <td>{{ item.name }}</td>
+      <td><img src="{{ item.logo }}" width="35" height="35"></td>
+      <td>
+        <div class="buttons">
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-get="{% url 'aggregator_link' type=type pk=pk bank=item.id %}"
+            hx-trigger="click"
+            hx-target="#{{ type }}s-here"
+            hx-swap="innerHTML"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-link"></i>
+              </span>
+            </span>
+          </button>
+        </div>
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>

core/templates/partials/aggregator-info.html

@@ -0,0 +1,94 @@
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>id</th>
+    <th>created</th>
+    <th>institution</th>
+    <th>accounts</th>
+    <th>payees</th>
+    <th>actions</th>
+  </thead>
+  {% for item in object_list %}
+    <tr>
+      <td>
+        <a
+          class="has-text-grey"
+          onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ item.id }}');">
+          <span class="icon" data-tooltip="Copy to clipboard">
+            <i class="fa-solid fa-copy" aria-hidden="true"></i>
+          </span>
+        </a>
+      </td>
+      <td>{{ item.created }}</td>
+      <td>{{ item.institution_id }}</td>
+      <td>{{ item.accounts|length }}</td>
+      <td>
+        {% for payee in item.requisition.payees.all %}
+          {{ payee.name }}{% if not forloop.last %}, {% endif %}
+        {% endfor %}
+      </td>
+      <td>
+        <div class="buttons">
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-get="{% url 'requisition_update' type=type aggregator_id=pk req_id=item.id %}"
+            hx-trigger="click"
+            hx-target="#modals-here"
+            hx-swap="innerHTML"
+            class="button">
+            <span class="icon-text">
+              <span class="icon has-text-black" data-tooltip="Configure">
+                <i class="fa-solid fa-wrench"></i>
+              </span>
+            </span>
+          </button>
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-delete="{% url 'req_delete' type=type pk=pk req_id=item.id %}"
+            hx-trigger="click"
+            hx-target="#modals-here"
+            hx-swap="innerHTML"
+            hx-confirm="Are you sure you wish to delete {{ item.id }}?"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-xmark"></i>
+              </span>
+            </span>
+          </button>
+          {% if type == 'page' %}
+            <a href="{% url 'req_info' type=type pk=pk req_id=item.id %}"><button
+              class="button">
+              <span class="icon-text">
+                <span class="icon">
+                  <i class="fa-solid fa-eye"></i>
+                </span>
+              </span>
+            </button>
+            </a>
+          {% else %}
+            <button
+              hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+              hx-get="{% url 'req_info' type=type pk=pk req_id=item.id %}"
+              hx-trigger="click"
+              hx-target="#{{ type }}s-here"
+              hx-swap="innerHTML"
+              class="button">
+              <span class="icon-text">
+                <span class="icon">
+                  <i class="fa-solid fa-eye"></i>
+                </span>
+              </span>
+            </button>
+          {% endif %}
+        </div>
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>

core/templates/partials/aggregator-list.html

@@ -0,0 +1,83 @@
+{% load cache %}
+{% load cachalot cache %}
+{% get_last_invalidation 'core.Aggregator' as last %}
+{% include 'mixins/partials/notify.html' %}
+{# cache 600 objects_aggregators request.user.id object_list type last #}
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>id</th>
+    <th>user</th>
+    <th>name</th>
+    <th>service</th>
+    <th>link group</th>
+    <th>enabled</th>
+    <th>actions</th>
+  </thead>
+  {% for item in object_list %}
+    <tr>
+      <td>
+        <a
+          class="has-text-grey"
+          onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ item.id }}');">
+          <span class="icon" data-tooltip="Copy to clipboard">
+            <i class="fa-solid fa-copy" aria-hidden="true"></i>
+          </span>
+        </a>
+      </td>
+      <td>{{ item.user }}</td>
+      <td><a href="{% url 'reqs' type='page' pk=item.id %}">{{ item.name }}</a></td>
+      <td>{{ item.get_service_display }}</td>
+      <td>{{ item.link_group|default_if_none:"—" }}</td>
+      <td>
+        {% if item.enabled %}
+          <span class="icon">
+            <i class="fa-solid fa-check"></i>
+          </span>
+        {% else %}
+          <span class="icon">
+            <i class="fa-solid fa-xmark"></i>
+          </span>
+        {% endif %}
+      </td>
+      <td>
+        <div class="buttons">
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-get="{% url 'aggregator_update' type=type pk=item.id %}"
+            hx-trigger="click"
+            hx-target="#{{ type }}s-here"
+            hx-swap="innerHTML"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-pencil"></i>
+              </span>
+            </span>
+          </button>
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-delete="{% url 'aggregator_delete' type=type pk=item.id %}"
+            hx-trigger="click"
+            hx-target="#modals-here"
+            hx-swap="innerHTML"
+            hx-confirm="Are you sure you wish to delete {{ item.name }}?"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-xmark"></i>
+              </span>
+            </span>
+          </button>
+        </div>
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>
+{# endcache #}

core/templates/partials/aggregator-req-info.html

@@ -0,0 +1,28 @@
+{% include 'mixins/partials/notify.html' %}
+
+<h1 class="title">{{ title_singular }} info</h1>
+{% for key, item in object.items %}
+  <h1 class="title is-4">Bank: {{ key }}</h1>
+  {% for account in item %}
+    <table class="table is-fullwidth is-hoverable box">
+      <thead>
+        <th>attribute</th>
+        <th>value</th>
+      </thead>
+      <tbody>
+        {% for key_i, item_i in account.items %}
+          <tr>
+            <th>{{ key_i }}</th>
+            <td>
+              {% if item_i is not None %}
+                {{ item_i }}
+              {% endif %}
+            </td>
+          </tr>
+
+        {% endfor %}
+      </tbody>
+    </table>
+  {% endfor %}
+  <hr/>
+{% endfor %}

core/templates/partials/banks-balances-list.html

@@ -0,0 +1,42 @@
+{% load cache %}
+{% load cachalot cache %}
+{% load nsep %}
+
+{% get_last_invalidation 'core.Aggregator' as last %}
+{% include 'mixins/partials/notify.html' %}
+{# cache 600 objects_banks_balances request.user.id object_list type last #}
+
+{% for bank, accounts in object_list.items %}
+  <h1 class="title is-4">{{ bank }}</h1>
+  <table
+    class="table is-fullwidth is-hoverable"
+    hx-target="#{{ bank }}-table"
+    id="{{ bank }}-table"
+    hx-swap="outerHTML"
+    hx-trigger="{{ context_object_name_singular }}Event from:body"
+    hx-get="{{ list_url }}">
+    <thead>
+      <th>currency</th>
+      <th>balance</th>
+      <th>id</th>
+
+    </thead>
+    {% for account in accounts %}
+      <tr>
+        <td>{{ account.currency }}</td>
+        <td>{{ account.balance }}</td>
+        <td>
+          <a
+            class="has-text-grey"
+            onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ account.account_id }}');">
+            <span class="icon" data-tooltip="Copy to clipboard">
+              <i class="fa-solid fa-copy" aria-hidden="true"></i>
+            </span>
+          </a>
+        </td>
+      </tr>
+    {% endfor %}
+
+  </table>
+{% endfor %}
+{# endcache #}

core/templates/partials/banks-currencies-list.html

@@ -0,0 +1,73 @@
+{% load cache %}
+{% load cachalot cache %}
+{% load nsep %}
+
+{% get_last_invalidation 'core.Aggregator' as last %}
+{% include 'mixins/partials/notify.html' %}
+{# cache 600 objects_banks_currencies request.user.id object_list type last #}
+
+{% for bank, accounts in object_list.items %}
+  <h1 class="title is-4">{{ bank.0 }} <code>{{ bank.1 }}</code>
+    <a
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-get="{% url 'requisition_update' type=type aggregator_id=bank.2 req_id=bank.1 %}"
+      hx-trigger="click"
+      hx-target="#{{ type }}s-here"
+      hx-swap="innerHTML">
+      <span class="icon has-text-black" data-tooltip="Configure">
+        <i class="fa-solid fa-wrench"></i>
+      </span>
+    </a>
+  </h1>
+  <table
+    class="table is-fullwidth is-hoverable"
+    hx-target="#{{ bank }}-table"
+    id="{{ bank }}-table"
+    hx-swap="outerHTML"
+    hx-trigger="{{ context_object_name_singular }}Event from:body"
+    hx-get="{{ list_url }}">
+    <thead>
+      <th>currency</th>
+      <th>owner</th>
+      <th>details</th>
+      <th>payment</th>
+      <th>id</th>
+      <th>actions</th>
+
+    </thead>
+    {% for account in accounts %}
+      <tr>
+        <td>{{ account.currency }}</td>
+        <td>{{ account.ownerName }}</td>
+        <td>{{ account.details|default_if_none:"—" }}</td>
+        <td>
+          {% for item in account.account_number.values %}
+            <code>{{ item|default_if_none:"—" }}</code>
+          {% endfor %}
+        </td>
+        <td>
+          <a
+            class="has-text-grey"
+            onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ account.account_id }}');">
+            <span class="icon" data-tooltip="Copy to clipboard">
+              <i class="fa-solid fa-copy" aria-hidden="true"></i>
+            </span>
+          </a>
+        </td>
+        <td>
+          <a href="{% url 'transactions' type='page' account_id=account.account_id aggregator_id=account.aggregator_id %}"><button
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-table-list"></i>
+              </span>
+            </span>
+          </button>
+          </a>
+        </td>
+      </tr>
+    {% endfor %}
+
+  </table>
+{% endfor %}
+{# endcache #}

core/templates/partials/banks-transactions-list.html

@@ -0,0 +1,70 @@
+{% load cache %}
+{% load cachalot cache %}
+{% get_last_invalidation 'core.Aggregator' as last %}
+{% include 'mixins/partials/notify.html' %}
+{# cache 600 objects_banks_transactions request.user.id object_list type last #}
+
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>id</th>
+    <th>ts</th>
+    <th>recipient</th>
+    <th>sender</th>
+    <th>amount</th>
+    <th>currency</th>
+    <th>reference</th>
+    <th>state</th>
+  </thead>
+  {% for item in object_list %}
+    <tr class="
+      {% if item.proprietaryBankTransactionCode == 'EXCHANGE' %}has-background-grey-light
+      {% elif item.amount < 0 %}has-background-danger-light
+      {% elif item.amount > 0 %}has-background-success-light
+      {% endif %}">
+      <td>
+        <a
+          class="has-text-grey"
+          onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ item.transaction_id }}');">
+          <span class="icon" data-tooltip="Copy to clipboard">
+            <i class="fa-solid fa-copy" aria-hidden="true"></i>
+          </span>
+        </a>
+      </td>
+      <td>{{ item.ts }}</td>
+      <td>
+        {{ item.creditorName }}
+        {% for item in item.creditorAccount.values %}
+          {{ item|default_if_none:"—" }}
+        {% endfor %}
+      </td>
+      <td>
+        {{ item.debtorName }}
+        {% for item in item.debtorAccount.values %}
+          {{ item|default_if_none:"—" }}
+        {% endfor %}
+      </td>
+      <td>{{ item.amount }}</td>
+      <td>{{ item.currency }}</td>
+      <td>{{ item.reference }}</td>
+      <td>
+        {% if item.state == 'pending' %}
+          <span class="icon has-text-warning" data-tooltip="Pending">
+            <i class="fa-solid fa-hourglass" aria-hidden="true"></i>
+          </span>
+        {% elif item.state == 'booked' %}
+          <span class="icon has-text-success" data-tooltip="Booked">
+            <i class="fa-solid fa-check" aria-hidden="true"></i>
+          </span>
+        {% endif %}
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>
+{# endcache #}