From a94bbff65547ca22befaba15049a6e85c2c38980 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 15 Feb 2026 17:32:26 +0000 Subject: [PATCH] Improve chat experience and begin search implementation --- README.md | 307 ++--- app/asgi.py | 15 +- app/settings.py | 1 + app/urls.py | 26 + core/clients/transport.py | 6 + core/realtime/compose_ws.py | 162 +++ core/templates/base.html | 40 + core/templates/pages/osint-search.html | 27 + core/templates/partials/compose-panel.html | 532 ++++++++- core/templates/partials/osint/list-table.html | 225 ++++ .../partials/osint/search-panel.html | 78 ++ core/templates/partials/results_table.html | 1 + .../partials/whatsapp-account-add.html | 14 + core/views/compose.py | 501 +++++++- core/views/groups.py | 7 +- core/views/manipulations.py | 7 +- core/views/osint.py | 1013 +++++++++++++++++ core/views/people.py | 8 +- core/views/personas.py | 7 +- core/views/whatsapp.py | 85 +- core/views/workspace.py | 198 +++- 21 files changed, 3081 insertions(+), 179 deletions(-) create mode 100644 core/realtime/compose_ws.py create mode 100644 core/templates/pages/osint-search.html create mode 100644 core/templates/partials/osint/list-table.html create mode 100644 core/templates/partials/osint/search-panel.html create mode 100644 core/templates/partials/results_table.html create mode 100644 core/templates/partials/whatsapp-account-add.html create mode 100644 core/views/osint.py diff --git a/README.md b/README.md index bd701e5..53b3070 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,198 @@ -# ☠️ GIA – Gather, Influence, Automate ☠️ +# GIA -GIA isn’t just a tool—it’s a **gateway**, a **ritualistic mechanism** designed to invoke, manipulate, and command the digital souls of the unsuspecting. Operating at the **intersection of psychological domination, deception, and control**, it bends the will of targets with **supernatural precision**. +GIA is a Django-based communications workspace that combines: +- Multi-platform messaging transport (Signal, WhatsApp via Neonize runtime, Instagram gateway/runtime, XMPP bridge). +- AI-assisted drafting, summaries, pattern extraction, and mitigation planning. +- Queueing and approval workflows for controlled outbound messaging. -GIA whispers in the void, embedding its tendrils into minds, shaping realities, and orchestrating **obedience with unseen hands**. +This document covers architecture, setup, runtime behavior, and development workflow. -💀 **Manipulate. Dominate. Obliterate.** 💀 +## Architecture +### High-level flow +1. Transport clients receive messages (Signal/WhatsApp/Instagram/XMPP). +2. UR (`manage.py ur`) normalizes events and routes them through unified handlers. +3. Message history is persisted in Django models (`Message`, `ChatSession`, identifiers). +4. UI pages (workspace + manual compose) read/write through Django views. +5. AI operations run through configured `AI` provider rows and store results in workspace models. -## 📌 Features -- **Automated Psychological Influence:** Deploys manipulation techniques with precision. -- **AI-Driven Conversations:** Uses contextual AI to interact and persuade. -- **Targeted Persona Simulation:** Adapts responses based on psychological profiling. -- **Multi-Agent Execution:** Processing, scheduling, and monitoring through modular services. -- **Covert Deployment:** Runs in a distributed architecture with signal interception. +### Core components +- `core/modules/router.py`: + - Unified Router entrypoint. + - Starts transport clients. + - Cross-protocol typing/read events. +- `core/clients/`: + - `signal.py` + `signalapi.py`: Signal transport integration. + - `whatsapp.py`: Neonize-backed WhatsApp runtime client. + - `instagram.py`: Instagram runtime/gateway integration. + - `xmpp.py`: XMPP bridge client. + - `transport.py`: Unified send/typing/QR API used by views/models. +- `core/messaging/`: + - AI execution wrapper. + - Message history persistence and enrichment helpers. +- `core/views/`: + - `workspace.py`: AI workspace, insights, mitigation plans/artifacts. + - `compose.py`: Manual chat mode, AI drafts/summary/engage shortcuts. + - `signal.py`, `whatsapp.py`, `instagram.py`: Service setup/account pages. -## 🚀 Quickstart Guide +### Data model highlights +- `Person` + `PersonIdentifier`: + - Identity layer across services (`signal`, `whatsapp`, `instagram`, `xmpp`). +- `ChatSession` + `Message`: + - Conversation storage. + - Delivery/read metadata fields on `Message`. +- `WorkspaceConversation`: + - AI workspace context per person/thread. + - Stability + commitment scores and confidence. +- `WorkspaceMetricSnapshot`: + - Historical metric snapshots for trend charts. +- Pattern mitigation models: + - `PatternMitigationPlan`, `Rule`, `Game`, `Correction`, exports, and auto settings. -### 🔧 Setting Up the Environment -GIA runs inside **Podman containers**. Ensure you have **Podman** and **Podman Compose** installed. Docker might work. Results may vary. +## Runtime services -1. **Clone the repository** - ```shell - ❯ git clone https://github.com/your-repo/gia.git - ❯ cd gia - ``` +`docker-compose.yml` defines these primary services: +- `app`: Django web process. +- `ur`: Unified Router runtime (transport clients). +- `scheduling`: Scheduled jobs. +- `migration`: Startup migrations. +- `collectstatic`: Static asset collection. +- `redis`: Cache/runtime signaling. +- `signal-cli-rest-api`: Signal backend. -2. Set up the environment variables - ```shell - ❯ cp stack.env.example stack.env - ``` +## Installation -3. Edit `stack.env` +### Prerequisites +- Podman + Podman Compose (preferred in this repo), or Docker Compose. +- Linux environment with bind-mount access for configured paths. -4. Build and start the containers - ```shell - ❯ make build - ❯ make run - ``` - -5. Run database migrations - ```shell - ❯ make migrate - ``` - -6. Create a superuser for Django Admin (optional but recommended) - ```shell - ❯ make auth - ``` - -7. Monitor logs - ```shell - ❯ make log - ``` - -## ⚙️ Deployment & Architecture -### 🏗️ Services Overview -| Service | Description | -|----------------|-------------| -| **app** | Main application container running **Uvicorn** for API handling. | -| **db** | ManticoreSearch-based database backend. | -| **redis** | Message queue for task distribution. | -| **signal-cli** | Handles Signal communications. | -| **processing** | Processes incoming messages and executes manipulations. | -| **scheduling** | Handles timed tasks and influence scheduling. | -| **migration** | Runs database migrations automatically on startup. | -| **collectstatic** | Collects static files for Django before launch. | - -## 🔥 Running Commands in a Container - -You can execute management commands inside the app container using: - ```shell - ❯ docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py " - ``` - -## 🛑 Stopping GIA -To stop all running services: - ```shell - ❯ make stop - ``` - -## 🕵️‍♂️ Operational Modes -GIA runs in different operation modes set via OPERATION: -| Mode | Description | -|--------|-------------| -| **uwsgi** | Runs behind Nginx for production. | -| **dev** | Direct execution via Django's built-in server (development mode). | - - -The default Podman entrypoint dynamically selects the correct process based on OPERATION. -Be sure to uncomment nginx if using dev, as the shipped setup expects an external `nginx` instance to point to the GIA `uwsgi` sock: -``` -location / { - include include/xf-only.conf; - include /etc/nginx/uwsgi_params; - uwsgi_pass unix:///code/vrun/uwsgi-gia.sock; - uwsgi_param Host $host; - uwsgi_param X-Real-IP $remote_addr; - uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for; - uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto; -} -location /static { - alias /code/xf/GIA/static/; -} +### 1) Clone and configure +```bash +git clone GIA +cd GIA +cp stack.env.example stack.env ``` -## 🔄 Persistent Data & Storage -| Mount Path (Host) | Purpose | -|---------------------------|-------------| -| **docker/uwsgi.ini** | Configuration for **uWSGI** execution. | -| **db.sqlite3** | SQLite database storage. | -| **/code/vrun/** | Sockets shared between services. | -| **signal-cli-config/** | Stores **Signal CLI** configuration and keys. | +Edit `stack.env` with at least: +- `SECRET_KEY` +- `ALLOWED_HOSTS` +- `CSRF_TRUSTED_ORIGINS` +- `SIGNAL_NUMBER` (if Signal enabled) +- `WHATSAPP_ENABLED=true|false` +- `INSTAGRAM_ENABLED=true|false` +- XMPP values if XMPP bridge is enabled -## 🔧 Additional Configuration -### Django Environment Variables +### 2) Build and start +```bash +make build +make run +``` -The following are required for proper operation: - ```shell - APP_PORT=5006 - REPO_DIR=. - APP_LOCAL_SETTINGS=./app/local_settings.py - APP_DATABASE_FILE=./db.sqlite3 - DOMAIN=example.com - URL=https://example.com - ALLOWED_HOSTS=example.com - NOTIFY_TOPIC=example-topic - CSRF_TRUSTED_ORIGINS=https://example.com - DEBUG=y - SECRET_KEY= - STATIC_ROOT=/code/static - REGISTRATION_OPEN=0 - OPERATION=uwsgi - BILLING_ENABLED=0 - ``` +### 3) Run migrations and create admin user +```bash +make migrate +make auth +``` -These can be set inside `stack.env`. +### 4) Follow logs +```bash +make log +``` -## ⚠️ Legal Disclaimer +## Local developer commands -**GIA is provided for research and educational purposes only.** -The developers and contributors are not responsible for any **misuse, illegal activities, or consequences** resulting from the use of this software. +```bash +make makemigrations +make migrate +make test +``` -By using GIA, you **acknowledge and agree** that: -- You are solely responsible for your actions and compliance with applicable laws. -- The software should not be used for unauthorized surveillance, coercion, or unethical influence. -- The authors disclaim all liability for damages resulting from its use. +Run ad-hoc manage commands: +```bash +docker-compose --env-file=stack.env run --rm app \ + sh -c ". /venv/bin/activate && python manage.py " +``` -GIA is a tool that must be used responsibly. **Do not deploy it where it violates privacy laws, cybersecurity regulations, or ethical guidelines.** +## Service setup pages + +- Signal: `/services/signal/` +- WhatsApp: `/services/whatsapp/` +- Instagram: `/services/instagram/` + +### WhatsApp pairing (Neonize) +WhatsApp account linking expects a real Neonize runtime QR: +1. Start stack with `WHATSAPP_ENABLED=true`. +2. Ensure `ur` is running and WhatsApp runtime initialized. +3. Open `Services -> WhatsApp -> Add Account`. +4. Scan QR from WhatsApp Linked Devices. + +If runtime QR is not available yet, UI returns a clear warning instead of a synthetic QR. + +## Manual compose mode + +- Route: `/compose/page/?service=&identifier=[&person=]` +- Features: + - Chat thread view. + - Send with two-step failsafe (`Arm Send` + `Confirm Intent`). + - AI overlay tools: + - Draft suggestions. + - Conversation summary. + - Quick engage send (shared framing, with its own two-step failsafe). + - Click-outside dismiss behavior for AI overlays. + +### Live updates and traffic minimization +- WebSocket endpoint: `/ws/compose/thread/` (ASGI path). +- Compose clients open one persistent WS per panel and receive incremental updates. +- HTTP polling remains as a fallback at a slower interval when WS is unavailable. + +This reduces repeated client GET requests compared with short polling loops. + +## AI workspace + +Route: `/ai/workspace/` + +Key capabilities: +- Summary, draft, pattern extraction. +- Mitigation planning and artifacts (rules/games/corrections). +- Insight metrics: + - Detail pages per metric. + - All-graphs page using Chart.js. + - Scoring help page describing formulas and interpretation. + +## Configuration notes + +- `app/local_settings.py` is imported by `app/settings.py`. +- Redis cache is expected at `/var/run/gia-redis.sock` unless overridden. +- Service runtime URLs are read by `core/clients/transport.py`: + - `SIGNAL_HTTP_URL` + - `WHATSAPP_HTTP_URL` + - `INSTAGRAM_HTTP_URL` + +## ASGI / WebSocket note + +`app/asgi.py` now multiplexes: +- HTTP to Django ASGI app. +- WebSocket path `/ws/compose/thread/` to compose realtime handler. + +Use an ASGI-capable server in environments where WebSockets are required. + +## Troubleshooting + +### “Send failed. Check service account state.” +- Verify service account is linked and runtime is connected. +- Check `ur` and service logs (`make log`). + +### WhatsApp QR not available +- Confirm `WHATSAPP_ENABLED=true`. +- Confirm Neonize initializes in `ur` logs. +- Retry Add Account after runtime is ready. + +### No compose realtime updates +- Verify ASGI deployment path and websocket upgrade support. +- Fallback polling should still update messages on interval. + +## Security and operational safety + +- Outbound actions in manual compose/engage are guarded by two-step failsafe toggles. +- Queue workflow exists for additional human approval in managed flows. +- Deploy only with lawful consent and compliant data handling for your jurisdiction. diff --git a/app/asgi.py b/app/asgi.py index 410ea53..03f7c1c 100644 --- a/app/asgi.py +++ b/app/asgi.py @@ -13,4 +13,17 @@ from django.core.asgi import get_asgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") -application = get_asgi_application() +django_asgi_app = get_asgi_application() + + +async def application(scope, receive, send): + if scope.get("type") == "websocket": + path = scope.get("path", "") + if path == "/ws/compose/thread/": + from core.realtime.compose_ws import compose_ws_application + + await compose_ws_application(scope, receive, send) + return + await send({"type": "websocket.close", "code": 4404}) + return + await django_asgi_app(scope, receive, send) diff --git a/app/settings.py b/app/settings.py index c7164aa..d1182b3 100644 --- a/app/settings.py +++ b/app/settings.py @@ -96,6 +96,7 @@ MIDDLEWARE = [ ] ROOT_URLCONF = "app.urls" +ASGI_APPLICATION = "app.asgi.application" TEMPLATES = [ { diff --git a/app/urls.py b/app/urls.py index 7b97b61..4c9e78d 100644 --- a/app/urls.py +++ b/app/urls.py @@ -30,6 +30,7 @@ from core.views import ( manipulations, messages, notifications, + osint, people, personas, queues, @@ -128,6 +129,26 @@ urlpatterns = [ compose.ComposeSend.as_view(), name="compose_send", ), + path( + "compose/drafts/", + compose.ComposeDrafts.as_view(), + name="compose_drafts", + ), + path( + "compose/summary/", + compose.ComposeSummary.as_view(), + name="compose_summary", + ), + path( + "compose/engage/preview/", + compose.ComposeEngagePreview.as_view(), + name="compose_engage_preview", + ), + path( + "compose/engage/send/", + compose.ComposeEngageSend.as_view(), + name="compose_engage_send", + ), path( "compose/thread/", compose.ComposeThread.as_view(), @@ -254,6 +275,11 @@ urlpatterns = [ ais.AIList.as_view(), name="ais", ), + path( + "search//", + osint.OSINTSearch.as_view(), + name="osint_search", + ), path( "ai//create/", ais.AICreate.as_view(), diff --git a/core/clients/transport.py b/core/clients/transport.py index c7ab7d9..ea4c2ab 100644 --- a/core/clients/transport.py +++ b/core/clients/transport.py @@ -397,6 +397,12 @@ def get_link_qr(service: str, device_name: str): if cached: return cached + if service_key == "whatsapp": + raise RuntimeError( + "Neonize has not provided a pairing QR yet. " + "Ensure UR is running with WHATSAPP_ENABLED=true and retry." + ) + token = secrets.token_urlsafe(24) uri = f"gia://{service_key}/link?device={device}&token={token}" update_runtime_state( diff --git a/core/realtime/compose_ws.py b/core/realtime/compose_ws.py new file mode 100644 index 0000000..b9ce47d --- /dev/null +++ b/core/realtime/compose_ws.py @@ -0,0 +1,162 @@ +import asyncio +import json +import time +from datetime import datetime, timezone as dt_timezone +from urllib.parse import parse_qs + +from asgiref.sync import sync_to_async +from django.core import signing + +from core.models import ChatSession, Message, PersonIdentifier +from core.views.compose import COMPOSE_WS_TOKEN_SALT + + +def _safe_int(value, default=0): + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _fmt_ts(ts_value): + try: + dt = datetime.fromtimestamp(int(ts_value) / 1000, tz=dt_timezone.utc) + return dt.strftime("%H:%M") + except Exception: + return str(ts_value or "") + + +def _serialize_message(msg): + author = str(msg.custom_author or "").strip() + return { + "id": str(msg.id), + "ts": int(msg.ts or 0), + "display_ts": _fmt_ts(msg.ts), + "text": str(msg.text or ""), + "author": author, + "outgoing": author.upper() in {"USER", "BOT"}, + } + + +def _load_since(user_id, service, identifier, person_id, after_ts, limit): + person_identifier = None + if person_id: + person_identifier = ( + PersonIdentifier.objects.filter( + user_id=user_id, + person_id=person_id, + service=service, + ).first() + or PersonIdentifier.objects.filter( + user_id=user_id, + person_id=person_id, + ).first() + ) + if person_identifier is None and identifier: + person_identifier = PersonIdentifier.objects.filter( + user_id=user_id, + service=service, + identifier=identifier, + ).first() + if person_identifier is None: + return {"messages": [], "last_ts": after_ts} + + session = ChatSession.objects.filter( + user_id=user_id, + identifier=person_identifier, + ).first() + if session is None: + return {"messages": [], "last_ts": after_ts} + + qs = Message.objects.filter( + user_id=user_id, + session=session, + ).order_by("ts") + if after_ts > 0: + qs = qs.filter(ts__gt=after_ts) + + rows = list(qs[: max(10, min(limit, 200))]) + newest = ( + Message.objects.filter(user_id=user_id, session=session) + .order_by("-ts") + .values_list("ts", flat=True) + .first() + ) + return { + "messages": [_serialize_message(row) for row in rows], + "last_ts": int(newest or after_ts or 0), + } + + +async def compose_ws_application(scope, receive, send): + if scope.get("type") != "websocket": + return + + query_string = (scope.get("query_string") or b"").decode("utf-8", errors="ignore") + params = parse_qs(query_string) + token = (params.get("token") or [""])[0] + try: + payload = signing.loads(token, salt=COMPOSE_WS_TOKEN_SALT) + except Exception: + await send({"type": "websocket.close", "code": 4401}) + return + + if _safe_int(payload.get("exp")) < int(time.time()): + await send({"type": "websocket.close", "code": 4401}) + return + + user_id = _safe_int(payload.get("u")) + service = str(payload.get("s") or "").strip() + identifier = str(payload.get("i") or "").strip() + person_id = str(payload.get("p") or "").strip() + + if user_id <= 0 or (not identifier and not person_id): + await send({"type": "websocket.close", "code": 4401}) + return + + await send({"type": "websocket.accept"}) + last_ts = 0 + limit = 100 + + while True: + event = None + try: + event = await asyncio.wait_for(receive(), timeout=1.2) + except asyncio.TimeoutError: + event = None + + if event and event.get("type") == "websocket.disconnect": + break + if event and event.get("type") == "websocket.receive": + try: + body = json.loads(event.get("text") or "{}") + except Exception: + body = {} + if body.get("kind") == "sync": + last_ts = max(last_ts, _safe_int(body.get("last_ts"), 0)) + + payload = await sync_to_async(_load_since)( + user_id=user_id, + service=service, + identifier=identifier, + person_id=person_id, + after_ts=last_ts, + limit=limit, + ) + messages = payload.get("messages") or [] + latest = _safe_int(payload.get("last_ts"), last_ts) + if messages: + last_ts = max(last_ts, latest) + await send( + { + "type": "websocket.send", + "text": json.dumps( + { + "messages": messages, + "last_ts": last_ts, + } + ), + } + ) + else: + last_ts = max(last_ts, latest) diff --git a/core/templates/base.html b/core/templates/base.html index d91376a..62c605f 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -212,6 +212,43 @@ z-index: 39 !important; } + .osint-table-shell { + border: 1px solid rgba(127, 127, 127, 0.2); + border-radius: 14px; + padding: 0.9rem; + background: rgba(255, 255, 255, 0.45); + } + + .osint-table-toolbar { + margin-bottom: 0.75rem; + } + + .osint-results-table-wrap { + border-radius: 10px; + overflow: auto; + } + + .osint-results-table th { + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.05em; + } + + .osint-sort-link { + color: inherit; + display: inline-flex; + align-items: center; + gap: 0.25rem; + } + + .osint-sort-link:hover { + color: #3273dc; + } + + .osint-search-form .button.is-fullwidth { + width: 100%; + } + @@ -235,6 +272,9 @@ Home {% if user.is_authenticated %} + + Search +
diff --git a/core/templates/pages/osint-search.html b/core/templates/pages/osint-search.html new file mode 100644 index 0000000..4ea94c2 --- /dev/null +++ b/core/templates/pages/osint-search.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} + +{% block content %} +
+
+
+
+

Search

+

+ Search across OSINT objects with sortable, paginated results. +

+
+
+ +
+
+ {% include "partials/osint/search-panel.html" %} +
+
+{% endblock %} diff --git a/core/templates/partials/compose-panel.html b/core/templates/partials/compose-panel.html index 42cfd1d..4b1f6f0 100644 --- a/core/templates/partials/compose-panel.html +++ b/core/templates/partials/compose-panel.html @@ -14,6 +14,18 @@

+ + + AI Workspace @@ -25,6 +37,46 @@ {% include "partials/compose-send-status.html" %}
+ +
+ data-last-ts="{{ last_ts }}" + data-ws-url="{{ compose_ws_url }}" + data-drafts-url="{{ compose_drafts_url }}" + data-summary-url="{{ compose_summary_url }}" + data-engage-preview-url="{{ compose_engage_preview_url }}" + data-engage-send-url="{{ compose_engage_send_url }}"> {% for msg in serialized_messages %}
@@ -59,9 +116,19 @@ + + {% if person %} {% endif %} +
+ + +
- @@ -79,6 +146,7 @@ @@ -192,6 +342,10 @@ return; } + const statusBox = document.getElementById(panelId + "-status"); + const popover = document.getElementById(panelId + "-popover"); + const csrfToken = "{{ csrf_token }}"; + window.giaComposePanels = window.giaComposePanels || {}; const previousState = window.giaComposePanels[panelId]; if (previousState && previousState.timer) { @@ -200,7 +354,17 @@ if (previousState && previousState.eventHandler) { document.body.removeEventListener("composeMessageSent", previousState.eventHandler); } - const panelState = { timer: null, polling: false }; + if (previousState && previousState.docClickHandler) { + document.removeEventListener("mousedown", previousState.docClickHandler); + } + const panelState = { + timer: null, + polling: false, + socket: null, + websocketReady: false, + activePanel: null, + engageToken: "" + }; window.giaComposePanels[panelId] = panelState; const toInt = function (value) { @@ -259,8 +423,20 @@ thread.appendChild(row); }; + const appendMessages = function (messages, forceScroll) { + const shouldStick = nearBottom() || forceScroll; + (messages || []).forEach(function (msg) { + appendBubble(msg); + lastTs = Math.max(lastTs, toInt(msg.ts)); + }); + thread.dataset.lastTs = String(lastTs); + if ((messages || []).length > 0) { + scrollToBottom(shouldStick); + } + }; + const poll = async function (forceScroll) { - if (panelState.polling) { + if (panelState.polling || panelState.websocketReady) { return; } panelState.polling = true; @@ -273,28 +449,19 @@ } params.set("limit", thread.dataset.limit || "60"); params.set("after_ts", String(lastTs)); - const url = thread.dataset.pollUrl + "?" + params.toString(); - const response = await fetch(url, { + const response = await fetch(thread.dataset.pollUrl + "?" + params.toString(), { method: "GET", credentials: "same-origin", - headers: { Accept: "application/json" }, + headers: { Accept: "application/json" } }); if (!response.ok) { return; } const payload = await response.json(); - const messages = Array.isArray(payload.messages) ? payload.messages : []; - const shouldStick = nearBottom() || forceScroll; - messages.forEach(function (msg) { - appendBubble(msg); - lastTs = Math.max(lastTs, toInt(msg.ts)); - }); + appendMessages(payload.messages || [], forceScroll); if (payload.last_ts !== undefined && payload.last_ts !== null) { lastTs = Math.max(lastTs, toInt(payload.last_ts)); - } - thread.dataset.lastTs = String(lastTs); - if (messages.length > 0) { - scrollToBottom(shouldStick); + thread.dataset.lastTs = String(lastTs); } } catch (err) { console.debug("compose poll error", err); @@ -303,9 +470,324 @@ } }; + const setupWebSocket = function () { + const wsPath = thread.dataset.wsUrl || ""; + if (!wsPath || !window.WebSocket) { + return; + } + const protocol = window.location.protocol === "https:" ? "wss://" : "ws://"; + const socketUrl = protocol + window.location.host + wsPath; + try { + const socket = new WebSocket(socketUrl); + panelState.socket = socket; + socket.onopen = function () { + panelState.websocketReady = true; + try { + socket.send(JSON.stringify({ kind: "sync", last_ts: lastTs })); + } catch (err) { + // Ignore. + } + }; + socket.onmessage = function (event) { + try { + const payload = JSON.parse(event.data || "{}"); + appendMessages(payload.messages || [], false); + if (payload.last_ts !== undefined && payload.last_ts !== null) { + lastTs = Math.max(lastTs, toInt(payload.last_ts)); + thread.dataset.lastTs = String(lastTs); + } + } catch (err) { + console.debug("compose websocket payload error", err); + } + }; + socket.onclose = function () { + panelState.websocketReady = false; + }; + socket.onerror = function () { + panelState.websocketReady = false; + }; + } catch (err) { + panelState.websocketReady = false; + } + }; + + const manualArm = form.querySelector(".manual-arm"); + const manualConfirm = form.querySelector(".manual-confirm"); + const armInput = form.querySelector("input[name='failsafe_arm']"); + const confirmInput = form.querySelector("input[name='failsafe_confirm']"); + const sendButton = form.querySelector(".compose-send-btn"); + const updateManualSafety = function () { + const arm = !!(manualArm && manualArm.checked); + const confirm = !!(manualConfirm && manualConfirm.checked); + if (armInput) { + armInput.value = arm ? "1" : "0"; + } + if (confirmInput) { + confirmInput.value = confirm ? "1" : "0"; + } + if (sendButton) { + sendButton.disabled = !(arm && confirm); + } + }; + if (manualArm) { + manualArm.addEventListener("change", updateManualSafety); + } + if (manualConfirm) { + manualConfirm.addEventListener("change", updateManualSafety); + } + updateManualSafety(); + + const setStatus = function (message, level) { + if (!statusBox) { + return; + } + if (!message) { + statusBox.innerHTML = ""; + return; + } + statusBox.innerHTML = '
' + message + "
"; + }; + + const hideAllCards = function () { + if (!popover) { + return; + } + popover.classList.add("is-hidden"); + popover.querySelectorAll(".compose-ai-card").forEach(function (card) { + card.classList.remove("is-active"); + }); + panelState.activePanel = null; + }; + + const showCard = function (kind) { + if (!popover) { + return null; + } + popover.classList.remove("is-hidden"); + let active = null; + popover.querySelectorAll(".compose-ai-card").forEach(function (card) { + const isActive = card.dataset.kind === kind; + card.classList.toggle("is-active", isActive); + if (isActive) { + active = card; + } + }); + panelState.activePanel = kind; + return active; + }; + + const queryParams = function () { + const params = new URLSearchParams(); + params.set("service", thread.dataset.service || ""); + params.set("identifier", thread.dataset.identifier || ""); + if (thread.dataset.person) { + params.set("person", thread.dataset.person); + } + params.set("limit", thread.dataset.limit || "60"); + return params; + }; + + const setCardLoading = function (card, loading) { + const loadingNode = card.querySelector(".compose-ai-loading"); + const contentNode = card.querySelector(".compose-ai-content"); + if (loadingNode) { + loadingNode.classList.toggle("is-hidden", !loading); + } + if (contentNode && loading) { + contentNode.innerHTML = ""; + } + }; + + const loadDrafts = async function () { + const card = showCard("drafts"); + if (!card) { + return; + } + setCardLoading(card, true); + try { + const response = await fetch(thread.dataset.draftsUrl + "?" + queryParams().toString(), { + method: "GET", + credentials: "same-origin", + headers: { Accept: "application/json" } + }); + const payload = await response.json(); + setCardLoading(card, false); + if (!payload.ok) { + card.querySelector(".compose-ai-content").textContent = payload.error || "Failed to load drafts."; + return; + } + const drafts = Array.isArray(payload.drafts) ? payload.drafts : []; + const container = card.querySelector(".compose-ai-content"); + container.innerHTML = ""; + drafts.forEach(function (item) { + const button = document.createElement("button"); + button.type = "button"; + button.className = "button is-light compose-draft-option"; + const strong = document.createElement("strong"); + strong.textContent = String(item.label || "Option") + ": "; + const span = document.createElement("span"); + span.textContent = String(item.text || ""); + button.appendChild(strong); + button.appendChild(span); + button.addEventListener("click", function () { + textarea.value = String(item.text || ""); + autosize(); + textarea.focus(); + hideAllCards(); + }); + container.appendChild(button); + }); + } catch (err) { + setCardLoading(card, false); + card.querySelector(".compose-ai-content").textContent = "Failed to load drafts."; + } + }; + + const loadSummary = async function () { + const card = showCard("summary"); + if (!card) { + return; + } + setCardLoading(card, true); + try { + const response = await fetch(thread.dataset.summaryUrl + "?" + queryParams().toString(), { + method: "GET", + credentials: "same-origin", + headers: { Accept: "application/json" } + }); + const payload = await response.json(); + setCardLoading(card, false); + if (!payload.ok) { + card.querySelector(".compose-ai-content").textContent = payload.error || "Failed to load summary."; + return; + } + card.querySelector(".compose-ai-content").textContent = String(payload.summary || ""); + } catch (err) { + setCardLoading(card, false); + card.querySelector(".compose-ai-content").textContent = "Failed to load summary."; + } + }; + + const bindEngageControls = function (card) { + const arm = card.querySelector(".engage-arm"); + const confirm = card.querySelector(".engage-confirm"); + const send = card.querySelector(".engage-send-btn"); + const sync = function () { + send.disabled = !(arm.checked && confirm.checked && panelState.engageToken); + }; + arm.addEventListener("change", sync); + confirm.addEventListener("change", sync); + send.addEventListener("click", async function () { + if (!panelState.engageToken) { + return; + } + const formData = new URLSearchParams(); + formData.set("service", thread.dataset.service || ""); + formData.set("identifier", thread.dataset.identifier || ""); + if (thread.dataset.person) { + formData.set("person", thread.dataset.person); + } + formData.set("engage_token", panelState.engageToken); + formData.set("failsafe_arm", arm.checked ? "1" : "0"); + formData.set("failsafe_confirm", confirm.checked ? "1" : "0"); + try { + const response = await fetch(thread.dataset.engageSendUrl, { + method: "POST", + credentials: "same-origin", + headers: { + "X-CSRFToken": csrfToken, + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json" + }, + body: formData.toString() + }); + const payload = await response.json(); + if (!payload.ok) { + setStatus(payload.error || "Engage send failed.", "danger"); + return; + } + setStatus(payload.message || "Shared engage sent.", "success"); + hideAllCards(); + poll(true); + } catch (err) { + setStatus("Engage send failed.", "danger"); + } + }); + }; + + const loadEngage = async function () { + const card = showCard("engage"); + if (!card) { + return; + } + setCardLoading(card, true); + panelState.engageToken = ""; + const sendBtn = card.querySelector(".engage-send-btn"); + const arm = card.querySelector(".engage-arm"); + const confirm = card.querySelector(".engage-confirm"); + arm.checked = false; + confirm.checked = false; + sendBtn.disabled = true; + try { + const response = await fetch(thread.dataset.engagePreviewUrl + "?" + queryParams().toString(), { + method: "GET", + credentials: "same-origin", + headers: { Accept: "application/json" } + }); + const payload = await response.json(); + setCardLoading(card, false); + if (!payload.ok) { + card.querySelector(".compose-ai-content").textContent = payload.error || "Failed to load engage preview."; + return; + } + panelState.engageToken = String(payload.token || ""); + let text = String(payload.preview || ""); + if (payload.artifact) { + text = text + "\n\nSource: " + String(payload.artifact); + } + card.querySelector(".compose-ai-content").textContent = text; + sendBtn.disabled = true; + } catch (err) { + setCardLoading(card, false); + card.querySelector(".compose-ai-content").textContent = "Failed to load engage preview."; + } + if (!card.dataset.bound) { + bindEngageControls(card); + card.dataset.bound = "1"; + } + }; + + panel.querySelectorAll(".js-ai-trigger").forEach(function (button) { + button.addEventListener("click", function () { + const kind = button.dataset.kind; + if (panelState.activePanel === kind) { + hideAllCards(); + return; + } + if (kind === "drafts") { + loadDrafts(); + } else if (kind === "summary") { + loadSummary(); + } else if (kind === "engage") { + loadEngage(); + } + }); + }); + + panelState.docClickHandler = function (event) { + if (!panel.contains(event.target)) { + hideAllCards(); + } + }; + document.addEventListener("mousedown", panelState.docClickHandler); + textarea.addEventListener("keydown", function (event) { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); + if (sendButton && sendButton.disabled) { + setStatus("Enable both send safety switches before sending.", "warning"); + return; + } form.requestSubmit(); } }); @@ -325,17 +807,23 @@ document.body.addEventListener("composeMessageSent", panelState.eventHandler); scrollToBottom(true); + setupWebSocket(); panelState.timer = setInterval(function () { if (!document.getElementById(panelId)) { clearInterval(panelState.timer); - document.body.removeEventListener( - "composeMessageSent", - panelState.eventHandler - ); + document.body.removeEventListener("composeMessageSent", panelState.eventHandler); + document.removeEventListener("mousedown", panelState.docClickHandler); + if (panelState.socket) { + try { + panelState.socket.close(); + } catch (err) { + // Ignore. + } + } delete window.giaComposePanels[panelId]; return; } poll(false); - }, 1800); + }, 4000); })(); diff --git a/core/templates/partials/osint/list-table.html b/core/templates/partials/osint/list-table.html new file mode 100644 index 0000000..36b004b --- /dev/null +++ b/core/templates/partials/osint/list-table.html @@ -0,0 +1,225 @@ +{% include 'mixins/partials/notify.html' %} +
+ + {% if osint_show_search %} +
+
+
+ +
+
+
+ +
+
+
+ +
+
+ + Reset + +
+
+ + {% endif %} + +
+ + + + {% for column in osint_columns %} + + {% endfor %} + {% if osint_show_actions %}{% endif %} + + + + {% for row in osint_rows %} + + {% for cell in row.cells %} + + {% endfor %} + {% if osint_show_actions %} + + {% endif %} + + {% empty %} + + + + {% endfor %} + +
+ {% if column.sortable %} + + {{ column.label }} + + {% if column.is_sorted and column.is_desc %} + + {% elif column.is_sorted %} + + {% else %} + + {% endif %} + + + {% else %} + {{ column.label }} + {% endif %} + Actions
+ {% if cell.kind == "id_copy" %} + + + + + {{ cell.value }} + + {% elif cell.kind == "bool" %} + {% if cell.value %} + + + + {% else %} + + + + {% endif %} + {% elif cell.kind == "datetime" %} + {% if cell.value %} + {{ cell.value|date:"M j, Y P" }} + {% else %} + - + {% endif %} + {% else %} + {% if cell.value or cell.value == 0 %} + {{ cell.value }} + {% else %} + - + {% endif %} + {% endif %} + +
+ {% for action in row.actions %} + {% if action.mode == "hx-get" %} + + {% elif action.mode == "hx-delete" %} + + {% elif action.mode == "link" %} + + + + {% endif %} + {% endfor %} +
+
+

No results found.

+
+
+ + {% if osint_pagination.enabled %} +
+ {% endif %} + +

+ {{ osint_result_count }} result{% if osint_result_count != 1 %}s{% endif %} +

+
diff --git a/core/templates/partials/osint/search-panel.html b/core/templates/partials/osint/search-panel.html new file mode 100644 index 0000000..e93493d --- /dev/null +++ b/core/templates/partials/osint/search-panel.html @@ -0,0 +1,78 @@ +
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ + +
+
+ +
+ + + Reset + +
+
+
+
+ +
+ {% include "partials/results_table.html" %} +
+
diff --git a/core/templates/partials/results_table.html b/core/templates/partials/results_table.html new file mode 100644 index 0000000..a5fbc50 --- /dev/null +++ b/core/templates/partials/results_table.html @@ -0,0 +1 @@ +{% include "partials/osint/list-table.html" %} diff --git a/core/templates/partials/whatsapp-account-add.html b/core/templates/partials/whatsapp-account-add.html new file mode 100644 index 0000000..aca2993 --- /dev/null +++ b/core/templates/partials/whatsapp-account-add.html @@ -0,0 +1,14 @@ +{% if object.ok %} + WhatsApp QR code + {% if object.warning %} +

{{ object.warning }}

+ {% endif %} +{% else %} +
+

WhatsApp QR Not Ready.

+

{{ object.error|default:"No Neonize pairing QR is available yet." }}

+ {% if object.warning %} +

{{ object.warning }}

+ {% endif %} +
+{% endif %} diff --git a/core/views/compose.py b/core/views/compose.py index 13ec712..18d311a 100644 --- a/core/views/compose.py +++ b/core/views/compose.py @@ -1,12 +1,15 @@ from __future__ import annotations import hashlib +import re import time from datetime import datetime, timezone as dt_timezone from urllib.parse import urlencode from asgiref.sync import async_to_sync from django.contrib.auth.mixins import LoginRequiredMixin +from django.core import signing +from django.core.cache import cache from django.http import HttpResponseBadRequest, JsonResponse from django.shortcuts import get_object_or_404, render from django.urls import reverse @@ -14,7 +17,21 @@ from django.utils import timezone as dj_timezone from django.views import View from core.clients import transport -from core.models import ChatSession, Message, Person, PersonIdentifier +from core.messaging import ai as ai_runner +from core.messaging.utils import messages_to_string +from core.models import ( + AI, + ChatSession, + Message, + PatternMitigationPlan, + Person, + PersonIdentifier, +) +from core.views.workspace import _build_engage_payload, _parse_draft_options + +COMPOSE_WS_TOKEN_SALT = "compose-ws" +COMPOSE_ENGAGE_TOKEN_SALT = "compose-engage" +COMPOSE_AI_CACHE_TTL = 60 * 30 def _default_service(service: str | None) -> str: @@ -64,6 +81,185 @@ def _serialize_message(msg: Message) -> dict: } +def _owner_name(user) -> str: + return ( + user.first_name + or user.get_full_name().strip() + or user.username + or "Me" + ) + + +def _compose_ws_token(user_id, service, identifier, person_id): + payload = { + "u": int(user_id), + "s": str(service or ""), + "i": str(identifier or ""), + "p": str(person_id) if person_id else "", + "exp": int(time.time()) + (60 * 60 * 12), + } + return signing.dumps(payload, salt=COMPOSE_WS_TOKEN_SALT) + + +def _compose_ai_cache_key(kind, user_id, service, identifier, person_id, last_ts, limit): + raw = "|".join( + [ + str(kind or ""), + str(user_id), + str(service or ""), + str(identifier or ""), + str(person_id or ""), + str(last_ts or 0), + str(limit or 0), + ] + ) + digest = hashlib.sha1(raw.encode("utf-8")).hexdigest() + return f"compose:{kind}:{digest}" + + +def _plain_text(value): + cleaned = re.sub(r"\s+", " ", str(value or "").strip()) + cleaned = re.sub(r"^\s*#{1,6}\s*", "", cleaned) + cleaned = re.sub(r"\*\*(.*?)\*\*", r"\1", cleaned) + cleaned = re.sub(r"`(.*?)`", r"\1", cleaned) + return cleaned.strip() + + +def _engage_body_only(value): + lines = [line.strip() for line in str(value or "").splitlines() if line.strip()] + if lines and lines[0].startswith("**"): + lines = lines[1:] + if lines and lines[0].lower() == "guidance:": + lines = lines[1:] + return _plain_text(" ".join(lines)) + + +def _messages_for_ai(user, person_identifier, limit): + if person_identifier is None: + return [] + session, _ = ChatSession.objects.get_or_create(user=user, identifier=person_identifier) + rows = list( + Message.objects.filter(user=user, session=session) + .select_related("session", "session__identifier", "session__identifier__person") + .order_by("-ts")[:limit] + ) + rows.reverse() + return rows + + +def _fallback_drafts(): + return [ + { + "label": "Soft", + "text": "I want us to stay connected. I am listening and I want to understand your perspective clearly.", + }, + { + "label": "Neutral", + "text": "I hear your point. Let us clarify what each of us means so we can move forward constructively.", + }, + { + "label": "Firm", + "text": "I want to resolve this respectfully. I will continue when we can keep the conversation constructive.", + }, + ] + + +def _build_draft_prompt(owner_name, person_name, transcript): + return [ + { + "role": "system", + "content": ( + "Generate exactly three short reply drafts for a chat. " + "Return labels Soft, Neutral, Firm. " + "Format:\nSoft: ...\nNeutral: ...\nFirm: ...\n" + "Each draft must be one to two sentences, plain text, no markdown." + ), + }, + { + "role": "user", + "content": ( + f"Me: {owner_name}\n" + f"Other: {person_name}\n" + f"Conversation:\n{transcript}" + ), + }, + ] + + +def _build_summary_prompt(owner_name, person_name, transcript): + return [ + { + "role": "system", + "content": ( + "Create a concise conversation summary with three sections. " + "Use this exact structure:\n" + "Headlines:\n- ...\n" + "Patterns:\n- ...\n" + "Suggested Next Message:\n- ...\n" + "Keep each bullet practical and specific." + ), + }, + { + "role": "user", + "content": ( + f"Me: {owner_name}\n" + f"Other: {person_name}\n" + f"Conversation:\n{transcript}" + ), + }, + ] + + +def _build_engage_prompt(owner_name, person_name, transcript): + return [ + { + "role": "system", + "content": ( + "Write one short de-escalating outreach in shared framing. " + "Use 'we/us/our' only. No names. One or two sentences." + ), + }, + { + "role": "user", + "content": ( + f"Me: {owner_name}\n" + f"Other: {person_name}\n" + f"Conversation:\n{transcript}" + ), + }, + ] + + +def _latest_plan_for_person(user, person): + if person is None: + return None + conversation = ( + PatternMitigationPlan.objects.filter( + user=user, + conversation__participants=person, + ) + .select_related("conversation") + .order_by("-updated_at") + .first() + ) + return conversation + + +def _best_engage_source(plan): + if plan is None: + return (None, "") + correction = plan.corrections.order_by("-created_at").first() + if correction: + return (correction, "correction") + rule = plan.rules.order_by("-created_at").first() + if rule: + return (rule, "rule") + game = plan.games.order_by("-created_at").first() + if game: + return (game, "game") + return (None, "") + + def _context_base(user, service, identifier, person): person_identifier = None if person is not None: @@ -143,6 +339,13 @@ def _panel_context( base["identifier"], base["person"].id if base["person"] else None, ) + ws_token = _compose_ws_token( + user_id=request.user.id, + service=base["service"], + identifier=base["identifier"], + person_id=base["person"].id if base["person"] else None, + ) + ws_url = f"/ws/compose/thread/?{urlencode({'token': ws_token})}" unique_raw = f"{base['service']}|{base['identifier']}|{request.user.id}" unique = hashlib.sha1(unique_raw.encode("utf-8")).hexdigest()[:12] @@ -164,6 +367,11 @@ def _panel_context( "render_mode": render_mode, "compose_page_url": urls["page_url"], "compose_widget_url": urls["widget_url"], + "compose_drafts_url": reverse("compose_drafts"), + "compose_summary_url": reverse("compose_summary"), + "compose_engage_preview_url": reverse("compose_engage_preview"), + "compose_engage_send_url": reverse("compose_engage_send"), + "compose_ws_url": ws_url, "ai_workspace_url": ( f"{reverse('ai_workspace')}?person={base['person'].id}" if base["person"] @@ -305,6 +513,285 @@ class ComposeThread(LoginRequiredMixin, View): return JsonResponse(payload) +class ComposeDrafts(LoginRequiredMixin, View): + def get(self, request): + service = _default_service(request.GET.get("service")) + identifier = str(request.GET.get("identifier") or "").strip() + person = None + person_id = request.GET.get("person") + if person_id: + person = get_object_or_404(Person, id=person_id, user=request.user) + if not identifier and person is None: + return JsonResponse({"ok": False, "error": "Missing contact identifier."}) + + base = _context_base(request.user, service, identifier, person) + limit = _safe_limit(request.GET.get("limit") or 60) + messages = _messages_for_ai(request.user, base["person_identifier"], limit) + if not messages: + return JsonResponse( + { + "ok": True, + "cached": False, + "drafts": _fallback_drafts(), + } + ) + + last_ts = int(messages[-1].ts or 0) + cache_key = _compose_ai_cache_key( + "drafts", + request.user.id, + base["service"], + base["identifier"], + base["person"].id if base["person"] else "", + last_ts, + limit, + ) + cached = cache.get(cache_key) + if cached: + return JsonResponse({"ok": True, "cached": True, "drafts": cached}) + + ai_obj = AI.objects.filter(user=request.user).first() + transcript = messages_to_string( + messages, + author_rewrites={ + "USER": _owner_name(request.user), + "BOT": "Assistant", + }, + ) + drafts = _fallback_drafts() + if ai_obj is not None: + try: + result = async_to_sync(ai_runner.run_prompt)( + _build_draft_prompt( + owner_name=_owner_name(request.user), + person_name=base["person"].name if base["person"] else "Other", + transcript=transcript, + ), + ai_obj, + ) + parsed = _parse_draft_options(result) + if parsed: + drafts = parsed + except Exception: + pass + + cache.set(cache_key, drafts, timeout=COMPOSE_AI_CACHE_TTL) + return JsonResponse({"ok": True, "cached": False, "drafts": drafts}) + + +class ComposeSummary(LoginRequiredMixin, View): + def get(self, request): + service = _default_service(request.GET.get("service")) + identifier = str(request.GET.get("identifier") or "").strip() + person = None + person_id = request.GET.get("person") + if person_id: + person = get_object_or_404(Person, id=person_id, user=request.user) + if not identifier and person is None: + return JsonResponse({"ok": False, "error": "Missing contact identifier."}) + + base = _context_base(request.user, service, identifier, person) + limit = _safe_limit(request.GET.get("limit") or 60) + messages = _messages_for_ai(request.user, base["person_identifier"], limit) + if not messages: + return JsonResponse({"ok": True, "cached": False, "summary": ""}) + + last_ts = int(messages[-1].ts or 0) + cache_key = _compose_ai_cache_key( + "summary", + request.user.id, + base["service"], + base["identifier"], + base["person"].id if base["person"] else "", + last_ts, + limit, + ) + cached = cache.get(cache_key) + if cached: + return JsonResponse({"ok": True, "cached": True, "summary": cached}) + + ai_obj = AI.objects.filter(user=request.user).first() + transcript = messages_to_string( + messages, + author_rewrites={ + "USER": _owner_name(request.user), + "BOT": "Assistant", + }, + ) + if ai_obj is None: + fallback = ( + "Headlines:\n" + "- Conversation loaded.\n" + "Patterns:\n" + "- Not enough AI context configured yet.\n" + "Suggested Next Message:\n" + "- I want us to keep this clear and constructive." + ) + cache.set(cache_key, fallback, timeout=COMPOSE_AI_CACHE_TTL) + return JsonResponse({"ok": True, "cached": False, "summary": fallback}) + + try: + summary = async_to_sync(ai_runner.run_prompt)( + _build_summary_prompt( + owner_name=_owner_name(request.user), + person_name=base["person"].name if base["person"] else "Other", + transcript=transcript, + ), + ai_obj, + ) + except Exception as exc: + return JsonResponse({"ok": False, "error": str(exc)}) + + summary = str(summary or "").strip() + cache.set(cache_key, summary, timeout=COMPOSE_AI_CACHE_TTL) + return JsonResponse({"ok": True, "cached": False, "summary": summary}) + + +class ComposeEngagePreview(LoginRequiredMixin, View): + def get(self, request): + service = _default_service(request.GET.get("service")) + identifier = str(request.GET.get("identifier") or "").strip() + person = None + person_id = request.GET.get("person") + if person_id: + person = get_object_or_404(Person, id=person_id, user=request.user) + if not identifier and person is None: + return JsonResponse({"ok": False, "error": "Missing contact identifier."}) + + base = _context_base(request.user, service, identifier, person) + limit = _safe_limit(request.GET.get("limit") or 60) + messages = _messages_for_ai(request.user, base["person_identifier"], limit) + transcript = messages_to_string( + messages, + author_rewrites={ + "USER": _owner_name(request.user), + "BOT": "Assistant", + }, + ) + + owner_name = _owner_name(request.user) + recipient_name = base["person"].name if base["person"] else "Other" + plan = _latest_plan_for_person(request.user, base["person"]) + source_obj, source_kind = _best_engage_source(plan) + + preview = "" + outbound = "" + artifact_label = "AI-generated" + if source_obj is not None: + payload = _build_engage_payload( + source_obj=source_obj, + source_kind=source_kind, + share_target="other", + framing="shared", + context_note="", + owner_name=owner_name, + recipient_name=recipient_name, + ) + preview = str(payload.get("preview") or "").strip() + outbound = _engage_body_only(payload.get("outbound") or "") + artifact_label = f"{source_kind.title()}: {getattr(source_obj, 'title', '')}" + else: + ai_obj = AI.objects.filter(user=request.user).first() + if ai_obj is not None: + try: + generated = async_to_sync(ai_runner.run_prompt)( + _build_engage_prompt(owner_name, recipient_name, transcript), + ai_obj, + ) + outbound = _plain_text(generated) + except Exception: + outbound = "" + if not outbound: + outbound = ( + "We should slow down, clarify what we mean, and respond with care." + ) + preview = f"**Shared Engage** (Correction)\n\nGuidance:\n{outbound}" + + token = signing.dumps( + { + "u": request.user.id, + "s": base["service"], + "i": base["identifier"], + "p": str(base["person"].id) if base["person"] else "", + "outbound": outbound, + "exp": int(time.time()) + (60 * 10), + }, + salt=COMPOSE_ENGAGE_TOKEN_SALT, + ) + return JsonResponse( + { + "ok": True, + "preview": preview, + "outbound": outbound, + "token": token, + "artifact": artifact_label, + } + ) + + +class ComposeEngageSend(LoginRequiredMixin, View): + def post(self, request): + service = _default_service(request.POST.get("service")) + identifier = str(request.POST.get("identifier") or "").strip() + person = None + person_id = request.POST.get("person") + if person_id: + person = get_object_or_404(Person, id=person_id, user=request.user) + if not identifier and person is None: + return JsonResponse({"ok": False, "error": "Missing contact identifier."}) + + failsafe_arm = str(request.POST.get("failsafe_arm") or "").strip() + failsafe_confirm = str(request.POST.get("failsafe_confirm") or "").strip() + if failsafe_arm != "1" or failsafe_confirm != "1": + return JsonResponse( + {"ok": False, "error": "Enable both send safety switches first."} + ) + + token = str(request.POST.get("engage_token") or "").strip() + if not token: + return JsonResponse({"ok": False, "error": "Missing engage token."}) + try: + payload = signing.loads(token, salt=COMPOSE_ENGAGE_TOKEN_SALT) + except Exception: + return JsonResponse({"ok": False, "error": "Invalid engage token."}) + + if int(payload.get("u") or 0) != int(request.user.id): + return JsonResponse({"ok": False, "error": "Token does not match user."}) + if int(payload.get("exp") or 0) < int(time.time()): + return JsonResponse({"ok": False, "error": "Engage token expired."}) + outbound = str(payload.get("outbound") or "").strip() + if not outbound: + return JsonResponse({"ok": False, "error": "Empty engage payload."}) + + base = _context_base(request.user, service, identifier, person) + ts = async_to_sync(transport.send_message_raw)( + base["service"], + base["identifier"], + text=outbound, + attachments=[], + ) + if not ts: + return JsonResponse({"ok": False, "error": "Send failed."}) + + if base["person_identifier"] is not None: + session, _ = ChatSession.objects.get_or_create( + user=request.user, + identifier=base["person_identifier"], + ) + ts_value = int(ts) if str(ts).isdigit() else int(time.time() * 1000) + Message.objects.create( + user=request.user, + session=session, + sender_uuid="", + text=outbound, + ts=ts_value, + delivered_ts=ts_value if str(ts).isdigit() else None, + custom_author="USER", + ) + + return JsonResponse({"ok": True, "message": "Shared engage sent."}) + + class ComposeSend(LoginRequiredMixin, View): def post(self, request): service = _default_service(request.POST.get("service")) @@ -320,6 +807,18 @@ class ComposeSend(LoginRequiredMixin, View): if not identifier and person is None: return HttpResponseBadRequest("Missing contact identifier.") + failsafe_arm = str(request.POST.get("failsafe_arm") or "").strip() + failsafe_confirm = str(request.POST.get("failsafe_confirm") or "").strip() + if failsafe_arm != "1" or failsafe_confirm != "1": + return render( + request, + "partials/compose-send-status.html", + { + "notice_message": "Enable both send safety switches before sending.", + "notice_level": "warning", + }, + ) + text = str(request.POST.get("text") or "").strip() if not text: return render( diff --git a/core/views/groups.py b/core/views/groups.py index 74e4b88..364d374 100644 --- a/core/views/groups.py +++ b/core/views/groups.py @@ -1,15 +1,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin -from mixins.views import ObjectCreate, ObjectDelete, ObjectList, ObjectUpdate +from mixins.views import ObjectCreate, ObjectDelete, ObjectUpdate from core.forms import GroupForm from core.models import Group +from core.views.osint import OSINTListBase from core.util import logs log = logs.get_logger(__name__) -class GroupList(LoginRequiredMixin, ObjectList): - list_template = "partials/group-list.html" +class GroupList(LoginRequiredMixin, OSINTListBase): + osint_scope = "groups" model = Group page_title = "Groups" diff --git a/core/views/manipulations.py b/core/views/manipulations.py index df7b9ce..4f7420e 100644 --- a/core/views/manipulations.py +++ b/core/views/manipulations.py @@ -1,15 +1,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin -from mixins.views import ObjectCreate, ObjectDelete, ObjectList, ObjectUpdate +from mixins.views import ObjectCreate, ObjectDelete, ObjectUpdate from core.forms import ManipulationForm from core.models import Manipulation +from core.views.osint import OSINTListBase from core.util import logs log = logs.get_logger(__name__) -class ManipulationList(LoginRequiredMixin, ObjectList): - list_template = "partials/manipulation-list.html" +class ManipulationList(LoginRequiredMixin, OSINTListBase): + osint_scope = "manipulations" model = Manipulation page_title = "Manipulations" diff --git a/core/views/osint.py b/core/views/osint.py new file mode 100644 index 0000000..f5e5b51 --- /dev/null +++ b/core/views/osint.py @@ -0,0 +1,1013 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from typing import Any, Callable +from urllib.parse import urlencode + +from django.contrib.auth.mixins import LoginRequiredMixin +from django.core.exceptions import FieldDoesNotExist +from django.core.paginator import Paginator +from django.db import models +from django.db.models import Q +from django.http import HttpResponseBadRequest +from django.shortcuts import render +from django.urls import reverse +from django.views import View +from mixins.views import ObjectList + +from core.models import Group, Manipulation, Persona, Person + + +def _context_type(request_type: str) -> str: + return "modal" if request_type == "page" else request_type + + +def _parse_bool(raw: str) -> bool | None: + lowered = raw.strip().lower() + truthy = {"1", "true", "yes", "y", "on", "enabled"} + falsy = {"0", "false", "no", "n", "off", "disabled"} + if lowered in truthy: + return True + if lowered in falsy: + return False + return None + + +def _preferred_related_text_field(model: type[models.Model]) -> str | None: + preferred = ("name", "alias", "title", "identifier", "model") + model_fields = {f.name: f for f in model._meta.get_fields()} + for candidate in preferred: + field = model_fields.get(candidate) + if isinstance(field, (models.CharField, models.TextField)): + return candidate + return None + + +def _url_with_query(base_url: str, query: dict[str, Any]) -> str: + params = {} + for key, value in query.items(): + if value is None: + continue + value_str = str(value).strip() + if value_str == "": + continue + params[key] = value_str + if not params: + return base_url + return f"{base_url}?{urlencode(params)}" + + +def _merge_query( + current_query: dict[str, Any], **updates: Any +) -> dict[str, Any]: + merged = dict(current_query) + for key, value in updates.items(): + if value is None or str(value).strip() == "": + merged.pop(key, None) + continue + merged[key] = value + return merged + + +@dataclass(frozen=True) +class OsintColumn: + key: str + label: str + accessor: Callable[[Any], Any] + sort_field: str | None = None + search_lookup: str | None = None + kind: str = "text" + + +@dataclass(frozen=True) +class OsintScopeConfig: + key: str + title: str + model: type[models.Model] + list_url_name: str + update_url_name: str + delete_url_name: str + default_sort: str + columns: tuple[OsintColumn, ...] + select_related: tuple[str, ...] = () + prefetch_related: tuple[str, ...] = () + delete_label: Callable[[Any], str] = str + extra_actions: Callable[[Any, str], list[dict[str, Any]]] = field( + default_factory=lambda: (lambda _item, _type: []) + ) + search_lookups: tuple[str, ...] = () + + +def _person_extra_actions(item: Person, _request_type: str) -> list[dict[str, Any]]: + return [ + { + "mode": "link", + "url": reverse( + "person_identifiers", + kwargs={"type": "page", "person": item.id}, + ), + "icon": "fa-solid fa-eye", + "title": "Identifiers", + } + ] + + +OSINT_SCOPES: dict[str, OsintScopeConfig] = { + "people": OsintScopeConfig( + key="people", + title="People", + model=Person, + list_url_name="people", + update_url_name="person_update", + delete_url_name="person_delete", + default_sort="name", + columns=( + OsintColumn( + key="id", + label="ID", + accessor=lambda item: item.id, + sort_field="id", + search_lookup="id__icontains", + kind="id_copy", + ), + OsintColumn( + key="name", + label="Name", + accessor=lambda item: item.name, + sort_field="name", + search_lookup="name__icontains", + ), + OsintColumn( + key="sentiment", + label="Sentiment", + accessor=lambda item: item.sentiment, + sort_field="sentiment", + ), + OsintColumn( + key="timezone", + label="Timezone", + accessor=lambda item: item.timezone, + sort_field="timezone", + search_lookup="timezone__icontains", + ), + OsintColumn( + key="last_interaction", + label="Last Interaction", + accessor=lambda item: item.last_interaction, + sort_field="last_interaction", + kind="datetime", + ), + ), + delete_label=lambda item: item.name, + extra_actions=_person_extra_actions, + search_lookups=( + "id__icontains", + "name__icontains", + "summary__icontains", + "profile__icontains", + "revealed__icontains", + "likes__icontains", + "dislikes__icontains", + "timezone__icontains", + ), + ), + "groups": OsintScopeConfig( + key="groups", + title="Groups", + model=Group, + list_url_name="groups", + update_url_name="group_update", + delete_url_name="group_delete", + default_sort="name", + columns=( + OsintColumn( + key="id", + label="ID", + accessor=lambda item: item.id, + sort_field="id", + search_lookup="id__icontains", + kind="id_copy", + ), + OsintColumn( + key="name", + label="Name", + accessor=lambda item: item.name, + sort_field="name", + search_lookup="name__icontains", + ), + OsintColumn( + key="people_count", + label="People", + accessor=lambda item: item.people.count(), + kind="number", + ), + ), + prefetch_related=("people",), + delete_label=lambda item: item.name, + search_lookups=( + "id__icontains", + "name__icontains", + "people__name__icontains", + ), + ), + "personas": OsintScopeConfig( + key="personas", + title="Personas", + model=Persona, + list_url_name="personas", + update_url_name="persona_update", + delete_url_name="persona_delete", + default_sort="alias", + columns=( + OsintColumn( + key="id", + label="ID", + accessor=lambda item: item.id, + sort_field="id", + search_lookup="id__icontains", + kind="id_copy", + ), + OsintColumn( + key="alias", + label="Alias", + accessor=lambda item: item.alias, + sort_field="alias", + search_lookup="alias__icontains", + ), + OsintColumn( + key="mbti", + label="MBTI", + accessor=lambda item: item.mbti, + sort_field="mbti", + search_lookup="mbti__icontains", + ), + OsintColumn( + key="mbti_identity", + label="MBTI Identity", + accessor=lambda item: item.mbti_identity, + sort_field="mbti_identity", + ), + OsintColumn( + key="humor_style", + label="Humor Style", + accessor=lambda item: item.humor_style, + sort_field="humor_style", + search_lookup="humor_style__icontains", + ), + OsintColumn( + key="tone", + label="Tone", + accessor=lambda item: item.tone, + sort_field="tone", + search_lookup="tone__icontains", + ), + OsintColumn( + key="trust", + label="Trust", + accessor=lambda item: item.trust, + sort_field="trust", + ), + OsintColumn( + key="adaptability", + label="Adaptability", + accessor=lambda item: item.adaptability, + sort_field="adaptability", + ), + ), + delete_label=lambda item: item.alias or str(item.id), + search_lookups=( + "id__icontains", + "alias__icontains", + "mbti__icontains", + "humor_style__icontains", + "tone__icontains", + "communication_style__icontains", + "core_values__icontains", + "inner_story__icontains", + "likes__icontains", + "dislikes__icontains", + ), + ), + "manipulations": OsintScopeConfig( + key="manipulations", + title="Manipulations", + model=Manipulation, + list_url_name="manipulations", + update_url_name="manipulation_update", + delete_url_name="manipulation_delete", + default_sort="name", + columns=( + OsintColumn( + key="id", + label="ID", + accessor=lambda item: item.id, + sort_field="id", + search_lookup="id__icontains", + kind="id_copy", + ), + OsintColumn( + key="name", + label="Name", + accessor=lambda item: item.name, + sort_field="name", + search_lookup="name__icontains", + ), + OsintColumn( + key="group", + label="Group", + accessor=lambda item: item.group, + sort_field="group__name", + search_lookup="group__name__icontains", + ), + OsintColumn( + key="ai", + label="AI", + accessor=lambda item: item.ai, + sort_field="ai__model", + search_lookup="ai__model__icontains", + ), + OsintColumn( + key="persona", + label="Persona", + accessor=lambda item: item.persona, + sort_field="persona__alias", + search_lookup="persona__alias__icontains", + ), + OsintColumn( + key="enabled", + label="Enabled", + accessor=lambda item: item.enabled, + sort_field="enabled", + kind="bool", + ), + OsintColumn( + key="mode", + label="Mode", + accessor=lambda item: item.mode, + sort_field="mode", + search_lookup="mode__icontains", + ), + OsintColumn( + key="filter_enabled", + label="Filter", + accessor=lambda item: item.filter_enabled, + sort_field="filter_enabled", + kind="bool", + ), + ), + select_related=("group", "ai", "persona"), + delete_label=lambda item: item.name, + search_lookups=( + "id__icontains", + "name__icontains", + "mode__icontains", + "group__name__icontains", + "ai__model__icontains", + "persona__alias__icontains", + ), + ), +} + + +class OSINTListBase(ObjectList): + list_template = "partials/osint/list-table.html" + paginate_by = 20 + osint_scope = "" + + def get_scope(self) -> OsintScopeConfig: + if self.osint_scope not in OSINT_SCOPES: + raise ValueError(f"Unknown OSINT scope: {self.osint_scope}") + return OSINT_SCOPES[self.osint_scope] + + def _list_url(self) -> str: + list_url_args = {} + for arg in self.list_url_args: + if arg in self.kwargs: + list_url_args[arg] = self.kwargs[arg] + return reverse(self.list_url_name, kwargs=list_url_args) + + def _active_sort(self, scope: OsintScopeConfig) -> tuple[str, str]: + direction = self.request.GET.get("dir", "asc").lower() + if direction not in {"asc", "desc"}: + direction = "asc" + allowed = {col.sort_field for col in scope.columns if col.sort_field} + sort_field = self.request.GET.get("sort") + if sort_field not in allowed: + sort_field = scope.default_sort + return sort_field, direction + + def get_ordering(self): + scope = self.get_scope() + sort_field, direction = self._active_sort(scope) + if not sort_field: + return None + if direction == "desc": + return f"-{sort_field}" + return sort_field + + def _search_lookups(self, scope: OsintScopeConfig) -> dict[str, str]: + lookups = {} + for column in scope.columns: + if column.search_lookup: + lookups[column.key] = column.search_lookup + return lookups + + def _query_dict(self) -> dict[str, Any]: + return {k: v for k, v in self.request.GET.items() if v not in {"", None}} + + def _apply_list_search( + self, queryset: models.QuerySet, scope: OsintScopeConfig + ) -> models.QuerySet: + query = self.request.GET.get("q", "").strip() + if not query: + return queryset + + lookups_by_field = self._search_lookups(scope) + selected_field = self.request.GET.get("field", "__all__").strip() + if selected_field in lookups_by_field: + selected_lookups = [lookups_by_field[selected_field]] + else: + selected_lookups = list(scope.search_lookups) or list( + lookups_by_field.values() + ) + + if not selected_lookups: + return queryset + + condition = Q() + for lookup in selected_lookups: + condition |= Q(**{lookup: query}) + queryset = queryset.filter(condition) + if any("__" in lookup for lookup in selected_lookups): + queryset = queryset.distinct() + return queryset + + def get_queryset(self, **kwargs): + queryset = super().get_queryset(**kwargs) + scope = self.get_scope() + if scope.select_related: + queryset = queryset.select_related(*scope.select_related) + if scope.prefetch_related: + queryset = queryset.prefetch_related(*scope.prefetch_related) + return self._apply_list_search(queryset, scope) + + def _build_column_context( + self, + scope: OsintScopeConfig, + list_url: str, + query_state: dict[str, Any], + ) -> list[dict[str, Any]]: + sort_field, direction = self._active_sort(scope) + columns = [] + for column in scope.columns: + if not column.sort_field: + columns.append( + { + "label": column.label, + "sortable": False, + "kind": column.kind, + } + ) + continue + + is_sorted = sort_field == column.sort_field + next_direction = "desc" if is_sorted and direction == "asc" else "asc" + sort_query = _merge_query( + query_state, + sort=column.sort_field, + dir=next_direction, + page=1, + ) + columns.append( + { + "label": column.label, + "sortable": True, + "kind": column.kind, + "is_sorted": is_sorted, + "is_desc": is_sorted and direction == "desc", + "sort_url": _url_with_query(list_url, sort_query), + } + ) + return columns + + def _build_rows( + self, + scope: OsintScopeConfig, + object_list: list[Any], + request_type: str, + ) -> list[dict[str, Any]]: + rows = [] + for item in object_list: + row = {"id": str(item.pk), "cells": [], "actions": []} + for column in scope.columns: + row["cells"].append( + { + "kind": column.kind, + "value": column.accessor(item), + } + ) + + update_url = reverse( + scope.update_url_name, + kwargs={"type": context_type, "pk": item.pk}, + ) + delete_url = reverse( + scope.delete_url_name, + kwargs={"type": context_type, "pk": item.pk}, + ) + + row["actions"].append( + { + "mode": "hx-get", + "url": update_url, + "target": f"#{context_type}s-here", + "icon": "fa-solid fa-pencil", + "title": "Edit", + } + ) + row["actions"].append( + { + "mode": "hx-delete", + "url": delete_url, + "target": "#modals-here", + "icon": "fa-solid fa-xmark", + "title": "Delete", + "confirm": ( + "Are you sure you wish to delete " + f"{scope.delete_label(item)}?" + ), + } + ) + row["actions"].extend(scope.extra_actions(item, context_type)) + rows.append(row) + return rows + + def _build_pagination( + self, + page_obj: Any, + list_url: str, + query_state: dict[str, Any], + ) -> dict[str, Any]: + if page_obj is None: + return {"enabled": False} + + pagination = { + "enabled": page_obj.paginator.num_pages > 1, + "count": page_obj.paginator.count, + "current": page_obj.number, + "total": page_obj.paginator.num_pages, + "has_previous": page_obj.has_previous(), + "has_next": page_obj.has_next(), + "previous_url": None, + "next_url": None, + "pages": [], + } + + if page_obj.has_previous(): + pagination["previous_url"] = _url_with_query( + list_url, + _merge_query(query_state, page=page_obj.previous_page_number()), + ) + if page_obj.has_next(): + pagination["next_url"] = _url_with_query( + list_url, + _merge_query(query_state, page=page_obj.next_page_number()), + ) + + for entry in page_obj.paginator.get_elided_page_range(page_obj.number): + if entry == "…": + pagination["pages"].append({"ellipsis": True}) + continue + pagination["pages"].append( + { + "ellipsis": False, + "number": entry, + "current": entry == page_obj.number, + "url": _url_with_query( + list_url, + _merge_query(query_state, page=entry), + ), + } + ) + return pagination + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + scope = self.get_scope() + request_type = self.kwargs.get("type", "modal") + list_url = self._list_url() + query_state = self._query_dict() + query_url = _url_with_query(list_url, query_state) + + field_options = [{"value": "__all__", "label": "All"}] + field_options.extend( + [ + {"value": key, "label": column.label} + for key, column in ( + (column.key, column) + for column in scope.columns + if column.search_lookup + ) + ] + ) + + context["osint_scope"] = scope.key + context["osint_title"] = scope.title + context["osint_table_id"] = f"{self.context_object_name}-table" + context["osint_event_name"] = f"{self.context_object_name_singular}Event" + context["osint_refresh_url"] = query_url or list_url + context["osint_columns"] = self._build_column_context( + scope, + list_url, + query_state, + ) + context["osint_rows"] = self._build_rows( + scope, + list(context["object_list"]), + request_type, + ) + context["osint_pagination"] = self._build_pagination( + context.get("page_obj"), + list_url, + query_state, + ) + context["osint_search_query"] = self.request.GET.get("q", "") + context["osint_search_field"] = self.request.GET.get("field", "__all__") + context["osint_search_fields"] = field_options + context["osint_show_search"] = True + context["osint_show_actions"] = True + context["osint_search_url"] = list_url + context["osint_result_count"] = context["osint_pagination"].get("count", 0) + return context + + +class OSINTSearch(LoginRequiredMixin, View): + allowed_types = {"page", "widget"} + page_template = "pages/osint-search.html" + widget_template = "mixins/wm/widget.html" + panel_template = "partials/osint/search-panel.html" + result_template = "partials/results_table.html" + per_page_default = 20 + per_page_max = 100 + + def _field_options( + self, model_cls: type[models.Model] + ) -> list[dict[str, str]]: + options = [] + for field in model_cls._meta.get_fields(): + if field.auto_created and not field.concrete and not field.many_to_many: + continue + if field.name == "user": + continue + options.append( + { + "value": field.name, + "label": field.verbose_name.title(), + } + ) + options.sort(key=lambda item: item["label"]) + return options + + def _field_q( + self, + model_cls: type[models.Model], + field_name: str, + query: str, + ) -> tuple[Q | None, bool]: + try: + field = model_cls._meta.get_field(field_name) + except FieldDoesNotExist: + return None, False + + if isinstance(field, (models.CharField, models.TextField, models.UUIDField)): + return Q(**{f"{field_name}__icontains": query}), False + if isinstance(field, models.BooleanField): + parsed = _parse_bool(query) + if parsed is None: + return None, False + return Q(**{field_name: parsed}), False + if isinstance(field, models.IntegerField): + try: + return Q(**{field_name: int(query)}), False + except ValueError: + return None, False + if isinstance(field, (models.FloatField, models.DecimalField)): + try: + value = Decimal(query) + except InvalidOperation: + return None, False + return Q(**{field_name: value}), False + if isinstance(field, models.DateField): + try: + parsed_date = date.fromisoformat(query) + except ValueError: + return None, False + return Q(**{field_name: parsed_date}), False + if isinstance(field, models.DateTimeField): + try: + parsed_dt = datetime.fromisoformat(query) + except ValueError: + try: + parsed_date = date.fromisoformat(query) + except ValueError: + return None, False + return Q(**{f"{field_name}__date": parsed_date}), False + return Q(**{field_name: parsed_dt}), False + if isinstance(field, models.ForeignKey): + related_text_field = _preferred_related_text_field(field.related_model) + if related_text_field: + return Q( + **{f"{field_name}__{related_text_field}__icontains": query} + ), False + return Q(**{f"{field_name}__id__icontains": query}), False + if isinstance(field, models.ManyToManyField): + related_text_field = _preferred_related_text_field(field.related_model) + if related_text_field: + return Q( + **{f"{field_name}__{related_text_field}__icontains": query} + ), True + return Q(**{f"{field_name}__id__icontains": query}), True + return None, False + + def _search_queryset( + self, + queryset: models.QuerySet, + model_cls: type[models.Model], + query: str, + field_name: str, + field_options: list[dict[str, str]], + ) -> models.QuerySet: + if not query: + return queryset + + if field_name != "__all__": + field_q, use_distinct = self._field_q(model_cls, field_name, query) + if field_q is None: + return queryset.none() + queryset = queryset.filter(field_q) + return queryset.distinct() if use_distinct else queryset + + condition = Q() + use_distinct = False + for option in field_options: + field_q, field_distinct = self._field_q( + model_cls, + option["value"], + query, + ) + if field_q is None: + continue + condition |= field_q + use_distinct = use_distinct or field_distinct + if not condition.children: + return queryset.none() + queryset = queryset.filter(condition) + return queryset.distinct() if use_distinct else queryset + + def _per_page(self, raw_value: str | None) -> int: + if not raw_value: + return self.per_page_default + try: + value = int(raw_value) + except ValueError: + return self.per_page_default + if value < 1: + return self.per_page_default + return min(value, self.per_page_max) + + def _scope_key(self, raw_scope: str | None) -> str: + if raw_scope in OSINT_SCOPES: + return raw_scope + return "people" + + def _query_state(self, request) -> dict[str, Any]: + return {k: v for k, v in request.GET.items() if v not in {None, ""}} + + def _active_sort(self, scope: OsintScopeConfig) -> tuple[str, str]: + direction = self.request.GET.get("dir", "asc").lower() + if direction not in {"asc", "desc"}: + direction = "asc" + allowed = {col.sort_field for col in scope.columns if col.sort_field} + sort_field = self.request.GET.get("sort") + if sort_field not in allowed: + sort_field = scope.default_sort + return sort_field, direction + + def _build_column_context( + self, + scope: OsintScopeConfig, + list_url: str, + query_state: dict[str, Any], + ) -> list[dict[str, Any]]: + sort_field, direction = self._active_sort(scope) + columns = [] + for column in scope.columns: + if not column.sort_field: + columns.append( + { + "label": column.label, + "sortable": False, + "kind": column.kind, + } + ) + continue + + is_sorted = sort_field == column.sort_field + next_direction = "desc" if is_sorted and direction == "asc" else "asc" + sort_query = _merge_query( + query_state, + sort=column.sort_field, + dir=next_direction, + page=1, + ) + columns.append( + { + "label": column.label, + "sortable": True, + "kind": column.kind, + "is_sorted": is_sorted, + "is_desc": is_sorted and direction == "desc", + "sort_url": _url_with_query(list_url, sort_query), + } + ) + return columns + + def _build_rows( + self, + scope: OsintScopeConfig, + object_list: list[Any], + request_type: str, + ) -> list[dict[str, Any]]: + context_type = _context_type(request_type) + rows = [] + for item in object_list: + row = {"id": str(item.pk), "cells": [], "actions": []} + for column in scope.columns: + row["cells"].append( + { + "kind": column.kind, + "value": column.accessor(item), + } + ) + rows.append(row) + return rows + + def _build_pagination( + self, + page_obj: Any, + list_url: str, + query_state: dict[str, Any], + ) -> dict[str, Any]: + if page_obj is None: + return {"enabled": False} + + pagination = { + "enabled": page_obj.paginator.num_pages > 1, + "count": page_obj.paginator.count, + "current": page_obj.number, + "total": page_obj.paginator.num_pages, + "has_previous": page_obj.has_previous(), + "has_next": page_obj.has_next(), + "previous_url": None, + "next_url": None, + "pages": [], + } + + if page_obj.has_previous(): + pagination["previous_url"] = _url_with_query( + list_url, + _merge_query(query_state, page=page_obj.previous_page_number()), + ) + if page_obj.has_next(): + pagination["next_url"] = _url_with_query( + list_url, + _merge_query(query_state, page=page_obj.next_page_number()), + ) + + for entry in page_obj.paginator.get_elided_page_range(page_obj.number): + if entry == "…": + pagination["pages"].append({"ellipsis": True}) + continue + pagination["pages"].append( + { + "ellipsis": False, + "number": entry, + "current": entry == page_obj.number, + "url": _url_with_query( + list_url, + _merge_query(query_state, page=entry), + ), + } + ) + return pagination + + def get(self, request, type): + if type not in self.allowed_types: + return HttpResponseBadRequest("Invalid type specified.") + + scope_key = self._scope_key(request.GET.get("scope")) + scope = OSINT_SCOPES[scope_key] + field_options = self._field_options(scope.model) + query = request.GET.get("q", "").strip() + field_name = request.GET.get("field", "__all__") + if field_name != "__all__": + allowed_fields = {option["value"] for option in field_options} + if field_name not in allowed_fields: + field_name = "__all__" + + queryset = scope.model.objects.filter(user=request.user) + if scope.select_related: + queryset = queryset.select_related(*scope.select_related) + if scope.prefetch_related: + queryset = queryset.prefetch_related(*scope.prefetch_related) + + queryset = self._search_queryset( + queryset, + scope.model, + query, + field_name, + field_options, + ) + + sort_field = request.GET.get("sort", scope.default_sort) + direction = request.GET.get("dir", "asc").lower() + allowed_sort_fields = { + column.sort_field for column in scope.columns if column.sort_field + } + if sort_field not in allowed_sort_fields: + sort_field = scope.default_sort + if direction not in {"asc", "desc"}: + direction = "asc" + if sort_field: + order_by = sort_field if direction == "asc" else f"-{sort_field}" + queryset = queryset.order_by(order_by) + + per_page = self._per_page(request.GET.get("per_page")) + paginator = Paginator(queryset, per_page) + page_obj = paginator.get_page(request.GET.get("page")) + object_list = list(page_obj.object_list) + + list_url = reverse("osint_search", kwargs={"type": type}) + query_state = self._query_state(request) + column_context = self._build_column_context( + scope, + list_url, + query_state, + ) + rows = self._build_rows( + scope, + object_list, + type, + ) + pagination = self._build_pagination( + page_obj, + list_url, + query_state, + ) + + context = { + "osint_scope": scope.key, + "osint_title": f"Search {scope.title}", + "osint_table_id": "osint-search-table", + "osint_event_name": "", + "osint_refresh_url": _url_with_query(list_url, query_state), + "osint_columns": column_context, + "osint_rows": rows, + "osint_pagination": pagination, + "osint_show_search": False, + "osint_show_actions": False, + "osint_result_count": paginator.count, + "osint_search_url": list_url, + "scope_options": [ + {"value": key, "label": conf.title} + for key, conf in OSINT_SCOPES.items() + ], + "field_options": field_options, + "selected_scope": scope.key, + "selected_field": field_name, + "search_query": query, + "selected_per_page": per_page, + "search_page_url": reverse("osint_search", kwargs={"type": "page"}), + "search_widget_url": reverse("osint_search", kwargs={"type": "widget"}), + } + + hx_target = request.headers.get("HX-Target") + if request.htmx and hx_target in {"osint-search-results", "osint-search-table"}: + response = render(request, self.result_template, context) + if type == "page": + response["HX-Replace-Url"] = _url_with_query(list_url, query_state) + return response + + if type == "widget": + widget_context = { + "title": "Search", + "unique": "osint-search-widget", + "window_content": self.panel_template, + "widget_options": 'gs-w="8" gs-h="14" gs-x="0" gs-y="0" gs-min-w="5"', + **context, + } + return render(request, self.widget_template, widget_context) + + return render(request, self.page_template, context) diff --git a/core/views/people.py b/core/views/people.py index f58ce6a..d86fd47 100644 --- a/core/views/people.py +++ b/core/views/people.py @@ -1,18 +1,18 @@ from django.contrib.auth.mixins import LoginRequiredMixin -from mixins.views import ObjectCreate, ObjectDelete, ObjectList, ObjectUpdate +from mixins.views import ObjectCreate, ObjectDelete, ObjectUpdate from core.forms import PersonForm from core.models import Person +from core.views.osint import OSINTListBase from core.util import logs log = logs.get_logger(__name__) -class PersonList(LoginRequiredMixin, ObjectList): - list_template = "partials/person-list.html" +class PersonList(LoginRequiredMixin, OSINTListBase): + osint_scope = "people" model = Person page_title = "People" - # page_subtitle = "Add times here in order to permit trading." list_url_name = "people" list_url_args = ["type"] diff --git a/core/views/personas.py b/core/views/personas.py index 2de57f2..d7f112c 100644 --- a/core/views/personas.py +++ b/core/views/personas.py @@ -1,15 +1,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin -from mixins.views import ObjectCreate, ObjectDelete, ObjectList, ObjectUpdate +from mixins.views import ObjectCreate, ObjectDelete, ObjectUpdate from core.forms import PersonaForm from core.models import Persona +from core.views.osint import OSINTListBase from core.util import logs log = logs.get_logger(__name__) -class PersonaList(LoginRequiredMixin, ObjectList): - list_template = "partials/persona-list.html" +class PersonaList(LoginRequiredMixin, OSINTListBase): + osint_scope = "personas" model = Persona page_title = "Personas" diff --git a/core/views/whatsapp.py b/core/views/whatsapp.py index bce5eb9..ac88d50 100644 --- a/core/views/whatsapp.py +++ b/core/views/whatsapp.py @@ -1,29 +1,94 @@ +from django.conf import settings +from django.shortcuts import render +from django.views import View +from mixins.views import ObjectList, ObjectRead + from core.clients import transport -from core.views.signal import Signal, SignalAccountAdd, SignalAccounts +from core.views.manage.permissions import SuperUserRequiredMixin -class WhatsApp(Signal): +class WhatsApp(SuperUserRequiredMixin, View): + template_name = "pages/signal.html" service = "whatsapp" page_title = "WhatsApp" accounts_url_name = "whatsapp_accounts" + def get(self, request): + return render( + request, + self.template_name, + { + "service": self.service, + "service_label": self.page_title, + "accounts_url_name": self.accounts_url_name, + }, + ) -class WhatsAppAccounts(SignalAccounts): + +class WhatsAppAccounts(SuperUserRequiredMixin, ObjectList): + list_template = "partials/signal-accounts.html" service = "whatsapp" + context_object_name_singular = "WhatsApp Account" context_object_name = "WhatsApp Accounts" list_url_name = "whatsapp_accounts" + list_url_args = ["type"] + + def _normalize_accounts(self, rows): + out = [] + for item in rows or []: + if isinstance(item, dict): + value = ( + item.get("number") + or item.get("id") + or item.get("jid") + or item.get("account") + ) + if value: + out.append(str(value)) + elif item: + out.append(str(item)) + return out def get_queryset(self, **kwargs): - self.extra_context = self._service_context( - service="whatsapp", - label="WhatsApp", - add_url_name="whatsapp_account_add", - show_contact_actions=False, - ) + self.extra_context = { + "service": "whatsapp", + "service_label": "WhatsApp", + "account_add_url_name": "whatsapp_account_add", + "show_contact_actions": False, + "endpoint_base": str( + getattr(settings, "WHATSAPP_HTTP_URL", "http://whatsapp:8080") + ).rstrip("/"), + "service_warning": transport.get_service_warning("whatsapp"), + } return self._normalize_accounts(transport.list_accounts("whatsapp")) -class WhatsAppAccountAdd(SignalAccountAdd): +class WhatsAppAccountAdd(SuperUserRequiredMixin, ObjectRead): + detail_template = "partials/whatsapp-account-add.html" service = "whatsapp" + context_object_name_singular = "Add Account" + context_object_name = "Add Account" detail_url_name = "whatsapp_account_add" + detail_url_args = ["type", "device"] + + def post(self, request, *args, **kwargs): + self.request = request + return super().get(request, *args, **kwargs) + + def get_object(self, **kwargs): + form_args = self.request.POST.dict() + device_name = form_args.get("device", "GIA Device") + try: + image_bytes = transport.get_link_qr(self.service, device_name) + return { + "ok": True, + "image_b64": transport.image_bytes_to_base64(image_bytes), + "warning": transport.get_service_warning(self.service), + } + except Exception as exc: + return { + "ok": False, + "error": str(exc), + "warning": transport.get_service_warning(self.service), + } diff --git a/core/views/workspace.py b/core/views/workspace.py index ddff7b0..7cb0ead 100644 --- a/core/views/workspace.py +++ b/core/views/workspace.py @@ -155,6 +155,55 @@ INSIGHT_METRICS = { "values often precede misunderstandings or withdrawal cycles." ), }, + "reciprocity_score": { + "title": "Reciprocity Component", + "group": "stability", + "history_field": "reciprocity_score", + "calculation": ( + "100 * (1 - |inbound - outbound| / total_messages). Higher means " + "more balanced participation." + ), + "psychology": ( + "Lower reciprocity can reflect perceived asymmetry and rising pursuit/" + "withdraw cycles." + ), + }, + "continuity_score": { + "title": "Continuity Component", + "group": "stability", + "history_field": "continuity_score", + "calculation": ( + "100 * min(1, distinct_sample_days / span_days). Higher means steadier " + "day-to-day continuity." + ), + "psychology": ( + "Drops can signal communication becoming episodic or reactive." + ), + }, + "response_score": { + "title": "Response Component", + "group": "stability", + "history_field": "response_score", + "calculation": ( + "Average of inbound and outbound response-lag scores, each mapped from " + "median lag to a 0-100 curve." + ), + "psychology": ( + "Lower response score can indicate delayed repair loops during tension." + ), + }, + "volatility_score": { + "title": "Volatility Component", + "group": "stability", + "history_field": "volatility_score", + "calculation": ( + "Derived from coefficient of variation of daily message counts and " + "inverted to a 0-100 stability signal." + ), + "psychology": ( + "High volatility can suggest inconsistent rhythm and reduced predictability." + ), + }, "stability_confidence": { "title": "Stability Confidence", "group": "confidence", @@ -219,6 +268,52 @@ INSIGHT_METRICS = { "Estimates user follow-through and consistency toward the counterpart." ), }, + "inbound_response_score": { + "title": "Inbound Response Score", + "group": "commitment", + "history_field": "inbound_response_score", + "calculation": ( + "Response-speed score built from median lag between user outbound and " + "counterpart inbound replies." + ), + "psychology": ( + "Lower values suggest delayed reciprocity from counterpart direction." + ), + }, + "outbound_response_score": { + "title": "Outbound Response Score", + "group": "commitment", + "history_field": "outbound_response_score", + "calculation": ( + "Response-speed score built from median lag between counterpart inbound " + "and user outbound replies." + ), + "psychology": "Lower values suggest slower follow-through from user direction.", + }, + "balance_inbound_score": { + "title": "Inbound Balance Score", + "group": "commitment", + "history_field": "balance_inbound_score", + "calculation": ( + "100 * min(1, inbound_messages / outbound_messages). Captures inbound " + "participation parity." + ), + "psychology": ( + "Lower values can indicate one-sided conversational load from user side." + ), + }, + "balance_outbound_score": { + "title": "Outbound Balance Score", + "group": "commitment", + "history_field": "balance_outbound_score", + "calculation": ( + "100 * min(1, outbound_messages / inbound_messages). Captures outbound " + "participation parity." + ), + "psychology": ( + "Lower values can indicate one-sided conversational load from counterpart side." + ), + }, "commitment_confidence": { "title": "Commit Confidence", "group": "confidence", @@ -334,6 +429,78 @@ INSIGHT_GRAPH_SPECS = [ "y_min": 0, "y_max": None, }, + { + "slug": "reciprocity_score", + "title": "Reciprocity Component", + "field": "reciprocity_score", + "group": "stability", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "continuity_score", + "title": "Continuity Component", + "field": "continuity_score", + "group": "stability", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "response_score", + "title": "Response Component", + "field": "response_score", + "group": "stability", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "volatility_score", + "title": "Volatility Component", + "field": "volatility_score", + "group": "stability", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "inbound_response_score", + "title": "Inbound Response Score", + "field": "inbound_response_score", + "group": "commitment", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "outbound_response_score", + "title": "Outbound Response Score", + "field": "outbound_response_score", + "group": "commitment", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "balance_inbound_score", + "title": "Inbound Balance Score", + "field": "balance_inbound_score", + "group": "commitment", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "balance_outbound_score", + "title": "Outbound Balance Score", + "field": "balance_outbound_score", + "group": "commitment", + "y_min": 0, + "y_max": 100, + }, + { + "slug": "last_event", + "title": "Last Event Timestamp", + "field": "source_event_ts", + "group": "timeline", + "y_min": None, + "y_max": None, + }, ] @@ -487,7 +654,10 @@ def _to_float(value): return float(value) -def _format_metric_value(conversation, metric_slug): +def _format_metric_value(conversation, metric_slug, latest_snapshot=None): + snapshot = latest_snapshot + if snapshot is None: + snapshot = conversation.metric_snapshots.first() if metric_slug == "platform": return conversation.get_platform_type_display() or "-" if metric_slug == "thread": @@ -498,6 +668,14 @@ def _format_metric_value(conversation, metric_slug): return conversation.get_stability_state_display() if metric_slug == "stability_score": return conversation.stability_score + if metric_slug == "reciprocity_score": + return snapshot.reciprocity_score if snapshot else None + if metric_slug == "continuity_score": + return snapshot.continuity_score if snapshot else None + if metric_slug == "response_score": + return snapshot.response_score if snapshot else None + if metric_slug == "volatility_score": + return snapshot.volatility_score if snapshot else None if metric_slug == "stability_confidence": return conversation.stability_confidence if metric_slug == "sample_messages": @@ -510,6 +688,14 @@ def _format_metric_value(conversation, metric_slug): return conversation.commitment_inbound_score if metric_slug == "commitment_outbound": return conversation.commitment_outbound_score + if metric_slug == "inbound_response_score": + return snapshot.inbound_response_score if snapshot else None + if metric_slug == "outbound_response_score": + return snapshot.outbound_response_score if snapshot else None + if metric_slug == "balance_inbound_score": + return snapshot.balance_inbound_score if snapshot else None + if metric_slug == "balance_outbound_score": + return snapshot.balance_outbound_score if snapshot else None if metric_slug == "commitment_confidence": return conversation.commitment_confidence if metric_slug == "commitment_computed": @@ -2713,7 +2899,8 @@ class AIWorkspaceInsightDetail(LoginRequiredMixin, View): person = get_object_or_404(Person, pk=person_id, user=request.user) conversation = _conversation_for_person(request.user, person) - value = _format_metric_value(conversation, metric) + latest_snapshot = conversation.metric_snapshots.first() + value = _format_metric_value(conversation, metric, latest_snapshot) group = INSIGHT_GROUPS[spec["group"]] points = [] if spec["history_field"]: @@ -2773,6 +2960,7 @@ class AIWorkspaceInsightHelp(LoginRequiredMixin, View): person = get_object_or_404(Person, pk=person_id, user=request.user) conversation = _conversation_for_person(request.user, person) + latest_snapshot = conversation.metric_snapshots.first() metrics = [] for slug, spec in INSIGHT_METRICS.items(): metrics.append( @@ -2783,7 +2971,11 @@ class AIWorkspaceInsightHelp(LoginRequiredMixin, View): "group_title": INSIGHT_GROUPS[spec["group"]]["title"], "calculation": spec["calculation"], "psychology": spec["psychology"], - "value": _format_metric_value(conversation, slug), + "value": _format_metric_value( + conversation, + slug, + latest_snapshot, + ), } )