From acedc01e830d1154c4c34660652e4b3b834825fb Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 8 Mar 2026 22:08:55 +0000 Subject: [PATCH] Fix all integrations --- INSTALL.md | 49 +- Makefile | 20 + README.md | 14 + app/local_settings.py | 6 + app/test_settings.py | 8 +- artifacts/audits/1-initial.json | 12 +- ...7-settings-integrity-and-controls-reorg.md | 158 +++ core/clients/signal.py | 523 ++++++++- core/clients/transport.py | 74 +- core/clients/whatsapp.py | 220 ++-- core/clients/xmpp.py | 1034 +++++++++-------- core/context_processors.py | 180 +-- core/gateway/builtin.py | 417 +++++++ core/mcp/tools.py | 99 ++ ...s_encrypt_component_messages_with_omemo.py | 15 + core/models.py | 1 + core/security/capabilities.py | 111 ++ core/security/command_policy.py | 36 +- core/tasks/engine.py | 172 ++- core/templates/base.html | 55 + core/templates/pages/codex-settings.html | 10 +- core/templates/pages/command-routing.html | 5 + .../pages/compose-contact-match.html | 13 +- core/templates/pages/security.html | 18 +- core/templates/pages/tasks-detail.html | 10 +- core/templates/pages/tasks-hub.html | 64 +- core/templates/pages/tasks-settings.html | 5 + core/tests/test_command_routing_variant_ui.py | 1 + core/tests/test_command_security_policy.py | 64 +- core/tests/test_cross_platform_messaging.py | 16 +- core/tests/test_mcp_tools.py | 31 + ...test_presence_query_and_compose_context.py | 90 +- core/tests/test_settings_integrity.py | 85 ++ core/tests/test_signal_relink.py | 19 + core/tests/test_signal_unlink_fallback.py | 57 + core/tests/test_tasks_pages_management.py | 22 + .../test_whatsapp_reaction_and_recalc.py | 55 +- core/tests/test_xmpp_approval_commands.py | 65 +- core/tests/test_xmpp_attachment_bridge.py | 103 ++ core/tests/test_xmpp_carbons.py | 165 +++ core/tests/test_xmpp_integration.py | 12 +- core/tests/test_xmpp_omemo_support.py | 36 +- core/views/automation.py | 27 +- core/views/compose.py | 313 ++++- core/views/signal.py | 31 +- core/views/system.py | 97 +- core/views/tasks.py | 59 + docker/uwsgi.ini | 11 +- scripts/quadlet/manage.sh | 20 +- scripts/quadlet/render_units.py | 2 +- stack.env.example | 7 +- .../memory/manage_manticore_container.sh | 2 +- utilities/prosody/modules/mod_privilege.lua | 121 ++ .../modules/mod_privileged_carbons.lua | 138 +++ utilities/prosody/prosody.cfg.lua | 7 + utilities/signal/Containerfile | 85 ++ vendor/django-crud-mixins/README.md | 4 +- vendor/django-crud-mixins/setup.cfg | 6 +- 58 files changed, 4120 insertions(+), 960 deletions(-) create mode 100644 artifacts/plans/2026-03-07-settings-integrity-and-controls-reorg.md create mode 100644 core/gateway/builtin.py create mode 100644 core/migrations/0044_userxmppsecuritysettings_encrypt_component_messages_with_omemo.py create mode 100644 core/security/capabilities.py create mode 100644 core/tests/test_settings_integrity.py create mode 100644 core/tests/test_xmpp_attachment_bridge.py create mode 100644 core/tests/test_xmpp_carbons.py create mode 100644 utilities/prosody/modules/mod_privilege.lua create mode 100644 utilities/prosody/modules/mod_privileged_carbons.lua create mode 100644 utilities/signal/Containerfile diff --git a/INSTALL.md b/INSTALL.md index e94d0ee..16bd100 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -88,6 +88,15 @@ Optional static token helper: make token TOKEN_USER= ``` +Local code-quality checks: + +```bash +make pre-commit +make pre-commit-glibc +``` + +`make pre-commit-glibc` selects `env` on musl systems and `genv` on glibc systems. + ## 6) Logs and health checks Tail logs: @@ -141,8 +150,21 @@ If blocked, use full recycle. - Manual compose: `/compose/page/` - AI workspace: `/ai/workspace/` - OSINT search: `/search/page/` +- Security encryption settings: `/settings/security/encryption/` +- Security permissions settings: `/settings/security/permissions/` +- Command routing settings: `/settings/command-routing/` +- Task automation settings: `/settings/tasks/` +- Task inbox / manual task creation: `/tasks/` -## 10) Common troubleshooting +## 10) Security and capability controls + +- `Require OMEMO encryption` rejects plaintext XMPP messages before command routing. +- `Encrypt gateway component chat replies with OMEMO` only affects gateway/component conversations. +- `Encrypt contact relay messages to your XMPP client with OMEMO` only affects relayed contact chats. +- Fine-grained capability policy is configured in `/settings/security/permissions/` and applies by scope, service, and optional channel pattern. +- Trusted OMEMO key enforcement depends on trusted key records, not only the most recently observed client key. + +## 11) Common troubleshooting ### A) Compose restart errors / dependency improper state @@ -237,6 +259,13 @@ make run Then approve/enable the `manticore` MCP server in VS Code when prompted. +The MCP task surface now supports canonical task creation/completion in GIA: + +- `tasks.create` +- `tasks.complete` +- `tasks.create_note` +- `tasks.link_artifact` + Optional ultra-light Rust MCP worker: ```bash @@ -247,6 +276,24 @@ make mcp-rust-build Then enable `manticore-rust-worker` in `/code/xf/.vscode/mcp.json`. It is intentionally `disabled: true` by default so the existing Python MCP server remains the baseline. +### H) Optional browser MCP for visual validation + +To validate compose/tasks/settings flows visually, add a browser-capable MCP server in your editor workspace alongside `manticore`. A Playwright-style browser MCP is the intended integration point for GIA UI checks. + +Recommended usage: + +- keep browser MCP outside host-network mode +- point it at the local GIA app URL/port from the running stack +- use it for page-load, form-flow, and visual regression checks on compose/tasks/settings pages + +### I) Task command shortcuts + +Gateway / XMPP / chat task commands now include: + +- `.l` -> list open tasks +- `.tasks add :: ` -> create a canonical task in a named project +- `.task add <title>` -> create a task inside the current mapped chat scope + ### C) Signal or WhatsApp send failures - Verify account/link status in service pages. diff --git a/Makefile b/Makefile index cd03ff0..3742b16 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ TOKEN_USER ?= m STACK_ID_CLEAN := $(shell sid="$${GIA_STACK_ID:-$${STACK_ID:-}}"; sid=$$(printf "%s" "$$sid" | tr -cs 'a-zA-Z0-9._-' '-' | sed 's/^-*//; s/-*$$//'); printf "%s" "$$sid") STACK_SUFFIX := $(if $(STACK_ID_CLEAN),_$(STACK_ID_CLEAN),) APP_CONTAINER := gia$(STACK_SUFFIX) +LOCAL_LIBC := $(shell if ldd --version 2>&1 | head -n1 | tr '[:upper:]' '[:lower:]' | grep -q musl; then printf musl; else printf glibc; fi) +LOCAL_VENV := $(if $(filter musl,$(LOCAL_LIBC)),env,genv) +PRE_COMMIT_BIN := $(firstword $(wildcard $(LOCAL_VENV)/bin/pre-commit) $(wildcard genv/bin/pre-commit) $(wildcard env/bin/pre-commit)) run: bash $(QUADLET_MGR) up @@ -31,6 +34,23 @@ test: exit 125; \ fi +pre-commit: + @if [ -x "$(PRE_COMMIT_BIN)" ]; then \ + "$(PRE_COMMIT_BIN)" run -a; \ + else \ + echo "No local pre-commit executable found in $(LOCAL_VENV)/bin, genv/bin, or env/bin." >&2; \ + exit 127; \ + fi + +pre-commit-glibc: + @if [ -x "$(PRE_COMMIT_BIN)" ]; then \ + echo "Using $(LOCAL_VENV) ($(LOCAL_LIBC))"; \ + "$(PRE_COMMIT_BIN)" run -a; \ + else \ + echo "No local pre-commit executable found in $(LOCAL_VENV)/bin, genv/bin, or env/bin." >&2; \ + exit 127; \ + fi + migrate: @if podman ps --format '{{.Names}}' | grep -qx "$(APP_CONTAINER)"; then \ podman exec "$(APP_CONTAINER)" sh -lc "cd /code && . /venv/bin/activate && python manage.py migrate"; \ diff --git a/README.md b/README.md index 77ad7b9..f0c1dcd 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,12 @@ GIA is a multi-transport communication workspace that unifies Signal, WhatsApp, - Unifies chats from multiple protocols in one interface. - Keeps conversation history in a shared model (`Person`, `PersonIdentifier`, `ChatSession`, `Message`). - Supports manual, queue-driven, and AI-assisted outbound messaging. +- Supports canonical task creation from chat commands, web UI, and MCP tooling. - Bridges messages across transports (including XMPP) with attachment handling. - Tracks delivery/read metadata and typing state events. - Provides AI workspace analytics, mitigation plans, and insight visualizations. +- Exposes fine-grained capability policy controls for gateway commands, task intake, and command execution. +- Separates XMPP encryption controls into plaintext rejection, component-chat encryption, and relayed-contact encryption. ## Operation Modes @@ -104,6 +107,14 @@ Core behavior: - XMPP bridge supports text, attachments, typing, and chat-state paths. - Signal and WhatsApp media relay paths are normalized via shared transport/media logic. +## Settings Model + +- `Security > Encryption`: transport-level XMPP/OMEMO controls, observed client state, and discovered key trust management. +- `Security > Permissions`: fine-grained capability policy by scope, service, and channel pattern. +- `Modules > Commands`: command profiles, bindings, and delivery behavior. +- `Modules > Task Automation`: task extraction defaults, channel overrides, and provider approval routing. +- `Modules > Business Plans`: generated document inbox and editor. + Key design points: - Prefer shared media preparation over per-service duplicated logic. @@ -121,6 +132,7 @@ Core components: - `core/clients/xmpp.py`: XMPP component bridge and media upload relay. - `rust/manticore-mcp-worker`: optional ultra-light MCP frontend for direct Manticore status/query/maintenance. - `core/views/compose.py`: Manual compose UX, polling/ws, send pipeline, media blob endpoint. +- `core/tasks/engine.py`: Canonical task creation/completion helpers used by chat commands and UI. - `core/views/workspace.py`: AI workspace operations and insight surfaces. - `core/views/osint.py`: Search/workspace OSINT interactions. @@ -144,6 +156,7 @@ After environment setup from `INSTALL.md`: 4. Open manual compose and test per-service send/receive. 5. Open AI workspace for analysis/mitigation workflows. 6. Verify queue workflows if approval mode is used. +7. Verify task creation from `/tasks/`, `.tasks add <project> :: <title>`, or scoped `.task add <title>`. Recommended functional smoke test: @@ -157,6 +170,7 @@ Recommended functional smoke test: - After runtime code changes, restart runtime services before validation. - Full environment recycle convention: `make stop && make run`. - If single-service restart fails due to dependency state, use full recycle. +- Local repository checks are available via `make pre-commit`; use `make pre-commit-glibc` when you want libc-based `env`/`genv` selection. ## Security & Reliability Notes diff --git a/app/local_settings.py b/app/local_settings.py index ab2548e..aec6103 100644 --- a/app/local_settings.py +++ b/app/local_settings.py @@ -91,6 +91,12 @@ XMPP_USER_DOMAIN = getenv("XMPP_USER_DOMAIN", "") XMPP_PORT = int(getenv("XMPP_PORT", "8888") or 8888) XMPP_SECRET = getenv("XMPP_SECRET") XMPP_OMEMO_DATA_DIR = getenv("XMPP_OMEMO_DATA_DIR", "") +XMPP_UPLOAD_SERVICE = getenv("XMPP_UPLOAD_SERVICE", "").strip() +XMPP_UPLOAD_JID = getenv("XMPP_UPLOAD_JID", "").strip() +if not XMPP_UPLOAD_SERVICE and XMPP_UPLOAD_JID: + XMPP_UPLOAD_SERVICE = XMPP_UPLOAD_JID +if not XMPP_UPLOAD_SERVICE and XMPP_USER_DOMAIN: + XMPP_UPLOAD_SERVICE = XMPP_USER_DOMAIN EVENT_LEDGER_DUAL_WRITE = getenv("EVENT_LEDGER_DUAL_WRITE", "false").lower() in trues CAPABILITY_ENFORCEMENT_ENABLED = ( diff --git a/app/test_settings.py b/app/test_settings.py index 191fd80..4bf1bf6 100644 --- a/app/test_settings.py +++ b/app/test_settings.py @@ -1,13 +1,11 @@ -"""Test-only settings overrides — used via DJANGO_SETTINGS_MODULE=app.test_settings.""" +from app.settings import * # noqa -from app.settings import * # noqa: F401, F403 CACHES = { "default": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "gia-test-cache", } } -INSTALLED_APPS = [app for app in INSTALLED_APPS if app != "cachalot"] # noqa: F405 - -CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} +CACHALOT_ENABLED = False diff --git a/artifacts/audits/1-initial.json b/artifacts/audits/1-initial.json index 03c46ee..f950029 100644 --- a/artifacts/audits/1-initial.json +++ b/artifacts/audits/1-initial.json @@ -102,7 +102,7 @@ "severity": "high", "category": "supply-chain", "rule": "GIT_PYTHON_DEP", - "title": "Git/URL Python Dependency: git+https://git.zm.is/XF/django-crud-mixins", + "title": "Git/URL Python Dependency: git+https://git.example.invalid/vendor/django-crud-mixins", "description": "Installing from git/URL bypasses PyPI integrity checks.", "fix": "Publish to PyPI or pin to a specific commit hash", "cwe": null, @@ -522,9 +522,9 @@ "severity": "medium", "category": "supply-chain", "rule": "UNPINNED_PYTHON_DEP", - "title": "Unpinned Python Dependency: git+https://git.zm.is/XF/django-crud-mixins", + "title": "Unpinned Python Dependency: git+https://git.example.invalid/vendor/django-crud-mixins", "description": "Python dependency without version pin. Pin to a specific version for reproducible builds.", - "fix": "Pin version: git+https://git.zm.is/XF/django-crud-mixins==x.y.z", + "fix": "Pin version: git+https://git.example.invalid/vendor/django-crud-mixins==x.y.z", "cwe": null, "owasp": null }, @@ -812,7 +812,7 @@ "severity": "high", "category": "supply-chain", "categoryLabel": "SUPPLY CHAIN", - "title": "Git/URL Python Dependency: git+https://git.zm.is/XF/django-crud-mixins", + "title": "Git/URL Python Dependency: git+https://git.example.invalid/vendor/django-crud-mixins", "file": "requirements.txt:26", "action": "Publish to PyPI or pin to a specific commit hash", "effort": "medium" @@ -1162,9 +1162,9 @@ "severity": "medium", "category": "supply-chain", "categoryLabel": "SUPPLY CHAIN", - "title": "Unpinned Python Dependency: git+https://git.zm.is/XF/django-crud-mixins", + "title": "Unpinned Python Dependency: git+https://git.example.invalid/vendor/django-crud-mixins", "file": "requirements.txt:26", - "action": "Pin version: git+https://git.zm.is/XF/django-crud-mixins==x.y.z", + "action": "Pin version: git+https://git.example.invalid/vendor/django-crud-mixins==x.y.z", "effort": "medium" }, { diff --git a/artifacts/plans/2026-03-07-settings-integrity-and-controls-reorg.md b/artifacts/plans/2026-03-07-settings-integrity-and-controls-reorg.md new file mode 100644 index 0000000..fdbc89d --- /dev/null +++ b/artifacts/plans/2026-03-07-settings-integrity-and-controls-reorg.md @@ -0,0 +1,158 @@ +# Plan: Settings Integrity and Controls Reorganization + +## Objective +Create a single coherent configuration model for Security, Commands, and Tasks so UI labels, enforcement behavior, docs, and navigation all match actual runtime behavior. + +## Current Integrity Findings + +### 1) Scope Registry and Enforcement Are Out of Sync +- Gateway command routes enforce scopes that are not exposed in Fine-Grained Security Scopes: + - `gateway.contacts` + - `gateway.help` + - `gateway.whoami` +- Fine-Grained Security Scopes currently expose only: + - `gateway.tasks`, `gateway.approval`, `tasks.submit`, `tasks.commands`, `command.bp`, `command.codex`, `command.claude` +- Result: users cannot configure all enforced gateway capabilities from the UI. + +### 2) “Require Trusted Fingerprint” Semantics Are Incorrect +- UI and labels imply trust-list based enforcement. +- Runtime policy enforcement checks `UserXmppOmemoState.latest_client_key` equality, not `UserXmppOmemoTrustedKey` trust records. +- Result: behavior is “match latest observed key,” not “require trusted fingerprint.” + +### 3) Command Surfaces Are Split Across Inconsistent Places +- Command Routing UI create flow exposes command slugs: `bp`, `codex`. +- Runtime command engine auto-bootstraps `claude` profile and bindings. +- Security scopes include `command.claude`, but Command Routing create UI does not. +- Result: commands are partially configurable depending on entrypoint. + +### 4) Task and Command Control Planes Interlock Implicitly, Not Explicitly +- Task settings contain provider approval routing (Codex/Claude approver service/identifier). +- Security permissions contain policy gates (`tasks.*`, `command.*`, `gateway.*`). +- Command Routing controls profile/binding/variant policy. +- These are tightly coupled but not represented as one layered model in UI/docs. + +### 5) Settings Shell Coverage Is Incomplete +- Shared settings hierarchy nav is context-processor driven and route-name based. +- Settings routes missing from modules/general/security group coverage include: + - `codex_settings` + - `codex_approval` + - `translation_preview` +- Result: some settings pages can miss expected local tabs/title context. + +### 6) Documentation Drift +- Undocumented or under-documented features now in production behavior: + - Fine-Grained Security Scopes + Global Scope Override + - OMEMO trust management and per-direction encryption toggles + - Business Plan Inbox under Settings Modules +- Potentially misleading documentation: + - Security wording implies trusted-key enforcement that is not implemented. + +## Reorganization Principles +1. One capability registry, reused by: + - Security Permissions UI + - command/task/gateway dispatch + - documentation generation +2. One settings-shell contract for every `/settings/*` page: + - title + - category tabs + - breadcrumb +3. Explicit layered model: + - Layer A: transport encryption/security + - Layer B: capability permissions (scope policy) + - Layer C: feature configuration (tasks/commands/providers) +4. No hardcoded duplicated scope lists in multiple files. + +## Target Information Architecture + +### Security +- `Encryption`: OMEMO transport controls + trust management. +- `Permissions`: Fine-Grained Security Scopes (capability policy only). +- `2FA`: account factor settings. + +### Modules +- `Commands`: command profiles, bindings, variant policies. +- `Tasks`: extraction/defaults/overrides/provider pipelines. +- `Translation`: translation bridge settings. +- `Availability`: adapter availability controls. +- `Business Plans`: inbox/editor for generated artifacts. + +### General +- `Notifications` +- `System` +- `Accessibility` + +### AI +- `Models` +- `Traces` + +## Phased Execution Plan + +## Phase 1: Canonical Capability Registry +1. Add a central capability registry module (single source of truth): + - key + - label + - description + - group (`gateway`, `tasks`, `commands`, `agentic`, etc.) + - owning feature page URL +2. Migrate SecurityPage scope rendering to this registry. +3. Migrate gateway/command/task dispatchers to reference registry keys. +4. Add automated integrity test: + - every enforced scope key must exist in registry + - every registry key marked user-configurable must appear in Permissions UI + +## Phase 2: Trusted Key Enforcement Correction +1. Define authoritative trust policy behavior: + - `require_trusted_fingerprint` must validate against `UserXmppOmemoTrustedKey`. +2. Preserve backwards compatibility via migration path: + - existing latest-key behavior can be temporarily represented as an optional fallback mode. +3. Update labels/help to match exact behavior. +4. Add tests: + - trusted key allows + - untrusted key denies + - unknown key denies + +## Phase 3: Commands/Tasks Control Plane Alignment +1. Unify command surface definitions: + - Command Routing create/edit options include all supported command slugs (`bp`, `codex`, `claude`). +2. Add explicit cross-links: + - Tasks settings references Command Routing and Permissions scopes directly. + - Command Routing references Permissions scopes affecting each profile. +3. Introduce capability-impact preview panel: + - for each command/task action, show effective allow/deny by scope and channel. + +## Phase 4: Settings Shell Normalization +1. Replace route-name allowlists in `settings_hierarchy_nav` with category mapping table. +2. Ensure all `/settings/*` pages declare category + tab metadata. +3. Include missing routes (`codex_settings`, `codex_approval`, `translation_preview`) in shell. +4. Add test to fail when a `/settings/*` route lacks shell metadata. + +## Phase 5: Documentation Synchronization +1. Add a settings matrix doc generated (or validated) from the capability registry: + - capability key + - UI location + - enforced by code path +2. Update `README.md` and `INSTALL.md` security/modules sections. +3. Add "policy semantics" section clarifying: + - encryption-required vs per-scope OMEMO requirements + - trusted key behavior + - global override precedence + +## Acceptance Criteria +- Every enforced scope is user-visible/configurable (or intentionally internal and documented). +- “Require Trusted Fingerprint” enforcement uses trust records, not only latest observed key. +- Command Routing and runtime-supported command slugs are aligned. +- All `/settings/*` pages show consistent settings shell navigation. +- Security/tasks/commands docs reflect real behavior and pass integrity checks. + +## Risks and Mitigations +- Risk: policy behavior change blocks existing workflows. + - Mitigation: add compatibility flag and staged rollout. +- Risk: registry migration introduces missing scope mappings. + - Mitigation: integrity test that compares runtime-enforced keys vs registry. +- Risk: UI complexity increase. + - Mitigation: keep layered model with concise, context-aware summaries. + +## Implementation Notes +- Keep migration incremental; avoid big-bang rewrite. +- Prioritize Phase 1 + Phase 2 first, because they are correctness and security semantics issues. +- Do not add new transport-specific branches; keep service-agnostic evaluation path in policy engine. diff --git a/core/clients/signal.py b/core/clients/signal.py index 71c0746..30dc1bf 100644 --- a/core/clients/signal.py +++ b/core/clients/signal.py @@ -280,6 +280,43 @@ def _extract_signal_delete(envelope): } +def _extract_signal_group_identifiers(payload: dict) -> list[str]: + envelope = payload.get("envelope") or {} + if not isinstance(envelope, dict): + return [] + + candidates = [] + paths = [ + ("group",), + ("groupV2",), + ("dataMessage", "groupInfo"), + ("dataMessage", "groupV2"), + ("syncMessage", "sentMessage", "groupInfo"), + ("syncMessage", "sentMessage", "groupV2"), + ("syncMessage", "sentMessage", "message", "groupInfo"), + ("syncMessage", "sentMessage", "message", "groupV2"), + ] + key_names = ("id", "groupId", "groupID", "internal_id", "masterKey") + for path in paths: + node = _get_nested(envelope, path) + if isinstance(node, str): + candidates.append(node) + continue + if not isinstance(node, dict): + continue + for key_name in key_names: + value = str(node.get(key_name) or "").strip() + if value: + candidates.append(value) + + unique = [] + for value in candidates: + cleaned = str(value or "").strip() + if cleaned and cleaned not in unique: + unique.append(cleaned) + return unique + + def _extract_signal_text(raw_payload, default_text=""): text = str(default_text or "").strip() if text: @@ -332,6 +369,22 @@ def _identifier_candidates(*values): return out +def _dedupe_person_identifiers(rows): + deduped = [] + seen = set() + for row in rows or []: + key = ( + int(getattr(row, "user_id", 0) or 0), + str(getattr(row, "person_id", "") or ""), + str(getattr(row, "service", "") or "").strip().lower(), + ) + if key in seen: + continue + seen.add(key) + deduped.append(row) + return deduped + + def _digits_only(value): return re.sub(r"[^0-9]", "", str(value or "").strip()) @@ -762,6 +815,31 @@ class HandleMessage(Command): log.warning("Signal reaction relay to XMPP failed: %s", exc) return + if reply_to_self and str(text or "").strip().startswith("."): + responded_user_ids = set() + for identifier in identifiers: + if identifier.user_id in responded_user_ids: + continue + responded_user_ids.add(identifier.user_id) + gateway_replies = await self.ur.xmpp.client.execute_gateway_command( + sender_user=identifier.user, + body=text, + service=self.service, + channel_identifier=str(identifier.identifier or ""), + sender_identifier=str(identifier.identifier or ""), + local_message=None, + message_meta={ + "signal": { + "source_uuid": str(effective_source_uuid or ""), + "source_number": str(effective_source_number or ""), + "reply_to_self": True, + } + }, + ) + for line in gateway_replies: + await c.send(f"[>] {line}") + return + # Handle attachments across multiple Signal payload variants. attachment_list = _extract_attachments(raw) xmpp_attachments = [] @@ -1087,6 +1165,130 @@ class SignalClient(ClientBase): self.client.register(HandleMessage(self.ur, self.service)) self._command_task = None self._raw_receive_task = None + self._catalog_refresh_task = None + + async def _signal_api_get_list( + self, session: aiohttp.ClientSession, path: str + ) -> list[dict]: + base = str(getattr(settings, "SIGNAL_HTTP_URL", "http://signal:8080")).rstrip( + "/" + ) + try: + async with session.get(f"{base}{path}") as response: + if response.status != 200: + body = str(await response.text()).strip()[:300] + self.log.warning( + "signal catalog fetch failed path=%s status=%s body=%s", + path, + response.status, + body or "-", + ) + return [] + payload = await response.json(content_type=None) + except Exception as exc: + self.log.warning("signal catalog fetch failed path=%s error=%s", path, exc) + return [] + return payload if isinstance(payload, list) else [] + + async def _refresh_runtime_catalog(self) -> None: + signal_number = str(getattr(settings, "SIGNAL_NUMBER", "") or "").strip() + if not signal_number: + return + + encoded_account = quote_plus(signal_number) + timeout = aiohttp.ClientTimeout(total=20) + async with aiohttp.ClientSession(timeout=timeout) as session: + identities = await self._signal_api_get_list( + session, f"/v1/identities/{encoded_account}" + ) + groups = await self._signal_api_get_list( + session, f"/v1/groups/{encoded_account}" + ) + + account_digits = _digits_only(signal_number) + contact_rows = [] + seen_contacts = set() + for identity in identities: + if not isinstance(identity, dict): + continue + number = str(identity.get("number") or "").strip() + uuid = str(identity.get("uuid") or "").strip() + if account_digits and number and _digits_only(number) == account_digits: + continue + identifiers = [] + for candidate in (number, uuid): + cleaned = str(candidate or "").strip() + if cleaned and cleaned not in identifiers: + identifiers.append(cleaned) + if not identifiers: + continue + key = ( + f"uuid:{uuid.lower()}" + if uuid + else f"phone:{_digits_only(number) or number}" + ) + if key in seen_contacts: + continue + seen_contacts.add(key) + contact_rows.append( + { + "identifier": number or uuid, + "identifiers": identifiers, + "name": str(identity.get("name") or "").strip(), + "number": number, + "uuid": uuid, + } + ) + + group_rows = [] + seen_groups = set() + for group in groups: + if not isinstance(group, dict): + continue + group_id = str(group.get("id") or "").strip() + internal_id = str(group.get("internal_id") or "").strip() + identifiers = [] + for candidate in (group_id, internal_id): + cleaned = str(candidate or "").strip() + if cleaned and cleaned not in identifiers: + identifiers.append(cleaned) + if not identifiers: + continue + key = group_id or internal_id + if key in seen_groups: + continue + seen_groups.add(key) + group_rows.append( + { + "identifier": group_id or internal_id, + "identifiers": identifiers, + "name": str(group.get("name") or "").strip(), + "id": group_id, + "internal_id": internal_id, + } + ) + + transport.update_runtime_state( + self.service, + accounts=[signal_number], + contacts=contact_rows, + groups=group_rows, + catalog_refreshed_at=int(time.time()), + catalog_error="", + ) + + async def _refresh_runtime_catalog_safe(self) -> None: + try: + await self._refresh_runtime_catalog() + except asyncio.CancelledError: + raise + except Exception as exc: + transport.update_runtime_state( + self.service, + catalog_error=str(exc).strip()[:300], + catalog_refreshed_at=int(time.time()), + ) + self.log.warning("signal catalog refresh failed: %s", exc) async def _drain_runtime_commands(self): """Process queued runtime commands (e.g., web UI sends via composite router).""" @@ -1237,7 +1439,104 @@ class SignalClient(ClientBase): self.log.warning(f"Command loop error: {exc}") await asyncio.sleep(1) - async def _resolve_signal_identifiers(self, source_uuid: str, source_number: str): + async def _resolve_signal_group_identifiers(self, group_candidates: list[str]): + unique_candidates = [] + for value in group_candidates or []: + cleaned = str(value or "").strip() + if cleaned and cleaned not in unique_candidates: + unique_candidates.append(cleaned) + if not unique_candidates: + return [] + + runtime_groups = transport.get_runtime_state(self.service).get("groups") or [] + expanded = list(unique_candidates) + for item in runtime_groups: + if not isinstance(item, dict): + continue + identifiers = [] + for candidate in item.get("identifiers") or []: + cleaned = str(candidate or "").strip() + if cleaned: + identifiers.append(cleaned) + if not identifiers: + continue + if not any(candidate in identifiers for candidate in unique_candidates): + continue + for candidate in identifiers: + if candidate not in expanded: + expanded.append(candidate) + + exact_identifiers = await sync_to_async(list)( + PersonIdentifier.objects.filter( + service=self.service, + identifier__in=expanded, + ).select_related("user", "person") + ) + if exact_identifiers: + return exact_identifiers + + bare_candidates = [] + for candidate in expanded: + bare = str(candidate or "").strip().split("@", 1)[0].strip() + if bare and bare not in bare_candidates: + bare_candidates.append(bare) + + links = await sync_to_async(list)( + PlatformChatLink.objects.filter( + service=self.service, + is_group=True, + chat_identifier__in=bare_candidates, + ) + .select_related("user", "person_identifier", "person") + .order_by("id") + ) + if not links: + return [] + + results = [] + seen_ids = set() + for link in links: + if link.person_identifier_id: + if link.person_identifier_id not in seen_ids: + seen_ids.add(link.person_identifier_id) + results.append(link.person_identifier) + continue + if not link.person_id: + continue + group_pi = await sync_to_async( + lambda: PersonIdentifier.objects.filter( + user=link.user, + person=link.person, + service=self.service, + identifier__in=expanded, + ) + .select_related("user", "person") + .first() + )() + if group_pi is None: + group_pi = await sync_to_async( + lambda: PersonIdentifier.objects.filter( + user=link.user, + person=link.person, + service=self.service, + ) + .select_related("user", "person") + .first() + )() + if group_pi is not None and group_pi.id not in seen_ids: + seen_ids.add(group_pi.id) + results.append(group_pi) + return results + + async def _resolve_signal_identifiers( + self, + source_uuid: str, + source_number: str, + group_candidates: list[str] | None = None, + ): + group_rows = await self._resolve_signal_group_identifiers(group_candidates or []) + if group_rows: + return _dedupe_person_identifiers(group_rows) candidates = _identifier_candidates(source_uuid, source_number) if not candidates: return [] @@ -1248,7 +1547,7 @@ class SignalClient(ClientBase): ) ) if identifiers: - return identifiers + return _dedupe_person_identifiers(identifiers) candidate_digits = {_digits_only(value) for value in candidates} candidate_digits = {value for value in candidate_digits if value} if not candidate_digits: @@ -1256,11 +1555,13 @@ class SignalClient(ClientBase): rows = await sync_to_async(list)( PersonIdentifier.objects.filter(service=self.service).select_related("user") ) - return [ - row - for row in rows - if _digits_only(getattr(row, "identifier", "")) in candidate_digits - ] + return _dedupe_person_identifiers( + [ + row + for row in rows + if _digits_only(getattr(row, "identifier", "")) in candidate_digits + ] + ) async def _auto_link_single_user_signal_identifier( self, source_uuid: str, source_number: str @@ -1301,7 +1602,123 @@ class SignalClient(ClientBase): fallback_identifier, int(owner.id), ) - return [pi] + return _dedupe_person_identifiers([pi]) + + async def _build_xmpp_relay_attachments(self, payload: dict): + attachment_list = _extract_attachments(payload) + xmpp_attachments = [] + compose_media_urls = [] + if not attachment_list: + return xmpp_attachments, compose_media_urls + + fetched_attachments = await asyncio.gather( + *[signalapi.fetch_signal_attachment(att["id"]) for att in attachment_list] + ) + for fetched, att in zip(fetched_attachments, attachment_list): + if not fetched: + self.log.warning( + "signal raw attachment fetch failed attachment_id=%s", att["id"] + ) + continue + xmpp_attachments.append( + { + "content": fetched["content"], + "content_type": fetched["content_type"], + "filename": fetched["filename"], + "size": fetched["size"], + } + ) + blob_key = media_bridge.put_blob( + service="signal", + content=fetched["content"], + filename=fetched["filename"], + content_type=fetched["content_type"], + ) + if blob_key: + compose_media_urls.append( + f"/compose/media/blob/?key={quote_plus(str(blob_key))}" + ) + return xmpp_attachments, compose_media_urls + + async def _relay_signal_inbound_to_xmpp( + self, + *, + identifiers, + relay_text, + xmpp_attachments, + compose_media_urls, + source_uuid, + source_number, + ts, + ): + resolved_text_by_session = {} + for identifier in identifiers: + user = identifier.user + session_key = (identifier.user.id, identifier.person.id) + mutate_manips = await sync_to_async(list)( + Manipulation.objects.filter( + group__people=identifier.person, + user=identifier.user, + mode="mutate", + filter_enabled=True, + enabled=True, + ) + ) + if mutate_manips: + uploaded_urls = [] + for manip in mutate_manips: + prompt = replies.generate_mutate_reply_prompt( + relay_text, + None, + manip, + None, + ) + result = await ai.run_prompt( + prompt, + manip.ai, + operation="signal_mutate", + ) + uploaded_urls = await self.ur.xmpp.client.send_from_external( + user, + identifier, + result, + False, + attachments=xmpp_attachments, + source_ref={ + "upstream_message_id": "", + "upstream_author": str( + source_uuid or source_number or "" + ), + "upstream_ts": int(ts or 0), + }, + ) + resolved_text = relay_text + if (not resolved_text) and uploaded_urls: + resolved_text = "\n".join(uploaded_urls) + elif (not resolved_text) and compose_media_urls: + resolved_text = "\n".join(compose_media_urls) + resolved_text_by_session[session_key] = resolved_text + continue + + uploaded_urls = await self.ur.xmpp.client.send_from_external( + user, + identifier, + relay_text, + False, + attachments=xmpp_attachments, + source_ref={ + "upstream_message_id": "", + "upstream_author": str(source_uuid or source_number or ""), + "upstream_ts": int(ts or 0), + }, + ) + resolved_text = relay_text + if (not resolved_text) and uploaded_urls: + resolved_text = "\n".join(uploaded_urls) + elif (not resolved_text) and compose_media_urls: + resolved_text = "\n".join(compose_media_urls) + resolved_text_by_session[session_key] = resolved_text + return resolved_text_by_session async def _process_raw_inbound_event(self, raw_message: str): try: @@ -1361,6 +1778,15 @@ class SignalClient(ClientBase): envelope = payload.get("envelope") or {} if not isinstance(envelope, dict): return + group_candidates = _extract_signal_group_identifiers(payload) + preferred_group_id = "" + for candidate in group_candidates: + cleaned = str(candidate or "").strip() + if cleaned.startswith("group."): + preferred_group_id = cleaned + break + if not preferred_group_id and group_candidates: + preferred_group_id = str(group_candidates[0] or "").strip() sync_sent_message = _get_nested(envelope, ("syncMessage", "sentMessage")) or {} if isinstance(sync_sent_message, dict) and sync_sent_message: raw_text = sync_sent_message.get("message") @@ -1395,6 +1821,7 @@ class SignalClient(ClientBase): identifiers = await self._resolve_signal_identifiers( destination_uuid, destination_number, + group_candidates, ) if not identifiers: identifiers = await self._auto_link_single_user_signal_identifier( @@ -1532,7 +1959,12 @@ class SignalClient(ClientBase): or str(payload.get("account") or "").strip() or "self" ) - source_chat_id = destination_number or destination_uuid or sender_key + source_chat_id = ( + preferred_group_id + or destination_number + or destination_uuid + or sender_key + ) reply_ref = reply_sync.extract_reply_ref(self.service, payload) for identifier in identifiers: session = await history.get_chat_session( @@ -1599,7 +2031,11 @@ class SignalClient(ClientBase): ): return - identifiers = await self._resolve_signal_identifiers(source_uuid, source_number) + identifiers = await self._resolve_signal_identifiers( + source_uuid, + source_number, + group_candidates, + ) reaction_payload = _extract_signal_reaction(envelope) edit_payload = _extract_signal_edit(envelope) delete_payload = _extract_signal_delete(envelope) @@ -1620,6 +2056,7 @@ class SignalClient(ClientBase): identifiers = await self._resolve_signal_identifiers( destination_uuid, destination_number, + group_candidates, ) if not identifiers: identifiers = await self._auto_link_single_user_signal_identifier( @@ -1730,7 +2167,13 @@ class SignalClient(ClientBase): text = _extract_signal_text( payload, str(data_message.get("message") or "").strip() ) - if not text: + relay_text = text + xmpp_attachments, compose_media_urls = await self._build_xmpp_relay_attachments( + payload + ) + if xmpp_attachments and _is_compose_blob_only_text(relay_text): + relay_text = "" + if not relay_text and not xmpp_attachments: return ts_raw = ( @@ -1753,8 +2196,17 @@ class SignalClient(ClientBase): or source_number or (identifiers[0].identifier if identifiers else "") ) - source_chat_id = source_number or source_uuid or sender_key + source_chat_id = preferred_group_id or source_number or source_uuid or sender_key reply_ref = reply_sync.extract_reply_ref(self.service, payload) + resolved_text_by_session = await self._relay_signal_inbound_to_xmpp( + identifiers=identifiers, + relay_text=relay_text, + xmpp_attachments=xmpp_attachments, + compose_media_urls=compose_media_urls, + source_uuid=source_uuid, + source_number=source_number, + ts=ts, + ) for identifier in identifiers: session = await history.get_chat_session(identifier.user, identifier) @@ -1773,10 +2225,14 @@ class SignalClient(ClientBase): )() if exists: continue + message_text = resolved_text_by_session.get( + (identifier.user.id, identifier.person.id), + relay_text if relay_text else "\n".join(compose_media_urls), + ) local_message = await history.store_message( session=session, sender=sender_key, - text=text, + text=message_text, ts=ts, outgoing=False, source_service=self.service, @@ -1792,7 +2248,7 @@ class SignalClient(ClientBase): await self.ur.message_received( self.service, identifier=identifier, - text=text, + text=message_text, ts=ts, payload=payload, local_message=local_message, @@ -1809,16 +2265,45 @@ class SignalClient(ClientBase): if not signal_number: return uri = f"ws://{SIGNAL_URL}/v1/receive/{signal_number}" + poll_uri = f"http://{SIGNAL_URL}/v1/receive/{signal_number}" + use_http_polling = False while not self._stopping: try: + transport.update_runtime_state(self.service, accounts=[signal_number]) + if use_http_polling: + timeout = aiohttp.ClientTimeout(total=15) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(poll_uri) as response: + response.raise_for_status() + payload = await response.json(content_type=None) + if isinstance(payload, dict): + payload = [payload] + if not isinstance(payload, list): + payload = [] + for item in payload: + if not isinstance(item, dict): + continue + await self._process_raw_inbound_event(json.dumps(item)) + continue async with websockets.connect(uri, ping_interval=None) as websocket: async for raw_message in websocket: await self._process_raw_inbound_event(raw_message) except asyncio.CancelledError: raise except Exception as exc: + if ( + not use_http_polling + and "server rejected WebSocket connection: HTTP 200" in str(exc) + ): + self.log.info( + "signal raw-receive switching to HTTP polling for %s", + signal_number, + ) + use_http_polling = True + continue self.log.warning("signal raw-receive loop error: %s", exc) await asyncio.sleep(2) + transport.update_runtime_state(self.service, accounts=[]) def start(self): self.log.info("Signal client starting...") @@ -1828,6 +2313,10 @@ class SignalClient(ClientBase): self._command_task = self.loop.create_task(self._command_loop()) if not self._raw_receive_task or self._raw_receive_task.done(): self._raw_receive_task = self.loop.create_task(self._raw_receive_loop()) - # Use direct websocket receive loop as primary ingestion path. - # signalbot's internal receive consumer can compete for the same stream - # and starve inbound events in this deployment, so we keep it disabled. + if not self._catalog_refresh_task or self._catalog_refresh_task.done(): + self._catalog_refresh_task = self.loop.create_task( + self._refresh_runtime_catalog_safe() + ) + # Keep signalbot's internal receive consumer disabled to avoid competing + # consumers. The raw loop adapts between websocket and HTTP polling + # depending on the deployed signal-cli-rest-api behavior. diff --git a/core/clients/transport.py b/core/clients/transport.py index f9a8212..3da5c92 100644 --- a/core/clients/transport.py +++ b/core/clients/transport.py @@ -5,6 +5,7 @@ import os import secrets import shutil import time +from pathlib import Path from typing import Any from urllib.parse import quote_plus @@ -444,23 +445,6 @@ def list_accounts(service: str): Return account identifiers for service UI list. """ service_key = _service_key(service) - if service_key == "signal": - import requests - - base = str(getattr(settings, "SIGNAL_HTTP_URL", "http://signal:8080")).rstrip( - "/" - ) - try: - response = requests.get(f"{base}/v1/accounts", timeout=20) - if not response.ok: - return [] - payload = orjson.loads(response.text or "[]") - if isinstance(payload, list): - return payload - except Exception: - return [] - return [] - state = get_runtime_state(service_key) accounts = state.get("accounts") or [] if service_key == "whatsapp" and not accounts: @@ -495,13 +479,24 @@ def _wipe_signal_cli_local_state() -> bool: Best-effort local signal-cli state reset for json-rpc deployments where REST account delete endpoints are unavailable. """ - config_roots = ( - "/code/signal-cli-config", - "/signal-cli-config", - "/home/.local/share/signal-cli", + config_roots = [] + base_dir = getattr(settings, "BASE_DIR", None) + if base_dir: + config_roots.append(str(Path(base_dir) / "signal-cli-config")) + config_roots.extend( + [ + "/code/signal-cli-config", + "/signal-cli-config", + "/home/.local/share/signal-cli", + ] ) removed_any = False + seen_roots = set() for root in config_roots: + root = str(root or "").strip() + if not root or root in seen_roots: + continue + seen_roots.add(root) if not os.path.isdir(root): continue try: @@ -554,7 +549,26 @@ def unlink_account(service: str, account: str) -> bool: continue if unlinked: return True - return _wipe_signal_cli_local_state() + wiped = _wipe_signal_cli_local_state() + if not wiped: + return False + # Best-effort verification: if the REST API still reports the same account, + # the runtime likely still holds active linked state and the UI should not + # claim relink is ready yet. + remaining_accounts = list_accounts("signal") + for row in remaining_accounts: + if isinstance(row, dict): + candidate = ( + row.get("number") + or row.get("id") + or row.get("jid") + or row.get("account") + ) + else: + candidate = row + if _account_key(str(candidate or "")) == _account_key(account_value): + return False + return True if service_key in {"whatsapp", "instagram"}: state = get_runtime_state(service_key) @@ -842,9 +856,7 @@ async def send_message_raw( runtime_result = await runtime_client.send_message_raw( recipient, text=text, - attachments=await prepare_outbound_attachments( - service_key, attachments or [] - ), + attachments=attachments or [], metadata=dict(metadata or {}), ) if runtime_result is not False and runtime_result is not None: @@ -853,11 +865,8 @@ async def send_message_raw( log.warning("%s runtime send failed: %s", service_key, exc) # Web/UI process cannot access UR in-process runtime client directly. # Hand off send to UR via shared cache command queue. - prepared_attachments = await prepare_outbound_attachments( - service_key, attachments or [] - ) command_attachments = [] - for att in prepared_attachments: + for att in (attachments or []): row = dict(att or {}) # Keep payload cache-friendly and avoid embedding raw bytes. for key in ("content",): @@ -1082,9 +1091,8 @@ async def fetch_attachment(service: str, attachment_ref: dict): service_key = _service_key(service) if service_key == "signal": attachment_id = attachment_ref.get("id") or attachment_ref.get("attachment_id") - if not attachment_id: - return None - return await signalapi.fetch_signal_attachment(attachment_id) + if attachment_id: + return await signalapi.fetch_signal_attachment(attachment_id) runtime_client = get_runtime_client(service_key) if runtime_client and hasattr(runtime_client, "fetch_attachment"): @@ -1160,7 +1168,7 @@ def get_link_qr(service: str, device_name: str): response = requests.get( f"{base}/v1/qrcodelink", params={"device_name": device}, - timeout=20, + timeout=5, ) response.raise_for_status() return response.content diff --git a/core/clients/whatsapp.py b/core/clients/whatsapp.py index be80b95..e77d0c1 100644 --- a/core/clients/whatsapp.py +++ b/core/clients/whatsapp.py @@ -2105,45 +2105,28 @@ class WhatsAppClient(ClientBase): """ Extract user-visible text from diverse WhatsApp message payload shapes. """ - candidates = ( - self._pluck(msg_obj, "conversation"), - self._pluck(msg_obj, "Conversation"), - self._pluck(msg_obj, "extendedTextMessage", "text"), - self._pluck(msg_obj, "ExtendedTextMessage", "Text"), - self._pluck(msg_obj, "extended_text_message", "text"), - self._pluck(msg_obj, "imageMessage", "caption"), - self._pluck(msg_obj, "videoMessage", "caption"), - self._pluck(msg_obj, "documentMessage", "caption"), - self._pluck(msg_obj, "ephemeralMessage", "message", "conversation"), - self._pluck( - msg_obj, "ephemeralMessage", "message", "extendedTextMessage", "text" - ), - self._pluck(msg_obj, "viewOnceMessage", "message", "conversation"), - self._pluck( - msg_obj, "viewOnceMessage", "message", "extendedTextMessage", "text" - ), - self._pluck(msg_obj, "viewOnceMessageV2", "message", "conversation"), - self._pluck( - msg_obj, "viewOnceMessageV2", "message", "extendedTextMessage", "text" - ), - self._pluck( - msg_obj, "viewOnceMessageV2Extension", "message", "conversation" - ), - self._pluck( - msg_obj, - "viewOnceMessageV2Extension", - "message", - "extendedTextMessage", - "text", - ), + for candidate in self._iter_message_variants(msg_obj): + for value in ( + self._pluck(candidate, "conversation"), + self._pluck(candidate, "Conversation"), + self._pluck(candidate, "extendedTextMessage", "text"), + self._pluck(candidate, "ExtendedTextMessage", "Text"), + self._pluck(candidate, "extended_text_message", "text"), + self._pluck(candidate, "imageMessage", "caption"), + self._pluck(candidate, "videoMessage", "caption"), + self._pluck(candidate, "documentMessage", "caption"), + ): + text = str(value or "").strip() + if text: + return text + for value in ( self._pluck(event_obj, "message", "conversation"), self._pluck(event_obj, "message", "extendedTextMessage", "text"), self._pluck(event_obj, "Message", "conversation"), self._pluck(event_obj, "Message", "extendedTextMessage", "text"), self._pluck(event_obj, "conversation"), self._pluck(event_obj, "text"), - ) - for value in candidates: + ): text = str(value or "").strip() if text: return text @@ -2318,7 +2301,40 @@ class WhatsAppClient(ClientBase): return str(user) return raw - def _is_media_message(self, message_obj): + def _iter_message_variants(self, message_obj, max_depth: int = 8): + wrapper_paths = ( + ("deviceSentMessage", "message"), + ("DeviceSentMessage", "Message"), + ("ephemeralMessage", "message"), + ("EphemeralMessage", "Message"), + ("viewOnceMessage", "message"), + ("ViewOnceMessage", "Message"), + ("viewOnceMessageV2", "message"), + ("ViewOnceMessageV2", "Message"), + ("viewOnceMessageV2Extension", "message"), + ("ViewOnceMessageV2Extension", "Message"), + ("editedMessage", "message"), + ("EditedMessage", "Message"), + ) + queue = [(message_obj, 0)] + seen = set() + while queue: + current, depth = queue.pop(0) + if current is None: + continue + marker = id(current) + if marker in seen: + continue + seen.add(marker) + yield current + if depth >= max_depth: + continue + for path in wrapper_paths: + nested = self._pluck(current, *path) + if nested is not None: + queue.append((nested, depth + 1)) + + def _direct_media_payload(self, message_obj): media_fields = ( "imageMessage", "videoMessage", @@ -2334,8 +2350,17 @@ class WhatsAppClient(ClientBase): for field in media_fields: value = self._pluck(message_obj, field) if value: - return True - return False + return value + return None + + def _resolve_media_message(self, message_obj): + for candidate in self._iter_message_variants(message_obj): + if self._direct_media_payload(candidate): + return candidate + return None + + def _is_media_message(self, message_obj): + return self._resolve_media_message(message_obj) is not None def _infer_media_content_type(self, message_obj): if self._pluck(message_obj, "imageMessage") or self._pluck( @@ -2439,13 +2464,14 @@ class WhatsAppClient(ClientBase): if not self._client: return [] msg_obj = self._pluck(event, "message") or self._pluck(event, "Message") - if msg_obj is None or not self._is_media_message(msg_obj): + media_msg = self._resolve_media_message(msg_obj) + if media_msg is None: return [] if not hasattr(self._client, "download_any"): return [] try: - payload = await self._maybe_await(self._client.download_any(msg_obj)) + payload = await self._maybe_await(self._client.download_any(media_msg)) except Exception as exc: self.log.warning("whatsapp media download failed: %s", exc) return [] @@ -2455,19 +2481,21 @@ class WhatsAppClient(ClientBase): if not isinstance(payload, (bytes, bytearray)): return [] - filename = self._pluck(msg_obj, "documentMessage", "fileName") or self._pluck( - msg_obj, "document_message", "file_name" + filename = self._pluck( + media_msg, "documentMessage", "fileName" + ) or self._pluck( + media_msg, "document_message", "file_name" ) content_type = ( - self._pluck(msg_obj, "documentMessage", "mimetype") - or self._pluck(msg_obj, "document_message", "mimetype") - or self._pluck(msg_obj, "imageMessage", "mimetype") - or self._pluck(msg_obj, "image_message", "mimetype") - or self._pluck(msg_obj, "videoMessage", "mimetype") - or self._pluck(msg_obj, "video_message", "mimetype") - or self._pluck(msg_obj, "audioMessage", "mimetype") - or self._pluck(msg_obj, "audio_message", "mimetype") - or self._infer_media_content_type(msg_obj) + self._pluck(media_msg, "documentMessage", "mimetype") + or self._pluck(media_msg, "document_message", "mimetype") + or self._pluck(media_msg, "imageMessage", "mimetype") + or self._pluck(media_msg, "image_message", "mimetype") + or self._pluck(media_msg, "videoMessage", "mimetype") + or self._pluck(media_msg, "video_message", "mimetype") + or self._pluck(media_msg, "audioMessage", "mimetype") + or self._pluck(media_msg, "audio_message", "mimetype") + or self._infer_media_content_type(media_msg) ) if not filename: ext = mimetypes.guess_extension( @@ -2651,6 +2679,38 @@ class WhatsAppClient(ClientBase): if not identifiers: return + is_self_chat = bool( + is_from_me + and str(sender or "").strip() + and str(chat or "").strip() + and str(sender).strip() == str(chat).strip() + ) + if is_self_chat and str(text or "").strip().startswith("."): + responded_user_ids = set() + reply_target = str(chat or sender or "").strip() + for identifier in identifiers: + if identifier.user_id in responded_user_ids: + continue + responded_user_ids.add(identifier.user_id) + replies = await self.ur.xmpp.client.execute_gateway_command( + sender_user=identifier.user, + body=text, + service=self.service, + channel_identifier=str(identifier.identifier or ""), + sender_identifier=str(identifier.identifier or ""), + local_message=None, + message_meta={ + "whatsapp": { + "sender": str(sender or ""), + "chat": str(chat or ""), + "self_chat": True, + } + }, + ) + for line in replies: + await self.send_message_raw(reply_target, f"[>] {line}") + return + attachments = await self._download_event_media(event) xmpp_attachments = [] if attachments: @@ -3186,28 +3246,20 @@ class WhatsAppClient(ClientBase): url = (attachment or {}).get("url") if url: safe_url = validate_attachment_url(url) - timeout = aiohttp.ClientTimeout(total=20) - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.get(safe_url) as response: - if response.status != 200: - return None - payload = await response.read() - filename, content_type = validate_attachment_metadata( - filename=(attachment or {}).get("filename") - or safe_url.rstrip("/").split("/")[-1] - or "attachment.bin", - content_type=(attachment or {}).get("content_type") - or response.headers.get( - "Content-Type", "application/octet-stream" - ), - size=len(payload), - ) - return { - "content": payload, - "filename": filename, - "content_type": content_type, - "size": len(payload), - } + filename, content_type = validate_attachment_metadata( + filename=(attachment or {}).get("filename") + or safe_url.rstrip("/").split("/")[-1] + or "attachment.bin", + content_type=(attachment or {}).get("content_type") + or "application/octet-stream", + size=(attachment or {}).get("size"), + ) + return { + "url": safe_url, + "filename": filename, + "content_type": content_type, + "size": (attachment or {}).get("size"), + } return None async def send_message_raw( @@ -3364,18 +3416,26 @@ class WhatsAppClient(ClientBase): payload = await self._fetch_attachment_payload(attachment) if not payload: continue - data = payload.get("content") or b"" + data = payload.get("content") + source_url = str(payload.get("url") or "").strip() try: filename, mime = validate_attachment_metadata( filename=payload.get("filename") or "attachment.bin", content_type=payload.get("content_type") or "application/octet-stream", size=payload.get("size") - or (len(data) if isinstance(data, (bytes, bytearray)) else 0), + or (len(data) if isinstance(data, (bytes, bytearray)) else None), ) except Exception as exc: self.log.warning("whatsapp blocked attachment: %s", exc) continue + file_arg = ( + data + if isinstance(data, (bytes, bytearray)) + else source_url + ) + if not file_arg: + continue mime = str(mime).lower() attachment_target = jid_obj if jid_obj is not None else jid send_method = "document" @@ -3392,27 +3452,31 @@ class WhatsAppClient(ClientBase): send_method, mime, filename, - len(data) if isinstance(data, (bytes, bytearray)) else 0, + ( + len(data) + if isinstance(data, (bytes, bytearray)) + else (payload.get("size") or 0) + ), ) try: if mime.startswith("image/") and hasattr(self._client, "send_image"): response = await self._maybe_await( - self._client.send_image(attachment_target, data, caption="") + self._client.send_image(attachment_target, file_arg, caption="") ) elif mime.startswith("video/") and hasattr(self._client, "send_video"): response = await self._maybe_await( - self._client.send_video(attachment_target, data, caption="") + self._client.send_video(attachment_target, file_arg, caption="") ) elif mime.startswith("audio/") and hasattr(self._client, "send_audio"): response = await self._maybe_await( - self._client.send_audio(attachment_target, data) + self._client.send_audio(attachment_target, file_arg) ) elif hasattr(self._client, "send_document"): response = await self._maybe_await( self._client.send_document( attachment_target, - data, + file_arg, filename=filename, mimetype=mime, caption="", diff --git a/core/clients/xmpp.py b/core/clients/xmpp.py index 3d6cbd0..54d935d 100644 --- a/core/clients/xmpp.py +++ b/core/clients/xmpp.py @@ -7,7 +7,7 @@ import re import time import uuid from pathlib import Path -from urllib.parse import parse_qs, urlparse, urlsplit +from urllib.parse import urlsplit import aiohttp from asgiref.sync import sync_to_async @@ -15,6 +15,7 @@ from django.conf import settings from django.utils.timezone import now from slixmpp.componentxmpp import ComponentXMPP from slixmpp.plugins.xep_0085.stanza import Active, Composing, Gone, Inactive, Paused +from slixmpp.plugins.xep_0356.permissions import MessagePermission from slixmpp.stanza import Message from slixmpp.xmlstream import register_stanza_plugin from slixmpp.xmlstream.handler import Callback @@ -22,18 +23,13 @@ from slixmpp.xmlstream.matcher import StanzaPath from slixmpp.xmlstream.stanzabase import ET from core.clients import ClientBase, transport -from core.gateway.commands import ( - GatewayCommandContext, - GatewayCommandRoute, - dispatch_gateway_command, +from core.gateway.builtin import ( + dispatch_builtin_gateway_command, + gateway_help_lines, ) from core.messaging import ai, history, replies, reply_sync, utils from core.models import ( ChatSession, - CodexPermissionRequest, - CodexRun, - DerivedTask, - ExternalSyncEvent, Manipulation, PatternMitigationAutoSettings, PatternMitigationCorrection, @@ -53,6 +49,10 @@ from core.security.attachments import ( from core.util import logs URL_PATTERN = re.compile(r"https?://[^\s<>'\"\\]+") +SIGNAL_UUID_PATTERN = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + re.IGNORECASE, +) EMOJI_ONLY_PATTERN = re.compile( r"^[\U0001F300-\U0001FAFF\u2600-\u27BF\uFE0F\u200D\u2640-\u2642\u2764]+$" ) @@ -60,12 +60,114 @@ TOTP_BASE32_SECRET_RE = re.compile(r"^[A-Z2-7]{16,}$") PUBSUB_NS = "http://jabber.org/protocol/pubsub" OMEMO_OLD_NS = "eu.siacs.conversations.axolotl" OMEMO_OLD_DEVICELIST_NODE = "eu.siacs.conversations.axolotl.devicelist" +XMPP_LOCALPART_ESCAPE_MAP = { + "\\": r"\5c", + " ": r"\20", + '"': r"\22", + "&": r"\26", + "'": r"\27", + "/": r"\2f", + ":": r"\3a", + "<": r"\3c", + ">": r"\3e", + "@": r"\40", + "|": r"\7c", +} +XMPP_LOCALPART_UNESCAPE_MAP = { + value: key for key, value in XMPP_LOCALPART_ESCAPE_MAP.items() +} +XMPP_LOCALPART_UNESCAPE_RE = re.compile( + "|".join( + sorted( + (re.escape(key) for key in XMPP_LOCALPART_UNESCAPE_MAP.keys()), + key=len, + reverse=True, + ) + ) +) def _clean_url(value): return str(value or "").strip().rstrip(".,);:!?\"'") +def _escape_xmpp_localpart(value): + return "".join(XMPP_LOCALPART_ESCAPE_MAP.get(ch, ch) for ch in str(value or "")) + + +def _unescape_xmpp_localpart(value): + text = str(value or "") + if not text: + return "" + return XMPP_LOCALPART_UNESCAPE_RE.sub( + lambda match: XMPP_LOCALPART_UNESCAPE_MAP.get(match.group(0), match.group(0)), + text, + ) + + +def _normalized_person_lookup_token(value): + return re.sub(r"[^a-z0-9]+", "", str(value or "").strip().lower()) + + +def _resolve_person_from_xmpp_localpart(*, user, localpart_value): + raw_value = _unescape_xmpp_localpart(localpart_value).strip() + if not raw_value: + return None + + # First try a straightforward case-insensitive name match. + person = Person.objects.filter(user=user, name__iexact=raw_value).first() + if person is not None: + return person + + # Fall back to normalized matching so `test-account`, `test_account`, + # and `test account` can resolve the same contact. + normalized_target = _normalized_person_lookup_token(raw_value) + if not normalized_target: + return None + for candidate in Person.objects.filter(user=user).only("id", "name"): + if _normalized_person_lookup_token(candidate.name) == normalized_target: + return candidate + return None + + +def _signal_identifier_rank(identifier_value): + identifier_text = str(identifier_value or "").strip() + if not identifier_text: + return 99 + if identifier_text.startswith("group."): + return 0 + if identifier_text.startswith("+"): + return 1 + if SIGNAL_UUID_PATTERN.fullmatch(identifier_text): + return 2 + return 3 + + +def _select_person_identifier(*, user, person, service=None): + rows = list( + PersonIdentifier.objects.filter( + user=user, + person=person, + **({"service": service} if service else {}), + ).order_by("id") + ) + if not rows: + return None + if service == "signal" and len(rows) > 1: + rows.sort(key=lambda row: (_signal_identifier_rank(row.identifier), row.id)) + chosen = rows[0] + if len(rows) > 1: + logging.getLogger(__name__).warning( + "Resolved multiple identifiers for user=%s person=%s service=%s; choosing %s from %s rows", + getattr(user, "id", None), + getattr(person, "id", None), + service or getattr(chosen, "service", ""), + getattr(chosen, "identifier", ""), + len(rows), + ) + return chosen + + def _filename_from_url(url_value): path = urlsplit(str(url_value or "")).path name = path.rsplit("/", 1)[-1] @@ -109,6 +211,34 @@ def _extract_xml_attachment_urls(message_stanza): return urls +def _attachment_rows_from_body_urls(body_text): + rows = [] + seen = set() + for raw_line in str(body_text or "").splitlines(): + candidate = _clean_url(raw_line) + if not candidate: + continue + try: + safe_url = validate_attachment_url(candidate) + filename, content_type = validate_attachment_metadata( + filename=_filename_from_url(safe_url), + content_type=_content_type_from_filename_or_url(safe_url), + ) + except Exception: + continue + if safe_url in seen: + continue + seen.add(safe_url) + rows.append( + { + "url": safe_url, + "filename": filename, + "content_type": content_type, + } + ) + return rows + + def _extract_xmpp_reaction(message_stanza): nodes = message_stanza.xml.findall(".//{urn:xmpp:reactions:0}reactions") if not nodes: @@ -367,6 +497,7 @@ class XMPPComponent(ComponentXMPP): self.add_event_handler( "roster_subscription_request", self.on_roster_subscription_request ) + self.add_event_handler("privileges_advertised", self.on_privileges_advertised) # Chat state handlers self.add_event_handler("chatstate_active", self.on_chatstate_active) @@ -438,6 +569,8 @@ class XMPPComponent(ComponentXMPP): await reply.send() async def on_iq_get(self, iq): + if iq["type"] != "get": + return try: iq_to = str(iq["to"] or "").split("/", 1)[0].strip().lower() except Exception: @@ -466,6 +599,8 @@ class XMPPComponent(ComponentXMPP): await self._reply_pubsub_items(iq, node_name) async def on_iq_set(self, iq): + if iq["type"] != "set": + return try: iq_to = str(iq["to"] or "").split("/", 1)[0].strip().lower() except Exception: @@ -521,6 +656,79 @@ class XMPPComponent(ComponentXMPP): def _user_jid(self, username): return f"{username}@{self._user_xmpp_domain()}" + def _contact_component_jid(self, person_name: str, service: str) -> str: + escaped_name = _escape_xmpp_localpart(str(person_name or "").strip().lower()) + return f"{escaped_name}|{str(service or '').strip().lower()}@{self.boundjid.bare}" + + def _build_privileged_outbound_message( + self, *, user_jid, contact_jid, body_text, attachment_url=None + ): + msg = self.make_message(mto=contact_jid, mfrom=user_jid, mtype="chat") + if not msg.get("id"): + msg["id"] = uuid.uuid4().hex + msg["body"] = str(body_text or "") + if attachment_url: + oob_element = ET.Element("{jabber:x:oob}x") + url_element = ET.SubElement(oob_element, "{jabber:x:oob}url") + url_element.text = str(attachment_url) + msg.xml.append(oob_element) + return msg + + def _can_send_privileged_messages(self, user_jid: str) -> bool: + domain = str(user_jid.split("@", 1)[1] if "@" in user_jid else "").strip() + if not domain: + return False + configured_domain = self._user_xmpp_domain() + if configured_domain and domain == configured_domain: + return True + plugin = self.plugin.get("xep_0356", None) + if plugin is None: + return False + perms = plugin.granted_privileges.get(domain) + if perms is None: + return False + return perms.message == MessagePermission.OUTGOING + + async def send_sent_carbon_copy( + self, *, user_jid, contact_jid, body_text, attachment_url=None + ) -> bool: + if not self._can_send_privileged_messages(user_jid): + self.log.debug( + "Skipping sent carbon copy for %s: outgoing message privilege unavailable", + user_jid, + ) + return False + plugin = self.plugin.get("xep_0356", None) + if plugin is None: + return False + try: + msg = self._build_privileged_outbound_message( + user_jid=user_jid, + contact_jid=contact_jid, + body_text=body_text, + attachment_url=attachment_url, + ) + domain = str(user_jid.split("@", 1)[1] if "@" in user_jid else "").strip() + perms = plugin.granted_privileges.get(domain) if domain else None + if perms is not None and perms.message == MessagePermission.OUTGOING: + plugin.send_privileged_message(msg) + else: + plugin._make_privileged_message(msg).send() + self.log.debug( + "Sent privileged carbon copy user=%s contact=%s", + user_jid, + contact_jid, + ) + return True + except Exception as exc: + self.log.warning( + "Failed to send privileged carbon copy user=%s contact=%s: %s", + user_jid, + contact_jid, + exc, + ) + return False + async def enable_carbons(self): """Enable XMPP Message Carbons (XEP-0280)""" try: @@ -549,10 +757,11 @@ class XMPPComponent(ComponentXMPP): recipient_username = recipient_jid # Parse recipient_name and recipient_service (e.g., "mark|signal") if "|" in recipient_username: - person_name, service = recipient_username.split("|") - person_name = person_name.title() # Capitalize for consistency + person_name, service = recipient_username.split("|", 1) + person_name = person_name.strip() + service = service.strip().lower() else: - person_name = recipient_username.title() + person_name = recipient_username.strip() service = None try: @@ -560,15 +769,31 @@ class XMPPComponent(ComponentXMPP): self.log.debug("Resolving XMPP sender user=%s", sender_username) user = User.objects.get(username=sender_username) - # Find Person object with name=person_name.lower() - self.log.debug("Resolving XMPP recipient person=%s", person_name.title()) - person = Person.objects.get(user=user, name=person_name.title()) + self.log.debug("Resolving XMPP recipient person=%s", person_name) + person = _resolve_person_from_xmpp_localpart( + user=user, localpart_value=person_name + ) + if person is None: + raise Person.DoesNotExist(f"No person found for '{person_name}'") # Ensure a PersonIdentifier exists for this user, person, and service self.log.debug("Resolving XMPP identifier service=%s", service) - identifier = PersonIdentifier.objects.get( - user=user, person=person, service=service - ) + if service: + identifier = _select_person_identifier( + user=user, + person=person, + service=service, + ) + if identifier is None: + raise PersonIdentifier.DoesNotExist( + f"No identifier found for person '{person_name}'" + ) + else: + identifier = _select_person_identifier(user=user, person=person) + if identifier is None: + raise PersonIdentifier.DoesNotExist( + f"No identifier found for person '{person_name}'" + ) return identifier except Exception as e: self.log.error(f"Failed to resolve identifier from XMPP message: {e}") @@ -871,419 +1096,71 @@ class XMPPComponent(ComponentXMPP): await sync_to_async(_save_row)() - _approval_event_prefix = "codex_approval" - - _APPROVAL_PROVIDER_COMMANDS = { - ".claude": "claude", - ".codex": "codex_cli", - } - - def _resolve_request_provider(self, request): - event = getattr(request, "external_sync_event", None) - if event is None: - return "" - return str(getattr(event, "provider", "") or "").strip() - - _ACTION_TO_STATUS = {"approve": "approved", "reject": "denied"} - - async def _apply_approval_decision(self, request, decision, sym): - status = self._ACTION_TO_STATUS.get(decision, decision) - request.status = status - await sync_to_async(request.save)(update_fields=["status"]) - run = None - if request.codex_run_id: - run = await sync_to_async(CodexRun.objects.get)(pk=request.codex_run_id) - run.status = "approved_waiting_resume" if status == "approved" else status - await sync_to_async(run.save)(update_fields=["status"]) - if request.external_sync_event_id: - evt = await sync_to_async(ExternalSyncEvent.objects.get)( - pk=request.external_sync_event_id - ) - evt.status = "ok" - await sync_to_async(evt.save)(update_fields=["status"]) - user = await sync_to_async(User.objects.get)(pk=request.user_id) - task = None - if run is not None and run.task_id: - task = await sync_to_async(DerivedTask.objects.get)(pk=run.task_id) - ikey = f"{self._approval_event_prefix}:{request.approval_key}:{status}" - await sync_to_async(ExternalSyncEvent.objects.get_or_create)( - idempotency_key=ikey, - defaults={ - "user": user, - "task": task, - "provider": "codex_cli", - "status": "pending", - "payload": {}, - "error": "", - }, - ) - - async def _approval_list_pending(self, user, scope, sym): - requests = await sync_to_async(list)( - CodexPermissionRequest.objects.filter(user=user, status="pending").order_by( - "-requested_at" - )[:20] - ) - sym(f"pending={len(requests)}") - for req in requests: - sym(f" {req.approval_key}: {req.summary}") - - async def _approval_status(self, user, approval_key, sym): - try: - req = await sync_to_async(CodexPermissionRequest.objects.get)( - user=user, approval_key=approval_key - ) - sym(f"status={req.status} key={req.approval_key}") - except CodexPermissionRequest.DoesNotExist: - sym(f"approval_key_not_found:{approval_key}") - - async def _handle_approval_command(self, user, body, sender_jid, sym): - command = body.strip() - for prefix, expected_provider in self._APPROVAL_PROVIDER_COMMANDS.items(): - if command.startswith(prefix + " ") or command == prefix: - sub = command[len(prefix) :].strip() - parts = sub.split() - if len(parts) >= 2 and parts[0] in ("approve", "reject"): - action, approval_key = parts[0], parts[1] - try: - req = await sync_to_async( - CodexPermissionRequest.objects.select_related( - "external_sync_event" - ).get - )(user=user, approval_key=approval_key) - except CodexPermissionRequest.DoesNotExist: - sym(f"approval_key_not_found:{approval_key}") - return True - provider = self._resolve_request_provider(req) - if not provider.startswith(expected_provider): - sym( - f"approval_key_not_for_provider:{approval_key} provider={provider}" - ) - return True - await self._apply_approval_decision(req, action, sym) - sym(f"{action}d: {approval_key}") - return True - sym(f"usage: {prefix} approve|reject <key>") - return True - - if not command.startswith(".approval"): - return False - - rest = command[len(".approval") :].strip() - - if rest.split() and rest.split()[0] in ("approve", "reject"): - parts = rest.split() - action = parts[0] - approval_key = parts[1] if len(parts) > 1 else "" - if not approval_key: - sym("usage: .approval approve|reject <key>") - return True - try: - req = await sync_to_async( - CodexPermissionRequest.objects.select_related( - "external_sync_event" - ).get - )(user=user, approval_key=approval_key) - except CodexPermissionRequest.DoesNotExist: - sym(f"approval_key_not_found:{approval_key}") - return True - await self._apply_approval_decision(req, action, sym) - sym(f"{action}d: {approval_key}") - return True - - if rest.startswith("list-pending"): - scope = rest[len("list-pending") :].strip() or "mine" - await self._approval_list_pending(user, scope, sym) - return True - - if rest.startswith("status "): - approval_key = rest[len("status ") :].strip() - await self._approval_status(user, approval_key, sym) - return True - - sym( - "approval: .approval approve|reject <key> | " - ".approval list-pending [all] | " - ".approval status <key>" - ) - return True - - async def _handle_tasks_command(self, user, body, sym): - command = body.strip() - if not command.startswith(".tasks"): - return False - rest = command[len(".tasks") :].strip() - - if rest.startswith("list"): - parts = rest.split() - status_filter = parts[1] if len(parts) > 1 else "open" - limit = int(parts[2]) if len(parts) > 2 and parts[2].isdigit() else 10 - tasks = await sync_to_async(list)( - DerivedTask.objects.filter( - user=user, status_snapshot=status_filter - ).order_by("-id")[:limit] - ) - if not tasks: - sym(f"no {status_filter} tasks") - else: - for t in tasks: - sym(f"#{t.reference_code} [{t.status_snapshot}] {t.title}") - return True - - if rest.startswith("show "): - ref = rest[len("show ") :].strip().lstrip("#") - try: - task = await sync_to_async(DerivedTask.objects.get)( - user=user, reference_code=ref - ) - sym(f"#{task.reference_code} {task.title}") - sym(f"status: {task.status_snapshot}") - except DerivedTask.DoesNotExist: - sym(f"task_not_found:#{ref}") - return True - - if rest.startswith("complete "): - ref = rest[len("complete ") :].strip().lstrip("#") - try: - task = await sync_to_async(DerivedTask.objects.get)( - user=user, reference_code=ref - ) - task.status_snapshot = "completed" - await sync_to_async(task.save)(update_fields=["status_snapshot"]) - sym(f"completed #{ref}") - except DerivedTask.DoesNotExist: - sym(f"task_not_found:#{ref}") - return True - - if rest.startswith("undo "): - ref = rest[len("undo ") :].strip().lstrip("#") - try: - task = await sync_to_async(DerivedTask.objects.get)( - user=user, reference_code=ref - ) - await sync_to_async(task.delete)() - sym(f"removed #{ref}") - except DerivedTask.DoesNotExist: - sym(f"task_not_found:#{ref}") - return True - - sym( - "tasks: .tasks list [status] [limit] | " - ".tasks show #<ref> | " - ".tasks complete #<ref> | " - ".tasks undo #<ref>" - ) - return True - - def _extract_totp_secret_candidate(self, command_text: str) -> str: - text = str(command_text or "").strip() - if not text: - return "" - lowered = text.lower() - if lowered.startswith("otpauth://"): - parsed = urlparse(text) - query = parse_qs(parsed.query or "") - return str((query.get("secret") or [""])[0] or "").strip() - if lowered.startswith(".totp"): - rest = text[len(".totp") :].strip() - if not rest: - return "" - parts = rest.split(maxsplit=1) - action = str(parts[0] or "").strip().lower() - if action in {"enroll", "set"} and len(parts) > 1: - return str(parts[1] or "").strip() - if action in {"status", "help"}: - return "" - return rest - return "" - - async def _handle_totp_command(self, user, body, sym): - command = str(body or "").strip() - lowered = command.lower() - if lowered.startswith(".totp status"): - exists = await sync_to_async( - lambda: __import__( - "django_otp.plugins.otp_totp.models", - fromlist=["TOTPDevice"], - ) - .TOTPDevice.objects.filter(user=user, confirmed=True) - .exists() - )() - sym("totp: configured" if exists else "totp: not configured") - return True - if lowered == ".totp help": - sym("totp: .totp enroll <base32-secret|otpauth-uri> | .totp status") - return True - - secret_candidate = self._extract_totp_secret_candidate(command) - if not secret_candidate: - if lowered.startswith(".totp"): - sym("usage: .totp enroll <base32-secret|otpauth-uri>") - return True - return False - - normalized = str(secret_candidate).replace(" ", "").strip().upper() - try: - key_bytes = base64.b32decode(normalized, casefold=True) - except Exception: - sym("totp: invalid secret format") - return True - if len(key_bytes) < 10: - sym("totp: secret too short") - return True - - def _save_device(): - from django_otp.plugins.otp_totp.models import TOTPDevice - - device = TOTPDevice.objects.filter(user=user).order_by("-id").first() - if device is None: - device = TOTPDevice(user=user, name="gateway") - device.key = key_bytes.hex() - device.confirmed = True - device.step = 30 - device.t0 = 0 - device.digits = 6 - device.tolerance = 1 - device.drift = 0 - device.save() - return device.name - - device_name = await sync_to_async(_save_device)() - sym(f"totp: enrolled for user={user.username} device={device_name}") - return True - async def _route_gateway_command( self, *, sender_user, body, + service, + channel_identifier, + sender_identifier, sender_jid, recipient_jid, local_message, message_meta, sym, ): - command_text = str(body or "").strip() - - async def _contacts_handler(_ctx, emit): - persons = await sync_to_async(list)( - Person.objects.filter(user=sender_user).order_by("name") - ) - if not persons: - emit("No contacts found.") - return True - emit("Contacts: " + ", ".join([p.name for p in persons])) - return True - - async def _help_handler(_ctx, emit): - for line in self._gateway_help_lines(): - emit(line) - return True - - async def _whoami_handler(_ctx, emit): - emit(str(sender_user.__dict__)) - return True - - async def _approval_handler(_ctx, emit): - return await self._handle_approval_command( - sender_user, command_text, sender_jid, emit - ) - - async def _tasks_handler(_ctx, emit): - return await self._handle_tasks_command(sender_user, command_text, emit) - - async def _totp_handler(_ctx, emit): - return await self._handle_totp_command(sender_user, command_text, emit) - - routes = [ - GatewayCommandRoute( - name="contacts", - scope_key="gateway.contacts", - matcher=lambda text: str(text or "").strip().lower() == ".contacts", - handler=_contacts_handler, - ), - GatewayCommandRoute( - name="help", - scope_key="gateway.help", - matcher=lambda text: str(text or "").strip().lower() == ".help", - handler=_help_handler, - ), - GatewayCommandRoute( - name="whoami", - scope_key="gateway.whoami", - matcher=lambda text: str(text or "").strip().lower() == ".whoami", - handler=_whoami_handler, - ), - GatewayCommandRoute( - name="approval", - scope_key="gateway.approval", - matcher=lambda text: str(text or "") - .strip() - .lower() - .startswith(".approval") - or any( - str(text or "").strip().lower().startswith(prefix + " ") - or str(text or "").strip().lower() == prefix - for prefix in self._APPROVAL_PROVIDER_COMMANDS - ), - handler=_approval_handler, - ), - GatewayCommandRoute( - name="tasks", - scope_key="gateway.tasks", - matcher=lambda text: str(text or "") - .strip() - .lower() - .startswith(".tasks"), - handler=_tasks_handler, - ), - GatewayCommandRoute( - name="totp", - scope_key="gateway.totp", - matcher=lambda text: bool(self._extract_totp_secret_candidate(text)), - handler=_totp_handler, - ), - ] - handled = await dispatch_gateway_command( - context=GatewayCommandContext( - user=sender_user, - source_message=local_message, - service="xmpp", - channel_identifier=str(sender_jid or ""), - sender_identifier=str(sender_jid or ""), - message_text=command_text, - message_meta=dict(message_meta or {}), - payload={ - "sender_jid": str(sender_jid or ""), - "recipient_jid": str(recipient_jid or ""), - }, - ), - routes=routes, + return await dispatch_builtin_gateway_command( + user=sender_user, + command_text=str(body or "").strip(), + service=str(service or "xmpp"), + channel_identifier=str(channel_identifier or sender_jid or ""), + sender_identifier=str(sender_identifier or sender_jid or ""), + source_message=local_message, + message_meta=dict(message_meta or {}), + payload={ + "sender_jid": str(sender_jid or ""), + "recipient_jid": str(recipient_jid or ""), + }, emit=sym, ) - if not handled and command_text.startswith("."): - sym("No such command") - return handled + + async def execute_gateway_command( + self, + *, + sender_user, + body, + service, + channel_identifier, + sender_identifier, + local_message=None, + message_meta=None, + sender_jid="", + recipient_jid="", + ) -> list[str]: + captured_replies: list[str] = [] + + def _capture(value): + text_value = str(value or "").strip() + if text_value: + captured_replies.append(text_value) + + await self._route_gateway_command( + sender_user=sender_user, + body=body, + service=service, + channel_identifier=channel_identifier, + sender_identifier=sender_identifier, + sender_jid=sender_jid, + recipient_jid=recipient_jid, + local_message=local_message, + message_meta=dict(message_meta or {}), + sym=_capture, + ) + return captured_replies def _gateway_help_lines(self): - return [ - "Gateway commands:", - " .contacts — list contacts", - " .whoami — show current user", - " .help — show this help", - " .totp enroll <secret|otpauth-uri> — enroll TOTP for this user", - " .totp status — show whether TOTP is configured", - "Approval commands:", - " .approval list-pending [all] — list pending approval requests", - " .approval approve <key> — approve a request", - " .approval reject <key> — reject a request", - " .approval status <key> — check request status", - "Task commands:", - " .tasks list [status] [limit] — list tasks", - " .tasks show #<ref> — show task details", - " .tasks complete #<ref> — mark task complete", - " .tasks undo #<ref> — remove task", - ] + return gateway_help_lines() async def _handle_mitigation_command(self, sender_user, body, sym): def parse_parts(raw): @@ -1744,8 +1621,8 @@ class XMPPComponent(ComponentXMPP): """ Handle incoming presence subscription requests. Accept subscriptions to: - 1. The gateway component itself (jews.zm.is) - for OMEMO device discovery - 2. User contacts at the gateway (user|service@jews.zm.is) - for user-to-user messaging + 1. The gateway component JID itself - for OMEMO device discovery + 2. User contacts at the gateway component JID - for user-to-user messaging """ sender_jid = str(pres["from"]).split("/")[0] # Bare JID (user@domain) @@ -1780,25 +1657,48 @@ class XMPPComponent(ComponentXMPP): # Parse recipient_name and recipient_service (e.g., "mark|signal") if "|" in recipient_username: - person_name, service = recipient_username.split("|") - person_name = person_name.title() # Capitalize for consistency + person_name, service = recipient_username.split("|", 1) + person_name = person_name.strip() + service = service.strip().lower() else: - person_name = recipient_username.title() + person_name = recipient_username.strip() service = None # Lookup user in Django self.log.debug("Resolving subscription user=%s", user_username) user = User.objects.get(username=user_username) - # Find Person object with name=person_name.lower() - self.log.debug("Resolving subscription person=%s", person_name.title()) - person = Person.objects.get(user=user, name=person_name.title()) + self.log.debug("Resolving subscription person=%s", person_name) + person = _resolve_person_from_xmpp_localpart( + user=user, localpart_value=person_name + ) + if person is None: + raise Person.DoesNotExist(f"No person found for '{person_name}'") # Ensure a PersonIdentifier exists for this user, person, and service self.log.debug("Resolving subscription identifier service=%s", service) - PersonIdentifier.objects.get(user=user, person=person, service=service) + if service: + identifier = _select_person_identifier( + user=user, + person=person, + service=service, + ) + if identifier is None: + raise PersonIdentifier.DoesNotExist( + f"No identifier found for person '{person_name}'" + ) + else: + fallback_identifier = _select_person_identifier( + user=user, + person=person, + ) + if fallback_identifier is None: + raise PersonIdentifier.DoesNotExist( + f"No identifier found for person '{person_name}'" + ) + service = str(fallback_identifier.service or "").strip().lower() - contact_jid = f"{person_name.lower()}|{service}@{self.boundjid.bare}" + contact_jid = self._contact_component_jid(person.name, service) # Accept the subscription self.send_presence(ptype="subscribed", pto=sender_jid, pfrom=contact_jid) @@ -1806,8 +1706,14 @@ class XMPPComponent(ComponentXMPP): f"Accepted subscription from {sender_jid}, sent from {contact_jid}" ) - # Send a presence request **from the recipient to the sender** (ASKS THEM TO ACCEPT BACK) - # self.send_presence(ptype="subscribe", pto=sender_jid, pfrom=contact_jid) + # Ask the XMPP client to grant reciprocal presence so roster-driven + # clients do not leave bridged contacts stuck unsubscribed/offline. + self.send_presence(ptype="subscribe", pto=sender_jid, pfrom=contact_jid) + self.log.debug( + "Requested reciprocal subscription from %s to %s", + contact_jid, + sender_jid, + ) # Add sender to roster # self.update_roster(sender_jid, name=sender_jid.split("@")[0]) @@ -1865,6 +1771,15 @@ class XMPPComponent(ComponentXMPP): self.log.debug("Skipping carbons enable for component session") await self._bootstrap_omemo_for_authentic_channel() + def on_privileges_advertised(self, *_args): + plugin = self.plugin.get("xep_0356", None) + if plugin is None: + return + self.log.info( + "XMPP privileged capabilities advertised: %s", + plugin.granted_privileges, + ) + async def _reconnect_loop(self): try: while True: @@ -1932,6 +1847,10 @@ class XMPPComponent(ComponentXMPP): getattr(settings, "XMPP_UPLOAD_SERVICE", "") or getattr(settings, "XMPP_UPLOAD_JID", "") ).strip() + candidate_services = [] + if upload_service_jid: + candidate_services.append(upload_service_jid) + if not upload_service_jid: discovered = None try: @@ -1955,60 +1874,80 @@ class XMPPComponent(ComponentXMPP): discovered_jid = "" else: discovered_jid = raw_discovered - upload_service_jid = discovered_jid if upload_service_jid: + candidate_services.append(upload_service_jid) self.log.info( "Discovered XMPP upload service via XEP-0363: %s", upload_service_jid, ) - else: - if not self._upload_config_warned: - self.log.warning( - "XMPP upload service not configured/discoverable; skipping attachment upload. " - "Set XMPP_UPLOAD_SERVICE (or XMPP_UPLOAD_JID)." - ) - self._upload_config_warned = True - return None - try: - slot = await self["xep_0363"].request_slot( - jid=upload_service_jid, - filename=filename, - content_type=content_type, - size=size, - ) + default_service = str(getattr(settings, "XMPP_USER_DOMAIN", "") or "").strip() + if default_service: + candidate_services.append(default_service) - if slot is None: - self.log.error(f"Failed to obtain upload slot for {filename}") - return None + deduped_services = [] + for candidate in candidate_services: + cleaned = str(candidate or "").strip() + if cleaned and cleaned not in deduped_services: + deduped_services.append(cleaned) - # Parse the XML response - root = ET.fromstring(str(slot)) # Convert to string if necessary - namespace = "{urn:xmpp:http:upload:0}" # Define the namespace - - get_url = root.find(f".//{namespace}get").attrib.get("url") - put_element = root.find(f".//{namespace}put") - put_url = put_element.attrib.get("url") - - # Extract the Authorization header correctly - header_element = put_element.find( - f"./{namespace}header[@name='Authorization']" - ) - auth_header = ( - header_element.text.strip() if header_element is not None else None - ) - - if not get_url or not put_url: - self.log.error(f"Missing URLs in upload slot: {slot}") - return None - - return get_url, put_url, auth_header - - except Exception as e: - self.log.error(f"Exception while requesting upload slot: {e}") + if not deduped_services: + if not self._upload_config_warned: + self.log.warning( + "XMPP upload service not configured/discoverable; skipping attachment upload. " + "Set XMPP_UPLOAD_SERVICE (or XMPP_UPLOAD_JID)." + ) + self._upload_config_warned = True return None + last_error = None + for service_jid in deduped_services: + try: + slot = await self["xep_0363"].request_slot( + jid=service_jid, + filename=filename, + content_type=content_type, + size=size, + ) + + if slot is None: + last_error = f"empty slot response from {service_jid}" + continue + + root = ET.fromstring(str(slot)) + namespace = "{urn:xmpp:http:upload:0}" + + get_url = root.find(f".//{namespace}get").attrib.get("url") + put_element = root.find(f".//{namespace}put") + put_url = put_element.attrib.get("url") + + header_element = put_element.find( + f"./{namespace}header[@name='Authorization']" + ) + auth_header = ( + header_element.text.strip() if header_element is not None else None + ) + + if not get_url or not put_url: + last_error = f"missing URLs in upload slot from {service_jid}" + continue + + if service_jid != deduped_services[0]: + self.log.info( + "XMPP upload service fallback succeeded via %s", + service_jid, + ) + return get_url, put_url, auth_header + except Exception as exc: + last_error = str(exc) + self.log.warning( + "XMPP upload slot request failed via %s: %s", service_jid, exc + ) + + self.log.error("Failed to obtain upload slot for %s: %s", filename, last_error) + return None + async def message(self, msg): """ Process incoming XMPP messages. @@ -2044,9 +1983,12 @@ class XMPPComponent(ComponentXMPP): original_msg = msg omemo_plugin = self._get_omemo_plugin() sender_omemo_fingerprint = "" + omemo_decrypt_error = "" + was_omemo_encrypted = False if omemo_plugin: try: if omemo_plugin.is_encrypted(msg): + was_omemo_encrypted = True decrypted, sender_device = await omemo_plugin.decrypt_message(msg) msg = decrypted sender_omemo_fingerprint = _format_omemo_identity_fingerprint( @@ -2054,6 +1996,7 @@ class XMPPComponent(ComponentXMPP): ) self.log.debug("OMEMO: decrypted message from %s", sender_jid) except Exception as exc: + omemo_decrypt_error = str(exc or "").strip() self.log.warning( "OMEMO: decryption failed from %s: %s", sender_jid, exc ) @@ -2141,6 +2084,12 @@ class XMPPComponent(ComponentXMPP): } ) + # Some XMPP clients send HTTP upload links only in the body text. + if not attachments: + body_url_attachments = _attachment_rows_from_body_urls(body) + if body_url_attachments: + attachments.extend(body_url_attachments) + if ( not body or body.strip().lower() in {"[no body]", "(no text)"} ) and attachments: @@ -2162,6 +2111,7 @@ class XMPPComponent(ComponentXMPP): joined_urls = "\n".join(attachment_urls_for_body).strip() if str(relay_body or "").strip() == joined_urls: relay_body = "" + body = "" self.log.debug("Extracted %s attachments from XMPP message", len(attachments)) # Log extracted information with variable name annotations @@ -2174,7 +2124,7 @@ class XMPPComponent(ComponentXMPP): self.log.debug(log_message) # Ensure recipient domain matches our configured component - expected_domain = settings.XMPP_JID # 'jews.zm.is' in your config + expected_domain = settings.XMPP_JID if recipient_domain != expected_domain: self.log.warning( f"Invalid recipient domain: {recipient_domain}, expected {expected_domain}" @@ -2202,6 +2152,7 @@ class XMPPComponent(ComponentXMPP): omemo_observation = _extract_sender_omemo_client_key(original_msg) # Enforce mandatory encryption policy. + sec_settings = None try: from core.models import UserXmppSecuritySettings @@ -2222,12 +2173,40 @@ class XMPPComponent(ComponentXMPP): except Exception as exc: self.log.warning("OMEMO policy check failed: %s", exc) + component_encrypt_with_omemo = True + if sec_settings is not None: + component_encrypt_with_omemo = bool( + getattr( + sec_settings, + "encrypt_component_messages_with_omemo", + True, + ) + ) + + if ( + was_omemo_encrypted + and omemo_decrypt_error + and not body.strip() + and not attachments + and not parsed_reaction + ): + await self.send_xmpp_message( + sender_jid, + settings.XMPP_JID, + "This gateway could not decrypt your OMEMO-encrypted message or attachment, so it was not delivered. Disable OMEMO for this contact or send it unencrypted.", + use_omemo_encryption=component_encrypt_with_omemo, + ) + return + if recipient_jid == settings.XMPP_JID: self.log.debug("Handling command message sent to gateway JID") if body.startswith("."): - await self._route_gateway_command( + gateway_replies = await self.execute_gateway_command( sender_user=sender_user, body=body, + service="xmpp", + channel_identifier=str(sender_jid or ""), + sender_identifier=str(sender_jid or ""), sender_jid=sender_jid, recipient_jid=recipient_jid, local_message=None, @@ -2241,38 +2220,47 @@ class XMPPComponent(ComponentXMPP): ), } }, - sym=sym, ) + for line in gateway_replies: + await self.send_xmpp_message( + sender_jid, + settings.XMPP_JID, + f"[>] {line}", + use_omemo_encryption=component_encrypt_with_omemo, + ) else: self.log.debug("Handling routed message to contact") if "|" in recipient_username: - recipient_name, recipient_service = recipient_username.split("|") - - recipient_name = recipient_name.title() + recipient_name, recipient_service = recipient_username.split("|", 1) + recipient_name = recipient_name.strip() + recipient_service = recipient_service.strip().lower() else: - recipient_name = recipient_username + recipient_name = recipient_username.strip() recipient_service = None - recipient_name = recipient_name.title() - - try: - person = Person.objects.get(user=sender_user, name=recipient_name) - except Person.DoesNotExist: + person = _resolve_person_from_xmpp_localpart( + user=sender_user, localpart_value=recipient_name + ) + if person is None: sym("This person does not exist.") + return if recipient_service: - try: - identifier = PersonIdentifier.objects.get( - user=sender_user, person=person, service=recipient_service - ) - except PersonIdentifier.DoesNotExist: + identifier = PersonIdentifier.objects.filter( + user=sender_user, person=person, service=recipient_service + ).first() + if identifier is None: sym("This service identifier does not exist.") + return else: # Get a random identifier identifier = PersonIdentifier.objects.filter( user=sender_user, person=person ).first() + if identifier is None: + sym("This service identifier does not exist.") + return recipient_service = identifier.service # sym(str(person.__dict__)) @@ -2381,7 +2369,6 @@ class XMPPComponent(ComponentXMPP): return # tss = await identifier.send(body, attachments=attachments) - # AM FIXING https://git.zm.is/XF/GIA/issues/5 session, _ = await sync_to_async(ChatSession.objects.get_or_create)( identifier=identifier, user=identifier.user, @@ -2421,6 +2408,17 @@ class XMPPComponent(ComponentXMPP): "omemo_client_key": str( omemo_observation.get("client_key") or "" ), + "attachments": [ + { + "url": str(item.get("url") or ""), + "filename": str(item.get("filename") or ""), + "content_type": str( + item.get("content_type") + or "application/octet-stream" + ), + } + for item in attachments + ], } }, ) @@ -2460,6 +2458,12 @@ class XMPPComponent(ComponentXMPP): "legacy_message_id": str(local_message.id), }, ) + self.log.info( + "Relayed XMPP message to %s attachment_count=%s text_len=%s", + recipient_service, + len(attachments), + len(str(relay_body or "")), + ) self.log.debug("Message sent unaltered") return @@ -2529,7 +2533,12 @@ class XMPPComponent(ComponentXMPP): except Exception as exc: self.log.warning("xmpp blocked outbound attachment: %s", exc) return None - headers = {"Content-Type": content_type} + headers = { + "Content-Type": content_type, + "Content-Length": str( + int(att.get("size") or len(att.get("content") or b"")) + ), + } if auth_header: headers["Authorization"] = auth_header @@ -2704,9 +2713,8 @@ class XMPPComponent(ComponentXMPP): ) return False - sender_jid = ( - f"{person_identifier.person.name.lower()}|" - f"{person_identifier.service}@{settings.XMPP_JID}" + sender_jid = self._contact_component_jid( + person_identifier.person.name, person_identifier.service ) recipient_jid = self._user_jid(user.username) await self.send_xmpp_reaction( @@ -2753,9 +2761,8 @@ class XMPPComponent(ComponentXMPP): msg.send() async def send_typing_for_person(self, user, person_identifier, started): - sender_jid = ( - f"{person_identifier.person.name.lower()}|" - f"{person_identifier.service}@{settings.XMPP_JID}" + sender_jid = self._contact_component_jid( + person_identifier.person.name, person_identifier.service ) recipient_jid = self._user_jid(user.username) await self.send_chat_state(recipient_jid, sender_jid, started) @@ -2771,7 +2778,9 @@ class XMPPComponent(ComponentXMPP): ): """Handles sending XMPP messages with text and attachments.""" - sender_jid = f"{person_identifier.person.name.lower()}|{person_identifier.service}@{settings.XMPP_JID}" + sender_jid = self._contact_component_jid( + person_identifier.person.name, person_identifier.service + ) recipient_jid = self._user_jid(person_identifier.user.username) relay_encrypt_with_omemo = True try: @@ -2791,7 +2800,21 @@ class XMPPComponent(ComponentXMPP): recipient_jid, sender_jid, f"YOU: {text}", + use_omemo_encryption=relay_encrypt_with_omemo, ) + try: + await self.send_sent_carbon_copy( + user_jid=recipient_jid, + contact_jid=sender_jid, + body_text=text, + ) + except Exception as exc: + self.log.warning( + "Sent carbon copy failed after fallback relay user=%s contact=%s: %s", + recipient_jid, + sender_jid, + exc, + ) transport.record_bridge_mapping( user_id=user.id, person_id=person_identifier.person_id, @@ -2878,6 +2901,21 @@ class XMPPComponent(ComponentXMPP): uploaded_rows = await asyncio.gather(*upload_tasks) # Upload files concurrently normalized_rows = [dict(row or {}) for row in uploaded_rows if row] for row in normalized_rows: + if is_outgoing_message: + try: + await self.send_sent_carbon_copy( + user_jid=recipient_jid, + contact_jid=sender_jid, + body_text=str(row.get("url") or ""), + attachment_url=str(row.get("url") or ""), + ) + except Exception as exc: + self.log.warning( + "Attachment sent carbon copy failed user=%s contact=%s: %s", + recipient_jid, + sender_jid, + exc, + ) transport.record_bridge_mapping( user_id=user.id, person_id=person_identifier.person_id, @@ -2950,6 +2988,8 @@ class XMPPClient(ClientBase): self.client.register_plugin("xep_0004") # Data Forms self.client.register_plugin("xep_0060") # PubSub self.client.register_plugin("xep_0199") # XMPP Ping + self.client.register_plugin("xep_0297") # Forwarded + self.client.register_plugin("xep_0356") # Privileged Entity self.client.register_plugin("xep_0085") # Chat State Notifications self.client.register_plugin("xep_0363") # HTTP File Upload diff --git a/core/context_processors.py b/core/context_processors.py index ce82229..0d43344 100644 --- a/core/context_processors.py +++ b/core/context_processors.py @@ -32,35 +32,93 @@ def settings_hierarchy_nav(request): translation_href = reverse("translation_settings") availability_href = reverse("availability_settings") - general_routes = { - "notifications_settings", - "notifications_update", - "system_settings", - "accessibility_settings", - } - security_routes = { - "security_settings", - "encryption_settings", - "permission_settings", - "security_2fa", - } - ai_routes = { - "ai_settings", - "ai_models", - "ais", - "ai_create", - "ai_update", - "ai_delete", - "ai_execution_log", - } - modules_routes = { - "modules_settings", - "command_routing", - "business_plan_inbox", - "business_plan_editor", - "tasks_settings", - "translation_settings", - "availability_settings", + categories = { + "general": { + "routes": { + "notifications_settings", + "notifications_update", + "system_settings", + "accessibility_settings", + }, + "title": "General", + "tabs": [ + ( + "Notifications", + notifications_href, + lambda: path == notifications_href, + ), + ("System", system_href, lambda: path == system_href), + ( + "Accessibility", + accessibility_href, + lambda: path == accessibility_href, + ), + ], + }, + "security": { + "routes": { + "security_settings", + "encryption_settings", + "permission_settings", + "security_2fa", + }, + "title": "Security", + "tabs": [ + ("Encryption", encryption_href, lambda: path == encryption_href), + ("Permissions", permissions_href, lambda: path == permissions_href), + ( + "2FA", + security_2fa_href, + lambda: path == security_2fa_href or namespace == "two_factor", + ), + ], + }, + "ai": { + "routes": { + "ai_settings", + "ai_models", + "ais", + "ai_create", + "ai_update", + "ai_delete", + "ai_execution_log", + }, + "title": "AI", + "tabs": [ + ("Models", ai_models_href, lambda: path == ai_models_href), + ("Traces", ai_traces_href, lambda: path == ai_traces_href), + ], + }, + "modules": { + "routes": { + "modules_settings", + "command_routing", + "business_plan_inbox", + "business_plan_editor", + "tasks_settings", + "translation_settings", + "translation_preview", + "availability_settings", + "codex_settings", + "codex_approval", + }, + "title": "Modules", + "tabs": [ + ("Commands", commands_href, lambda: path == commands_href), + ( + "Business Plans", + business_plans_href, + lambda: url_name in {"business_plan_inbox", "business_plan_editor"}, + ), + ("Task Automation", tasks_href, lambda: path == tasks_href), + ( + "Translation", + translation_href, + lambda: url_name in {"translation_settings", "translation_preview"}, + ), + ("Availability", availability_href, lambda: path == availability_href), + ], + }, } two_factor_security_routes = { @@ -72,55 +130,29 @@ def settings_hierarchy_nav(request): "phone_delete", } - if url_name in general_routes: - settings_nav = { - "title": "General", - "tabs": [ - _tab("Notifications", notifications_href, path == notifications_href), - _tab("System", system_href, path == system_href), - _tab("Accessibility", accessibility_href, path == accessibility_href), - ], - } - elif url_name in security_routes or ( + if url_name in categories["general"]["routes"]: + category = categories["general"] + elif url_name in categories["security"]["routes"] or ( namespace == "two_factor" and url_name in two_factor_security_routes ): - settings_nav = { - "title": "Security", - "tabs": [ - _tab("Encryption", encryption_href, path == encryption_href), - _tab("Permissions", permissions_href, path == permissions_href), - _tab( - "2FA", - security_2fa_href, - path == security_2fa_href or namespace == "two_factor", - ), - ], - } - elif url_name in ai_routes: - settings_nav = { - "title": "AI", - "tabs": [ - _tab("Models", ai_models_href, path == ai_models_href), - _tab("Traces", ai_traces_href, path == ai_traces_href), - ], - } - elif url_name in modules_routes: - settings_nav = { - "title": "Modules", - "tabs": [ - _tab("Commands", commands_href, path == commands_href), - _tab( - "Business Plans", - business_plans_href, - url_name in {"business_plan_inbox", "business_plan_editor"}, - ), - _tab("Task Automation", tasks_href, path == tasks_href), - _tab("Translation", translation_href, path == translation_href), - _tab("Availability", availability_href, path == availability_href), - ], - } + category = categories["security"] + elif url_name in categories["ai"]["routes"]: + category = categories["ai"] + elif url_name in categories["modules"]["routes"]: + category = categories["modules"] else: + category = None + + if category is None: settings_nav = None + else: + settings_nav = { + "title": str(category.get("title") or "Settings"), + "tabs": [ + _tab(label, href, bool(is_active())) + for label, href, is_active in category.get("tabs", []) + ], + } if not settings_nav: return {} diff --git a/core/gateway/builtin.py b/core/gateway/builtin.py new file mode 100644 index 0000000..c58623b --- /dev/null +++ b/core/gateway/builtin.py @@ -0,0 +1,417 @@ +from __future__ import annotations + +import re + +from asgiref.sync import sync_to_async + +from core.gateway.commands import ( + GatewayCommandContext, + GatewayCommandRoute, + dispatch_gateway_command, +) +from core.models import ( + CodexPermissionRequest, + CodexRun, + DerivedTask, + ExternalSyncEvent, + Person, + TaskProject, + User, +) +from core.tasks.engine import create_task_record_and_sync, mark_task_completed_and_sync + +APPROVAL_PROVIDER_COMMANDS = { + ".claude": "claude", + ".codex": "codex_cli", +} +APPROVAL_EVENT_PREFIX = "codex_approval" +ACTION_TO_STATUS = {"approve": "approved", "reject": "denied"} +TASK_COMMAND_MATCH_RE = re.compile(r"^\s*(?:\.tasks\b|\.l\b|\.list\b)", re.IGNORECASE) + + +def gateway_help_lines() -> list[str]: + return [ + "Gateway commands:", + " .contacts — list contacts", + " .whoami — show current user", + " .help — show this help", + "Approval commands:", + " .approval list-pending [all] — list pending approval requests", + " .approval approve <key> — approve a request", + " .approval reject <key> — reject a request", + " .approval status <key> — check request status", + "Task commands:", + " .l — shortcut for open task list", + " .tasks list [status] [limit] — list tasks", + " .tasks add <project> :: <title> — create task in project", + " .tasks show #<ref> — show task details", + " .tasks complete #<ref> — mark task complete", + " .tasks undo #<ref> — remove task", + ] + + +def _resolve_request_provider(request): + event = getattr(request, "external_sync_event", None) + if event is None: + return "" + return str(getattr(event, "provider", "") or "").strip() + + +async def _apply_approval_decision(request, decision): + status = ACTION_TO_STATUS.get(decision, decision) + request.status = status + await sync_to_async(request.save)(update_fields=["status"]) + run = None + if request.codex_run_id: + run = await sync_to_async(CodexRun.objects.get)(pk=request.codex_run_id) + run.status = "approved_waiting_resume" if status == "approved" else status + await sync_to_async(run.save)(update_fields=["status"]) + if request.external_sync_event_id: + evt = await sync_to_async(ExternalSyncEvent.objects.get)( + pk=request.external_sync_event_id + ) + evt.status = "ok" + await sync_to_async(evt.save)(update_fields=["status"]) + user = await sync_to_async(User.objects.get)(pk=request.user_id) + task = None + if run is not None and run.task_id: + task = await sync_to_async(DerivedTask.objects.get)(pk=run.task_id) + ikey = f"{APPROVAL_EVENT_PREFIX}:{request.approval_key}:{status}" + await sync_to_async(ExternalSyncEvent.objects.get_or_create)( + idempotency_key=ikey, + defaults={ + "user": user, + "task": task, + "provider": "codex_cli", + "status": "pending", + "payload": {}, + "error": "", + }, + ) + + +async def _approval_list_pending(user, scope, emit): + _ = scope + requests = await sync_to_async(list)( + CodexPermissionRequest.objects.filter(user=user, status="pending").order_by( + "-requested_at" + )[:20] + ) + emit(f"pending={len(requests)}") + for req in requests: + emit(f" {req.approval_key}: {req.summary}") + + +async def _approval_status(user, approval_key, emit): + try: + req = await sync_to_async(CodexPermissionRequest.objects.get)( + user=user, approval_key=approval_key + ) + emit(f"status={req.status} key={req.approval_key}") + except CodexPermissionRequest.DoesNotExist: + emit(f"approval_key_not_found:{approval_key}") + + +async def handle_approval_command(user, body, emit): + command = str(body or "").strip() + for prefix, expected_provider in APPROVAL_PROVIDER_COMMANDS.items(): + if command.startswith(prefix + " ") or command == prefix: + sub = command[len(prefix) :].strip() + parts = sub.split() + if len(parts) >= 2 and parts[0] in ("approve", "reject"): + action, approval_key = parts[0], parts[1] + try: + req = await sync_to_async( + CodexPermissionRequest.objects.select_related( + "external_sync_event" + ).get + )(user=user, approval_key=approval_key) + except CodexPermissionRequest.DoesNotExist: + emit(f"approval_key_not_found:{approval_key}") + return True + provider = _resolve_request_provider(req) + if not provider.startswith(expected_provider): + emit( + f"approval_key_not_for_provider:{approval_key} provider={provider}" + ) + return True + await _apply_approval_decision(req, action) + emit(f"{action}d: {approval_key}") + return True + emit(f"usage: {prefix} approve|reject <key>") + return True + + if not command.startswith(".approval"): + return False + + rest = command[len(".approval") :].strip() + + if rest.split() and rest.split()[0] in ("approve", "reject"): + parts = rest.split() + action = parts[0] + approval_key = parts[1] if len(parts) > 1 else "" + if not approval_key: + emit("usage: .approval approve|reject <key>") + return True + try: + req = await sync_to_async( + CodexPermissionRequest.objects.select_related("external_sync_event").get + )(user=user, approval_key=approval_key) + except CodexPermissionRequest.DoesNotExist: + emit(f"approval_key_not_found:{approval_key}") + return True + await _apply_approval_decision(req, action) + emit(f"{action}d: {approval_key}") + return True + + if rest.startswith("list-pending"): + scope = rest[len("list-pending") :].strip() or "mine" + await _approval_list_pending(user, scope, emit) + return True + + if rest.startswith("status "): + approval_key = rest[len("status ") :].strip() + await _approval_status(user, approval_key, emit) + return True + + emit( + "approval: .approval approve|reject <key> | " + ".approval list-pending [all] | " + ".approval status <key>" + ) + return True + + +def _parse_task_create(rest: str) -> tuple[str, str]: + text = str(rest or "").strip() + if not text.lower().startswith("add "): + return "", "" + payload = text[4:].strip() + if "::" in payload: + project_name, title = payload.split("::", 1) + return str(project_name or "").strip(), str(title or "").strip() + return "", "" + + +async def handle_tasks_command( + user, + body, + emit, + *, + service: str = "", + channel_identifier: str = "", + sender_identifier: str = "", +): + command = str(body or "").strip() + lower_command = command.lower() + if not TASK_COMMAND_MATCH_RE.match(command): + return False + if lower_command.startswith(".tasks"): + rest = command[len(".tasks") :].strip() + elif lower_command.startswith(".list") or lower_command.startswith(".l"): + rest = "list" + else: + rest = "list " + command[2:].strip() if len(command) > 2 else "list" + + if rest.startswith("list"): + parts = rest.split() + status_filter = parts[1] if len(parts) > 1 else "open" + limit = int(parts[2]) if len(parts) > 2 and parts[2].isdigit() else 10 + tasks = await sync_to_async(list)( + DerivedTask.objects.filter( + user=user, status_snapshot=status_filter + ).order_by("-id")[:limit] + ) + if not tasks: + emit(f"no {status_filter} tasks") + else: + for task in tasks: + emit(f"#{task.reference_code} [{task.status_snapshot}] {task.title}") + return True + + project_name, title = _parse_task_create(rest) + if project_name or rest.startswith("add "): + if not project_name or not title: + emit("usage: .tasks add <project> :: <title>") + return True + project = await sync_to_async( + lambda: TaskProject.objects.filter(user=user, name__iexact=project_name) + .order_by("name") + .first() + )() + if project is None: + emit(f"project_not_found:{project_name}") + return True + task, _event = await create_task_record_and_sync( + user=user, + project=project, + title=title, + source_service=str(service or "web").strip().lower() or "web", + source_channel=str(channel_identifier or "").strip(), + actor_identifier=str(sender_identifier or "").strip(), + immutable_payload={ + "origin": "gateway.tasks.add", + "channel_service": str(service or "").strip().lower(), + "channel_identifier": str(channel_identifier or "").strip(), + }, + event_payload={ + "command": ".tasks add", + "via": "gateway_builtin", + }, + ) + emit(f"created #{task.reference_code} [{project.name}] {task.title}") + return True + + if rest.startswith("show "): + ref = rest[len("show ") :].strip().lstrip("#") + try: + task = await sync_to_async(DerivedTask.objects.get)( + user=user, reference_code=ref + ) + emit(f"#{task.reference_code} {task.title}") + emit(f"status: {task.status_snapshot}") + except DerivedTask.DoesNotExist: + emit(f"task_not_found:#{ref}") + return True + + if rest.startswith("complete "): + ref = rest[len("complete ") :].strip().lstrip("#") + try: + task = await sync_to_async(DerivedTask.objects.select_related("project").get)( + user=user, reference_code=ref + ) + await mark_task_completed_and_sync( + task=task, + actor_identifier=str(sender_identifier or "").strip(), + payload={ + "marker": ref, + "command": ".tasks complete", + "via": "gateway_builtin", + }, + ) + emit(f"completed #{ref}") + except DerivedTask.DoesNotExist: + emit(f"task_not_found:#{ref}") + return True + + if rest.startswith("undo "): + ref = rest[len("undo ") :].strip().lstrip("#") + try: + task = await sync_to_async(DerivedTask.objects.get)( + user=user, reference_code=ref + ) + await sync_to_async(task.delete)() + emit(f"removed #{ref}") + except DerivedTask.DoesNotExist: + emit(f"task_not_found:#{ref}") + return True + + emit( + "tasks: .l | .tasks list [status] [limit] | " + ".tasks add <project> :: <title> | " + ".tasks show #<ref> | " + ".tasks complete #<ref> | " + ".tasks undo #<ref>" + ) + return True + + +async def dispatch_builtin_gateway_command( + *, + user, + command_text: str, + service: str, + channel_identifier: str, + sender_identifier: str, + source_message, + message_meta: dict, + payload: dict, + emit, +) -> bool: + text = str(command_text or "").strip() + + async def _contacts_handler(_ctx, out): + persons = await sync_to_async(list)(Person.objects.filter(user=user).order_by("name")) + if not persons: + out("No contacts found.") + return True + out("Contacts: " + ", ".join([p.name for p in persons])) + return True + + async def _help_handler(_ctx, out): + for line in gateway_help_lines(): + out(line) + return True + + async def _whoami_handler(_ctx, out): + out(str(user.__dict__)) + return True + + async def _approval_handler(_ctx, out): + return await handle_approval_command(user, text, out) + + async def _tasks_handler(_ctx, out): + return await handle_tasks_command( + user, + text, + out, + service=service, + channel_identifier=channel_identifier, + sender_identifier=sender_identifier, + ) + + routes = [ + GatewayCommandRoute( + name="contacts", + scope_key="gateway.contacts", + matcher=lambda value: str(value or "").strip().lower() == ".contacts", + handler=_contacts_handler, + ), + GatewayCommandRoute( + name="help", + scope_key="gateway.help", + matcher=lambda value: str(value or "").strip().lower() == ".help", + handler=_help_handler, + ), + GatewayCommandRoute( + name="whoami", + scope_key="gateway.whoami", + matcher=lambda value: str(value or "").strip().lower() == ".whoami", + handler=_whoami_handler, + ), + GatewayCommandRoute( + name="approval", + scope_key="gateway.approval", + matcher=lambda value: str(value or "").strip().lower().startswith(".approval") + or any( + str(value or "").strip().lower().startswith(prefix + " ") + or str(value or "").strip().lower() == prefix + for prefix in APPROVAL_PROVIDER_COMMANDS + ), + handler=_approval_handler, + ), + GatewayCommandRoute( + name="tasks", + scope_key="gateway.tasks", + matcher=lambda value: bool(TASK_COMMAND_MATCH_RE.match(str(value or ""))), + handler=_tasks_handler, + ), + ] + + handled = await dispatch_gateway_command( + context=GatewayCommandContext( + user=user, + source_message=source_message, + service=str(service or "xmpp"), + channel_identifier=str(channel_identifier or ""), + sender_identifier=str(sender_identifier or ""), + message_text=text, + message_meta=dict(message_meta or {}), + payload=dict(payload or {}), + ), + routes=routes, + emit=emit, + ) + if not handled and text.startswith("."): + emit("No such command") + return handled diff --git a/core/mcp/tools.py b/core/mcp/tools.py index 317a58a..6f4dd4b 100644 --- a/core/mcp/tools.py +++ b/core/mcp/tools.py @@ -1,10 +1,12 @@ from __future__ import annotations +import datetime import json import time from pathlib import Path from typing import Any +from asgiref.sync import async_to_sync from django.conf import settings from django.db.models import Q from django.utils import timezone @@ -27,9 +29,12 @@ from core.models import ( MemoryChangeRequest, MemoryItem, TaskArtifactLink, + TaskEpic, + TaskProject, User, WorkspaceConversation, ) +from core.tasks.engine import create_task_record_and_sync, mark_task_completed_and_sync from core.util import logs log = logs.get_logger("mcp-tools") @@ -508,6 +513,55 @@ def tool_tasks_search(arguments: dict[str, Any]) -> dict[str, Any]: return tool_tasks_list(arguments) +def tool_tasks_create(arguments: dict[str, Any]) -> dict[str, Any]: + user_id = int(arguments.get("user_id")) + project_id = str(arguments.get("project_id") or "").strip() + title = str(arguments.get("title") or "").strip() + if not project_id: + raise ValueError("project_id is required") + if not title: + raise ValueError("title is required") + project = TaskProject.objects.filter(user_id=user_id, id=project_id).first() + if project is None: + raise ValueError("project_id not found") + epic_id = str(arguments.get("epic_id") or "").strip() + epic = None + if epic_id: + epic = TaskEpic.objects.filter(project=project, id=epic_id).first() + if epic is None: + raise ValueError("epic_id not found for project") + + due_at = str(arguments.get("due_date") or "").strip() + due_date = None + if due_at: + try: + due_date = datetime.date.fromisoformat(due_at) + except Exception as exc: + raise ValueError("due_date must be YYYY-MM-DD") from exc + + task, event = async_to_sync(create_task_record_and_sync)( + user=project.user, + project=project, + epic=epic, + title=title, + source_service=str(arguments.get("source_service") or "web").strip().lower() + or "web", + source_channel=str(arguments.get("source_channel") or "").strip(), + actor_identifier=str(arguments.get("actor_identifier") or "").strip(), + due_date=due_date, + assignee_identifier=str(arguments.get("assignee_identifier") or "").strip(), + immutable_payload={ + "source": "mcp.tasks.create", + "requested_by": str(arguments.get("actor_identifier") or "").strip(), + }, + event_payload={ + "source": "mcp.tasks.create", + "via": "mcp", + }, + ) + return {"task": _task_payload(task), "event": _event_payload(event)} + + def tool_tasks_get(arguments: dict[str, Any]) -> dict[str, Any]: task = _resolve_task(arguments) payload = _task_payload(task) @@ -555,6 +609,17 @@ def tool_tasks_create_note(arguments: dict[str, Any]) -> dict[str, Any]: return {"task": _task_payload(task), "event": _event_payload(event)} +def tool_tasks_complete(arguments: dict[str, Any]) -> dict[str, Any]: + task = _resolve_task(arguments) + event = async_to_sync(mark_task_completed_and_sync)( + task=task, + actor_identifier=str(arguments.get("actor_identifier") or "").strip(), + payload={"source": "mcp.tasks.complete", "via": "mcp"}, + ) + task.refresh_from_db() + return {"task": _task_payload(task), "event": _event_payload(event)} + + def tool_tasks_link_artifact(arguments: dict[str, Any]) -> dict[str, Any]: task = _resolve_task(arguments) kind = str(arguments.get("kind") or "").strip() or "note" @@ -987,6 +1052,26 @@ TOOL_DEFS: dict[str, dict[str, Any]] = { }, "handler": tool_tasks_search, }, + "tasks.create": { + "description": "Create a canonical task inside GIA.", + "inputSchema": { + "type": "object", + "properties": { + "user_id": {"type": "integer"}, + "project_id": {"type": "string"}, + "epic_id": {"type": "string"}, + "title": {"type": "string"}, + "due_date": {"type": "string"}, + "assignee_identifier": {"type": "string"}, + "actor_identifier": {"type": "string"}, + "source_service": {"type": "string"}, + "source_channel": {"type": "string"}, + }, + "required": ["user_id", "project_id", "title"], + "additionalProperties": False, + }, + "handler": tool_tasks_create, + }, "tasks.get": { "description": "Get one derived task by ID, including links.", "inputSchema": { @@ -1029,6 +1114,20 @@ TOOL_DEFS: dict[str, dict[str, Any]] = { }, "handler": tool_tasks_create_note, }, + "tasks.complete": { + "description": "Mark a task completed and append a completion event.", + "inputSchema": { + "type": "object", + "properties": { + "task_id": {"type": "string"}, + "user_id": {"type": "integer"}, + "actor_identifier": {"type": "string"}, + }, + "required": ["task_id"], + "additionalProperties": False, + }, + "handler": tool_tasks_complete, + }, "tasks.link_artifact": { "description": "Link an artifact (URI/path) to a task.", "inputSchema": { diff --git a/core/migrations/0044_userxmppsecuritysettings_encrypt_component_messages_with_omemo.py b/core/migrations/0044_userxmppsecuritysettings_encrypt_component_messages_with_omemo.py new file mode 100644 index 0000000..060c339 --- /dev/null +++ b/core/migrations/0044_userxmppsecuritysettings_encrypt_component_messages_with_omemo.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0043_userxmppsecuritysettings_encrypt_contact_messages_with_omemo"), + ] + + operations = [ + migrations.AddField( + model_name="userxmppsecuritysettings", + name="encrypt_component_messages_with_omemo", + field=models.BooleanField(default=True), + ), + ] diff --git a/core/models.py b/core/models.py index e8672c9..f4d1190 100644 --- a/core/models.py +++ b/core/models.py @@ -2979,6 +2979,7 @@ class UserXmppSecuritySettings(models.Model): related_name="xmpp_security_settings", ) require_omemo = models.BooleanField(default=False) + encrypt_component_messages_with_omemo = models.BooleanField(default=True) encrypt_contact_messages_with_omemo = models.BooleanField(default=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) diff --git a/core/security/capabilities.py b/core/security/capabilities.py new file mode 100644 index 0000000..4712db8 --- /dev/null +++ b/core/security/capabilities.py @@ -0,0 +1,111 @@ +"""Canonical capability registry for command/task/gateway scope policy.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class CapabilityScope: + key: str + label: str + description: str + group: str + configurable: bool = True + owner_path: str = "/settings/security/permissions/" + + +GLOBAL_SCOPE_KEY = "global.override" + +CAPABILITY_SCOPES: tuple[CapabilityScope, ...] = ( + CapabilityScope( + key="gateway.contacts", + label="Gateway contacts command", + description="Handles .contacts over gateway channels.", + group="gateway", + ), + CapabilityScope( + key="gateway.help", + label="Gateway help command", + description="Handles .help over gateway channels.", + group="gateway", + ), + CapabilityScope( + key="gateway.whoami", + label="Gateway whoami command", + description="Handles .whoami over gateway channels.", + group="gateway", + ), + CapabilityScope( + key="gateway.tasks", + label="Gateway .tasks commands", + description="Handles .tasks list/show/complete/undo over gateway channels.", + group="tasks", + ), + CapabilityScope( + key="gateway.approval", + label="Gateway approval commands", + description="Handles .approval/.codex/.claude approve/deny over gateway channels.", + group="command", + ), + CapabilityScope( + key="tasks.submit", + label="Task submissions from chat", + description="Controls automatic task creation from inbound messages.", + group="tasks", + owner_path="/settings/tasks/", + ), + CapabilityScope( + key="tasks.commands", + label="Task command verbs (.task/.undo/.epic)", + description="Controls explicit task command verbs.", + group="tasks", + owner_path="/settings/tasks/", + ), + CapabilityScope( + key="command.bp", + label="Business plan command", + description="Controls Business Plan command execution.", + group="command", + owner_path="/settings/command-routing/", + ), + CapabilityScope( + key="command.codex", + label="Codex command", + description="Controls Codex command execution.", + group="agentic", + owner_path="/settings/command-routing/", + ), + CapabilityScope( + key="command.claude", + label="Claude command", + description="Controls Claude command execution.", + group="agentic", + owner_path="/settings/command-routing/", + ), +) + +SCOPE_BY_KEY = {row.key: row for row in CAPABILITY_SCOPES} + +GROUP_LABELS: dict[str, str] = { + "gateway": "Gateway", + "tasks": "Tasks", + "command": "Commands", + "agentic": "Agentic", + "other": "Other", +} + + +def all_scope_keys(*, configurable_only: bool = False) -> list[str]: + rows = [ + row.key + for row in CAPABILITY_SCOPES + if (not configurable_only or bool(row.configurable)) + ] + return rows + + +def scope_record(scope_key: str) -> CapabilityScope | None: + key = str(scope_key or "").strip().lower() + return SCOPE_BY_KEY.get(key) + diff --git a/core/security/command_policy.py b/core/security/command_policy.py index f3e0d6f..f77d117 100644 --- a/core/security/command_policy.py +++ b/core/security/command_policy.py @@ -2,7 +2,10 @@ from __future__ import annotations from dataclasses import dataclass -from core.models import CommandSecurityPolicy, UserXmppOmemoState +from core.models import ( + CommandSecurityPolicy, + UserXmppOmemoTrustedKey, +) GLOBAL_SCOPE_KEY = "global.override" OVERRIDE_OPTIONS = {"per_scope", "on", "off"} @@ -64,7 +67,7 @@ def _match_channel(rule: str, channel: str) -> bool: return current == value -def _omemo_facts(ctx: CommandSecurityContext) -> tuple[str, str]: +def _omemo_facts(ctx: CommandSecurityContext) -> tuple[str, str, str]: message_meta = dict(ctx.message_meta or {}) payload = dict(ctx.payload or {}) xmpp_meta = dict(message_meta.get("xmpp") or {}) @@ -76,7 +79,10 @@ def _omemo_facts(ctx: CommandSecurityContext) -> tuple[str, str]: client_key = str( xmpp_meta.get("omemo_client_key") or payload.get("omemo_client_key") or "" ).strip() - return status, client_key + sender_jid = str( + xmpp_meta.get("sender_jid") or payload.get("sender_jid") or "" + ).strip() + return status, client_key, sender_jid def _channel_allowed_for_rules(rules: dict, service: str, channel: str) -> bool: @@ -192,7 +198,7 @@ def evaluate_command_policy( reason=f"channel={channel or '-'} not allowed by global override", ) - omemo_status, omemo_client_key = _omemo_facts(context) + omemo_status, omemo_client_key, sender_jid = _omemo_facts(context) if require_omemo and omemo_status != "detected": return CommandPolicyDecision( allowed=False, @@ -205,15 +211,25 @@ def evaluate_command_policy( return CommandPolicyDecision( allowed=False, code="trusted_fingerprint_required", - reason=f"scope={scope} requires trusted OMEMO fingerprint", + reason=f"scope={scope} requires a trusted OMEMO key", ) - state = UserXmppOmemoState.objects.filter(user=user).first() - expected_key = str(getattr(state, "latest_client_key", "") or "").strip() - if not expected_key or expected_key != omemo_client_key: + jid_bare = ( + str(sender_jid.split("/", 1)[0] if sender_jid else "").strip().lower() + ) + trusted_query = UserXmppOmemoTrustedKey.objects.filter( + user=user, + key_type="client_key", + key_id=omemo_client_key, + trusted=True, + ) + if jid_bare: + trusted_query = trusted_query.filter(jid__iexact=jid_bare) + trusted_match = trusted_query.order_by("-updated_at").first() + if trusted_match is None: return CommandPolicyDecision( allowed=False, - code="fingerprint_mismatch", - reason=f"scope={scope} OMEMO fingerprint does not match enrolled key", + code="trusted_key_missing", + reason=f"scope={scope} requires a trusted OMEMO key for this sender", ) return CommandPolicyDecision(allowed=True) diff --git a/core/tasks/engine.py b/core/tasks/engine.py index 99b03ba..ca0e33e 100644 --- a/core/tasks/engine.py +++ b/core/tasks/engine.py @@ -65,6 +65,10 @@ _TASK_COMPLETE_CMD_RE = re.compile( r"^\s*\.task\s+(?:complete|done|close)\s+#?(?P<reference>[A-Za-z0-9_-]+)\s*$", re.IGNORECASE, ) +_TASK_ADD_CMD_RE = re.compile( + r"^\s*\.task\s+(?:add|create|new)\s+(?P<title>.+?)\s*$", + re.IGNORECASE | re.DOTALL, +) _DUE_ISO_RE = re.compile( r"\b(?:due|by)\s+(\d{4}-\d{2}-\d{2})\b", re.IGNORECASE, @@ -367,6 +371,102 @@ def _next_reference(user, project) -> str: return str(DerivedTask.objects.filter(user=user, project=project).count() + 1) +def create_task_record( + *, + user, + project, + title: str, + source_service: str, + source_channel: str, + origin_message: Message | None = None, + actor_identifier: str = "", + epic: TaskEpic | None = None, + due_date: datetime.date | None = None, + assignee_identifier: str = "", + immutable_payload: dict | None = None, + event_payload: dict | None = None, + status_snapshot: str = "open", +) -> tuple[DerivedTask, DerivedTaskEvent]: + reference = _next_reference(user, project) + task = DerivedTask.objects.create( + user=user, + project=project, + epic=epic, + title=str(title or "").strip()[:255] or "Untitled task", + source_service=str(source_service or "web").strip() or "web", + source_channel=str(source_channel or "").strip(), + origin_message=origin_message, + reference_code=reference, + status_snapshot=str(status_snapshot or "open").strip() or "open", + due_date=due_date, + assignee_identifier=str(assignee_identifier or "").strip(), + immutable_payload=dict(immutable_payload or {}), + ) + event = DerivedTaskEvent.objects.create( + task=task, + event_type="created", + actor_identifier=str(actor_identifier or "").strip(), + source_message=origin_message, + payload=dict(event_payload or {}), + ) + return task, event + + +async def create_task_record_and_sync( + *, + user, + project, + title: str, + source_service: str, + source_channel: str, + origin_message: Message | None = None, + actor_identifier: str = "", + epic: TaskEpic | None = None, + due_date: datetime.date | None = None, + assignee_identifier: str = "", + immutable_payload: dict | None = None, + event_payload: dict | None = None, + status_snapshot: str = "open", +) -> tuple[DerivedTask, DerivedTaskEvent]: + task, event = await sync_to_async(create_task_record)( + user=user, + project=project, + title=title, + source_service=source_service, + source_channel=source_channel, + origin_message=origin_message, + actor_identifier=actor_identifier, + epic=epic, + due_date=due_date, + assignee_identifier=assignee_identifier, + immutable_payload=immutable_payload, + event_payload=event_payload, + status_snapshot=status_snapshot, + ) + await _emit_sync_event(task, event, "create") + return task, event + + +async def mark_task_completed_and_sync( + *, + task: DerivedTask, + actor_identifier: str = "", + source_message: Message | None = None, + payload: dict | None = None, +) -> DerivedTaskEvent: + task.status_snapshot = "completed" + await sync_to_async(task.save)(update_fields=["status_snapshot"]) + event = await sync_to_async(DerivedTaskEvent.objects.create)( + task=task, + event_type="completion_marked", + actor_identifier=str(actor_identifier or "").strip(), + source_message=source_message, + payload=dict(payload or {}), + ) + await _emit_sync_event(task, event, "complete") + return event + + async def _derive_title(message: Message) -> str: text = str(message.text or "").strip() if not text: @@ -600,6 +700,49 @@ async def _handle_scope_task_commands( await _send_scope_message(source, message, "\n".join(lines)) return True + create_match = _TASK_ADD_CMD_RE.match(body) + if create_match: + task_text = str(create_match.group("title") or "").strip() + if not task_text: + await _send_scope_message( + source, message, "[task] title is required for .task add." + ) + return True + epic = source.epic + epic_name = _extract_epic_name_from_text(task_text) + if epic_name: + epic, _ = await sync_to_async(TaskEpic.objects.get_or_create)( + project=source.project, + name=epic_name, + ) + cleaned_task_text = _strip_epic_token(task_text) + task, _event = await create_task_record_and_sync( + user=message.user, + project=source.project, + epic=epic, + title=cleaned_task_text[:255], + source_service=source.service or message.source_service or "web", + source_channel=source.channel_identifier or message.source_chat_id or "", + origin_message=message, + actor_identifier=str(message.sender_uuid or ""), + due_date=_parse_due_date(cleaned_task_text), + assignee_identifier=_parse_assignee(cleaned_task_text), + immutable_payload={ + "origin_text": text, + "task_text": cleaned_task_text, + "source": "chat_manual_command", + }, + event_payload={ + "origin_text": text, + "command": ".task add", + "via": "chat_command", + }, + ) + await _send_scope_message( + source, message, f"[task] created #{task.reference_code}: {task.title}" + ) + return True + undo_match = _UNDO_TASK_RE.match(body) if undo_match: reference = str(undo_match.group("reference") or "").strip() @@ -690,11 +833,8 @@ async def _handle_scope_task_commands( source, message, f"[task] #{reference} not found." ) return True - task.status_snapshot = "completed" - await sync_to_async(task.save)(update_fields=["status_snapshot"]) - event = await sync_to_async(DerivedTaskEvent.objects.create)( + await mark_task_completed_and_sync( task=task, - event_type="completion_marked", actor_identifier=str(message.sender_uuid or ""), source_message=message, payload={ @@ -703,7 +843,6 @@ async def _handle_scope_task_commands( "via": "chat_command", }, ) - await _emit_sync_event(task, event, "complete") await _send_scope_message( source, message, f"[task] completed #{task.reference_code}: {task.title}" ) @@ -763,6 +902,7 @@ def _is_task_command_candidate(text: str) -> bool: if ( _LIST_TASKS_RE.match(body) or _LIST_TASKS_CMD_RE.match(body) + or _TASK_ADD_CMD_RE.match(body) or _TASK_SHOW_RE.match(body) or _TASK_COMPLETE_CMD_RE.match(body) or _UNDO_TASK_RE.match(body) @@ -779,6 +919,7 @@ def _is_explicit_task_command(text: str) -> bool: return bool( _LIST_TASKS_RE.match(body) or _LIST_TASKS_CMD_RE.match(body) + or _TASK_ADD_CMD_RE.match(body) or _TASK_SHOW_RE.match(body) or _TASK_COMPLETE_CMD_RE.match(body) or _UNDO_TASK_RE.match(body) @@ -878,16 +1019,12 @@ async def process_inbound_task_intelligence(message: Message) -> None: ) return - task.status_snapshot = "completed" - await sync_to_async(task.save)(update_fields=["status_snapshot"]) - event = await sync_to_async(DerivedTaskEvent.objects.create)( + await mark_task_completed_and_sync( task=task, - event_type="completion_marked", actor_identifier=str(message.sender_uuid or ""), source_message=message, payload={"marker": ref_code}, ) - await _emit_sync_event(task, event, "complete") return for source in sources: @@ -913,10 +1050,9 @@ async def process_inbound_task_intelligence(message: Message) -> None: source_chat_id=message.source_chat_id, ) title = await _derive_title_with_flags(cloned_message, flags) - reference = await sync_to_async(_next_reference)(message.user, source.project) parsed_due_date = _parse_due_date(task_text) parsed_assignee = _parse_assignee(task_text) - task = await sync_to_async(DerivedTask.objects.create)( + task, event = await create_task_record_and_sync( user=message.user, project=source.project, epic=epic, @@ -924,8 +1060,7 @@ async def process_inbound_task_intelligence(message: Message) -> None: source_service=source.service or message.source_service or "web", source_channel=source.channel_identifier or message.source_chat_id or "", origin_message=message, - reference_code=reference, - status_snapshot="open", + actor_identifier=str(message.sender_uuid or ""), due_date=parsed_due_date, assignee_identifier=parsed_assignee, immutable_payload={ @@ -933,15 +1068,8 @@ async def process_inbound_task_intelligence(message: Message) -> None: "task_text": task_text, "flags": flags, }, + event_payload={"origin_text": text}, ) - event = await sync_to_async(DerivedTaskEvent.objects.create)( - task=task, - event_type="created", - actor_identifier=str(message.sender_uuid or ""), - source_message=message, - payload={"origin_text": text}, - ) - await _emit_sync_event(task, event, "create") if bool(flags.get("announce_task_id", False)): try: await send_message_raw( diff --git a/core/templates/base.html b/core/templates/base.html index 2dd8ad2..6b2dfc2 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -167,6 +167,10 @@ .panel, .box, .modal { background-color: var(--modal-color) !important; } + .box { + border: 1px solid rgba(25, 33, 52, 0.08); + box-shadow: 0 18px 48px rgba(20, 28, 45, 0.08); + } .modal, .modal.box{ background-color: var(--background-color) !important; } @@ -200,6 +204,57 @@ .navbar { background-color:rgba(0, 0, 0, 0.03) !important; } + .section > .container.gia-page-shell, + .section > .container { + max-width: 1340px; + } + .gia-page-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 1rem; + flex-wrap: wrap; + } + .gia-page-header .title, + .gia-page-header .subtitle { + max-width: 72ch; + } + .gia-page-header .subtitle { + margin-bottom: 0; + } + .table thead th { + position: sticky; + top: 0; + background: rgba(248, 250, 252, 0.96) !important; + backdrop-filter: blur(6px); + z-index: 1; + } + [data-theme="dark"] .table thead th { + background: rgba(44, 44, 44, 0.96) !important; + } + .table td, + .table th { + vertical-align: top; + } + .help { + max-width: 78ch; + } + .button.is-light { + border-color: rgba(27, 38, 59, 0.12); + } + .input, + .textarea, + .select select { + border-color: rgba(27, 38, 59, 0.18); + box-shadow: none; + } + .input:focus, + .textarea:focus, + .select select:focus { + border-color: rgba(27, 99, 214, 0.8); + box-shadow: 0 0 0 0.125em rgba(27, 99, 214, 0.14); + } .grid-stack-item-content, .floating-window { diff --git a/core/templates/pages/codex-settings.html b/core/templates/pages/codex-settings.html index b90b27b..5a6e641 100644 --- a/core/templates/pages/codex-settings.html +++ b/core/templates/pages/codex-settings.html @@ -2,9 +2,13 @@ {% block content %} <section class="section"> - <div class="container"> - <h1 class="title is-4">Codex Status</h1> - <p class="subtitle is-6">Global per-user Codex task-sync status, runs, and approvals.</p> + <div class="container gia-page-shell"> + <div class="gia-page-header"> + <div> + <h1 class="title is-4">Codex Status</h1> + <p class="subtitle is-6">Worker-backed task sync status, runs, and approvals for the canonical GIA task store.</p> + </div> + </div> <article class="box"> <div class="codex-inline-stats"> diff --git a/core/templates/pages/command-routing.html b/core/templates/pages/command-routing.html index 1e427f7..51a332c 100644 --- a/core/templates/pages/command-routing.html +++ b/core/templates/pages/command-routing.html @@ -5,6 +5,11 @@ <div class="container"> <h1 class="title is-4">Command Routing</h1> <p class="subtitle is-6">Configure commands, channel bindings, and per-command delivery in a predictable way.</p> + <p class="help"> + Related controls: + <a href="{% url 'tasks_settings' %}">Task Automation</a> and + <a href="{% url 'permission_settings' %}">Security Permissions</a>. + </p> {% if scope_service and scope_identifier %} <article class="notification is-info is-light"> Scoped to this chat only: <strong>{{ scope_service }}</strong> · <code>{{ scope_identifier }}</code> diff --git a/core/templates/pages/compose-contact-match.html b/core/templates/pages/compose-contact-match.html index 9192032..3b21d5a 100644 --- a/core/templates/pages/compose-contact-match.html +++ b/core/templates/pages/compose-contact-match.html @@ -190,13 +190,22 @@ data-person="{{ row.linked_person_name|default:'-'|lower }}" data-detected="{{ row.detected_name|default:'-'|lower }}" data-identifier="{{ row.identifier|lower }}" - data-search="{{ row.linked_person_name|default:'-'|lower }} {{ row.detected_name|default:'-'|lower }} {{ row.service|lower }} {{ row.identifier|lower }}"> + data-search="{{ row.linked_person_name|default:'-'|lower }} {{ row.detected_name|default:'-'|lower }} {{ row.service|lower }} {{ row.identifier_search|default:row.identifier|lower }}"> <td data-discovered-col="0" class="discovered-col-0">{{ row.linked_person_name|default:"-" }}</td> <td data-discovered-col="1" class="discovered-col-1">{{ row.detected_name|default:"-" }}</td> <td data-discovered-col="2" class="discovered-col-2"> {{ row.service|title }} </td> - <td data-discovered-col="3" class="discovered-col-3"><code>{{ row.identifier }}</code></td> + <td data-discovered-col="3" class="discovered-col-3"> + <code>{{ row.identifier }}</code> + {% if row.identifier_aliases %} + <div class="is-size-7 has-text-grey mt-1"> + {% for alias in row.identifier_aliases %} + <div><code>{{ alias }}</code></div> + {% endfor %} + </div> + {% endif %} + </td> <td data-discovered-col="4" class="discovered-col-4"> {% if not row.linked_person %} <div class="buttons are-small"> diff --git a/core/templates/pages/security.html b/core/templates/pages/security.html index 4d6da48..2ee7226 100644 --- a/core/templates/pages/security.html +++ b/core/templates/pages/security.html @@ -48,12 +48,20 @@ <p class="help is-size-7 has-text-grey">This is separate from command-scope policy checks such as Require Trusted Fingerprint.</p> </div> <input type="hidden" name="encrypt_contact_messages_with_omemo" value="0"> + <input type="hidden" name="encrypt_component_messages_with_omemo" value="0"> + <div class="field mt-3"> + <label class="checkbox"> + <input type="checkbox" name="encrypt_component_messages_with_omemo" value="1"{% if security_settings.encrypt_component_messages_with_omemo %} checked{% endif %}> + Encrypt gateway component chat replies with OMEMO + </label> + <p class="help is-size-7 has-text-grey mt-1">Controls only gateway/component command replies (for example, <code>.tasks</code> or approvals) sent to your XMPP client.</p> + </div> <div class="field mt-3"> <label class="checkbox"> <input type="checkbox" name="encrypt_contact_messages_with_omemo" value="1"{% if security_settings.encrypt_contact_messages_with_omemo %} checked{% endif %}> Encrypt contact relay messages to your XMPP client with OMEMO </label> - <p class="help is-size-7 has-text-grey mt-1">When enabled, relay text from contacts is sent with OMEMO when available. If disabled, relay text is sent in plaintext.</p> + <p class="help is-size-7 has-text-grey mt-1">Controls relayed contact chat text. Keep this off if you want normal contact chats while securing only component workflows.</p> </div> <button class="button is-link is-small" type="submit">Save</button> </form> @@ -194,7 +202,7 @@ <div class="box"> <h2 class="title is-6">Global Scope Override</h2> <p class="is-size-7 has-text-grey mb-3"> - This scope can force settings across all Command Security Scopes. + This scope can force settings across all Fine-Grained Security Scopes. </p> <div class="box" style="margin: 0; border: 1px solid rgba(60, 60, 60, 0.12);"> <form method="post"> @@ -341,7 +349,7 @@ {% endfor %} </select> </div> - <input class="input is-small" name="allowed_channel_pattern" value="{{ rule.pattern }}" placeholder="m@zm.is* or 1203*"> + <input class="input is-small" name="allowed_channel_pattern" value="{{ rule.pattern }}" placeholder="user@example.test* or 1203*"> <button class="button is-small is-light is-danger channel-rule-remove" type="button">Remove</button> </div> {% endfor %} @@ -454,7 +462,7 @@ {% endfor %} </select> </div> - <input class="input is-small scope-editable" data-lock-state="free" name="allowed_channel_pattern" value="{{ rule.pattern }}" placeholder="m@zm.is* or 1203*"> + <input class="input is-small scope-editable" data-lock-state="free" name="allowed_channel_pattern" value="{{ rule.pattern }}" placeholder="user@example.test* or 1203*"> <button class="button is-small is-light is-danger channel-rule-remove scope-editable" data-lock-state="free" type="button">Remove</button> </div> {% endfor %} @@ -516,7 +524,7 @@ {% endfor %} </select> </div> - <input class="input is-small scope-editable" data-lock-state="free" name="allowed_channel_pattern" value="" placeholder="m@zm.is* or 1203*"> + <input class="input is-small scope-editable" data-lock-state="free" name="allowed_channel_pattern" value="" placeholder="user@example.test* or 1203*"> <button class="button is-small is-light is-danger channel-rule-remove scope-editable" data-lock-state="free" type="button">Remove</button> </div> </template> diff --git a/core/templates/pages/tasks-detail.html b/core/templates/pages/tasks-detail.html index bd0fccc..e113951 100644 --- a/core/templates/pages/tasks-detail.html +++ b/core/templates/pages/tasks-detail.html @@ -1,8 +1,12 @@ {% extends "base.html" %} {% block content %} - <section class="section"><div class="container"> - <h1 class="title is-4">Task #{{ task.reference_code }}: {{ task.title }}</h1> - <p class="subtitle is-6">{{ task.project.name }}{% if task.epic %} / {{ task.epic.name }}{% endif %} · {{ task.status_snapshot }}</p> + <section class="section"><div class="container gia-page-shell"> + <div class="gia-page-header"> + <div> + <h1 class="title is-4">Task #{{ task.reference_code }}: {{ task.title }}</h1> + <p class="subtitle is-6">{{ task.project.name }}{% if task.epic %} / {{ task.epic.name }}{% endif %} · {{ task.status_snapshot }}</p> + </div> + </div> <p class="is-size-7 has-text-grey" style="margin-top:-0.65rem; margin-bottom: 0.65rem;"> Created by {{ task.creator_label|default:"Unknown" }} {% if task.origin_message_id %} diff --git a/core/templates/pages/tasks-hub.html b/core/templates/pages/tasks-hub.html index 22c434b..eb0e677 100644 --- a/core/templates/pages/tasks-hub.html +++ b/core/templates/pages/tasks-hub.html @@ -1,9 +1,13 @@ {% extends "base.html" %} {% block content %} <section class="section"> - <div class="container"> - <h1 class="title is-4">Task Inbox</h1> - <p class="subtitle is-6">Immutable tasks derived from chat activity.</p> + <div class="container gia-page-shell"> + <div class="gia-page-header"> + <div> + <h1 class="title is-4">Task Inbox</h1> + <p class="subtitle is-6">Canonical tasks live in GIA. Chats, XMPP, web UI, and agent tooling all operate on the same records.</p> + </div> + </div> <div class="buttons" style="margin-bottom: 0.75rem;"> <a class="button is-small is-link is-light" href="{% url 'tasks_settings' %}{% if scope.person_id or scope.service or scope.identifier %}?{% if scope.person_id %}person={{ scope.person_id|urlencode }}{% endif %}{% if scope.service %}{% if scope.person_id %}&{% endif %}service={{ scope.service|urlencode }}{% endif %}{% if scope.identifier %}{% if scope.person_id or scope.service %}&{% endif %}identifier={{ scope.identifier|urlencode }}{% endif %}{% endif %}">Task Automation</a> </div> @@ -139,6 +143,60 @@ </article> </div> <div class="column"> + <article class="box"> + <div class="is-flex is-justify-content-space-between is-align-items-center" style="gap: 0.5rem; margin-bottom: 0.6rem; flex-wrap: wrap;"> + <h2 class="title is-6" style="margin: 0;">Create Task</h2> + {% if scope.service or scope.identifier %} + <span class="tag task-ui-badge">{{ scope.service|default:"web" }}{% if scope.identifier %} · {{ scope.identifier }}{% endif %}</span> + {% else %} + <span class="tag task-ui-badge">web</span> + {% endif %} + </div> + <p class="help" style="margin-bottom: 0.65rem;">Use this to create canonical tasks directly from the web UI without relying on WhatsApp-derived source state.</p> + <form method="post"> + {% csrf_token %} + <input type="hidden" name="action" value="task_create"> + <input type="hidden" name="person" value="{{ scope.person_id }}"> + <input type="hidden" name="service" value="{{ scope.service }}"> + <input type="hidden" name="identifier" value="{{ scope.identifier }}"> + <div class="columns is-multiline"> + <div class="column is-7"> + <label class="label is-size-7">Title</label> + <input class="input is-small" name="title" placeholder="Ship MCP browser validation for compose"> + </div> + <div class="column is-5"> + <label class="label is-size-7">Project</label> + <div class="select is-small is-fullwidth"> + <select name="project_id"> + {% for project in project_choices %} + <option value="{{ project.id }}" {% if selected_project and selected_project.id == project.id %}selected{% endif %}>{{ project.name }}</option> + {% endfor %} + </select> + </div> + </div> + <div class="column is-5"> + <label class="label is-size-7">Epic (optional)</label> + <div class="select is-small is-fullwidth"> + <select name="epic_id"> + <option value="">No epic</option> + {% for epic in epic_choices %} + <option value="{{ epic.id }}">{{ epic.project.name }} / {{ epic.name }}</option> + {% endfor %} + </select> + </div> + </div> + <div class="column is-3"> + <label class="label is-size-7">Due Date</label> + <input class="input is-small" type="date" name="due_date"> + </div> + <div class="column is-4"> + <label class="label is-size-7">Assignee</label> + <input class="input is-small" name="assignee_identifier" placeholder="@operator"> + </div> + </div> + <button class="button is-small is-link" type="submit">Create Task</button> + </form> + </article> <article class="box"> <h2 class="title is-6">Recent Derived Tasks</h2> <table class="table is-fullwidth is-striped is-size-7"> diff --git a/core/templates/pages/tasks-settings.html b/core/templates/pages/tasks-settings.html index 504b44c..435e7fe 100644 --- a/core/templates/pages/tasks-settings.html +++ b/core/templates/pages/tasks-settings.html @@ -4,6 +4,11 @@ <div class="container tasks-settings-page"> <h1 class="title is-4">Task Automation</h1> <p class="subtitle is-6">Project defaults flow into channel overrides. Use Quick Setup for normal operation; open Advanced Setup for full controls.</p> + <p class="help"> + Related controls: + <a href="{% url 'command_routing' %}">Commands</a> and + <a href="{% url 'permission_settings' %}">Security Permissions</a>. + </p> <div class="notification is-light"> <div class="content is-size-7"> diff --git a/core/tests/test_command_routing_variant_ui.py b/core/tests/test_command_routing_variant_ui.py index 00c7194..f3e5606 100644 --- a/core/tests/test_command_routing_variant_ui.py +++ b/core/tests/test_command_routing_variant_ui.py @@ -33,6 +33,7 @@ class CommandRoutingVariantUITests(TestCase): self.assertContains(response, "bp set range") self.assertContains(response, "Send status to egress") self.assertContains(response, "Codex (codex)") + self.assertContains(response, "Claude (claude)") def test_variant_policy_update_persists(self): response = self.client.post( diff --git a/core/tests/test_command_security_policy.py b/core/tests/test_command_security_policy.py index 7961a83..0fe4980 100644 --- a/core/tests/test_command_security_policy.py +++ b/core/tests/test_command_security_policy.py @@ -20,7 +20,7 @@ from core.models import ( Person, PersonIdentifier, User, - UserXmppOmemoState, + UserXmppOmemoTrustedKey, ) from core.security.command_policy import CommandSecurityContext, evaluate_command_policy @@ -37,7 +37,7 @@ class CommandSecurityPolicyTests(TestCase): user=self.user, person=self.person, service="xmpp", - identifier="policy-user@zm.is", + identifier="policy-user@example.test", ) self.session = ChatSession.objects.create( user=self.user, @@ -58,7 +58,7 @@ class CommandSecurityPolicyTests(TestCase): profile=profile, direction="ingress", service="xmpp", - channel_identifier="policy-user@zm.is", + channel_identifier="policy-user@example.test", enabled=True, ) CommandSecurityPolicy.objects.create( @@ -74,13 +74,13 @@ class CommandSecurityPolicyTests(TestCase): text="#bp#", ts=1000, source_service="xmpp", - source_chat_id="policy-user@zm.is", + source_chat_id="policy-user@example.test", message_meta={}, ) results = async_to_sync(process_inbound_message)( CommandContext( service="xmpp", - channel_identifier="policy-user@zm.is", + channel_identifier="policy-user@example.test", message_id=str(msg.id), user_id=self.user.id, message_text="#bp#", @@ -101,12 +101,13 @@ class CommandSecurityPolicyTests(TestCase): require_omemo=True, require_trusted_omemo_fingerprint=True, ) - UserXmppOmemoState.objects.create( + UserXmppOmemoTrustedKey.objects.create( user=self.user, - status="detected", - latest_client_key="sid:abc", - last_sender_jid="policy-user@zm.is/phone", - last_target_jid="jews.zm.is", + jid="policy-user@example.test", + key_type="client_key", + key_id="sid:abc", + trusted=True, + source="test", ) outputs: list[str] = [] @@ -119,11 +120,15 @@ class CommandSecurityPolicyTests(TestCase): user=self.user, source_message=None, service="xmpp", - channel_identifier="policy-user@zm.is", - sender_identifier="policy-user@zm.is/phone", + channel_identifier="policy-user@example.test", + sender_identifier="policy-user@example.test/phone", message_text=".tasks list", message_meta={ - "xmpp": {"omemo_status": "detected", "omemo_client_key": "sid:abc"} + "xmpp": { + "omemo_status": "detected", + "omemo_client_key": "sid:abc", + "sender_jid": "policy-user@example.test/phone", + } }, payload={}, ), @@ -161,8 +166,8 @@ class CommandSecurityPolicyTests(TestCase): user=self.user, source_message=None, service="xmpp", - channel_identifier="policy-user@zm.is", - sender_identifier="policy-user@zm.is/phone", + channel_identifier="policy-user@example.test", + sender_identifier="policy-user@example.test/phone", message_text=".tasks list", message_meta={"xmpp": {"omemo_status": "no_omemo"}}, payload={}, @@ -200,7 +205,7 @@ class CommandSecurityPolicyTests(TestCase): scope_key="gateway.tasks", context=CommandSecurityContext( service="xmpp", - channel_identifier="policy-user@zm.is", + channel_identifier="policy-user@example.test", message_meta={}, payload={}, ), @@ -226,3 +231,30 @@ class CommandSecurityPolicyTests(TestCase): ) self.assertFalse(decision.allowed) self.assertEqual("service_not_allowed", decision.code) + + def test_trusted_key_requirement_blocks_untrusted_key(self): + CommandSecurityPolicy.objects.create( + user=self.user, + scope_key="gateway.tasks", + enabled=True, + require_omemo=True, + require_trusted_omemo_fingerprint=True, + ) + decision = evaluate_command_policy( + user=self.user, + scope_key="gateway.tasks", + context=CommandSecurityContext( + service="xmpp", + channel_identifier="policy-user@example.test", + message_meta={ + "xmpp": { + "omemo_status": "detected", + "omemo_client_key": "sid:missing", + "sender_jid": "policy-user@example.test/phone", + } + }, + payload={}, + ), + ) + self.assertFalse(decision.allowed) + self.assertEqual("trusted_key_missing", decision.code) diff --git a/core/tests/test_cross_platform_messaging.py b/core/tests/test_cross_platform_messaging.py index 86e481b..e50a13a 100644 --- a/core/tests/test_cross_platform_messaging.py +++ b/core/tests/test_cross_platform_messaging.py @@ -304,16 +304,16 @@ class XMPPReplyExtractionTests(SimpleTestCase): "xmpp", { "reply_source_message_id": "xmpp-anchor-001", - "reply_source_chat_id": "user@zm.is/mobile", + "reply_source_chat_id": "user@example.test/mobile", }, ) self.assertEqual("xmpp-anchor-001", ref.get("reply_source_message_id")) self.assertEqual("xmpp", ref.get("reply_source_service")) - self.assertEqual("user@zm.is/mobile", ref.get("reply_source_chat_id")) + self.assertEqual("user@example.test/mobile", ref.get("reply_source_chat_id")) def test_extract_reply_ref_returns_empty_for_missing_id(self): ref = reply_sync.extract_reply_ref( - "xmpp", {"reply_source_chat_id": "user@zm.is"} + "xmpp", {"reply_source_chat_id": "user@example.test"} ) self.assertEqual({}, ref) @@ -333,7 +333,7 @@ class XMPPReplyResolutionTests(TestCase): user=self.user, person=self.person, service="xmpp", - identifier="contact@zm.is", + identifier="contact@example.test", ) self.session = ChatSession.objects.create( user=self.user, identifier=self.identifier @@ -345,8 +345,8 @@ class XMPPReplyResolutionTests(TestCase): text="xmpp anchor", source_service="xmpp", source_message_id="xmpp-anchor-001", - source_chat_id="contact@zm.is/mobile", - sender_uuid="contact@zm.is", + source_chat_id="contact@example.test/mobile", + sender_uuid="contact@example.test", ) def test_resolve_reply_target_by_source_message_id(self): @@ -354,7 +354,7 @@ class XMPPReplyResolutionTests(TestCase): "xmpp", { "reply_source_message_id": "xmpp-anchor-001", - "reply_source_chat_id": "contact@zm.is/mobile", + "reply_source_chat_id": "contact@example.test/mobile", }, ) target = async_to_sync(reply_sync.resolve_reply_target)( @@ -371,7 +371,7 @@ class XMPPReplyResolutionTests(TestCase): target_ts=int(self.anchor.ts), emoji="🔥", source_service="xmpp", - actor="contact@zm.is", + actor="contact@example.test", remove=False, payload={"target_xmpp_id": "xmpp-anchor-001"}, ) diff --git a/core/tests/test_mcp_tools.py b/core/tests/test_mcp_tools.py index b6c596b..3910771 100644 --- a/core/tests/test_mcp_tools.py +++ b/core/tests/test_mcp_tools.py @@ -67,6 +67,8 @@ class MCPToolTests(TestCase): names = {item["name"] for item in tool_specs()} self.assertIn("manticore.status", names) self.assertIn("memory.propose", names) + self.assertIn("tasks.create", names) + self.assertIn("tasks.complete", names) self.assertIn("tasks.link_artifact", names) self.assertIn("wiki.create_article", names) self.assertIn("project.get_runbook", names) @@ -102,6 +104,35 @@ class MCPToolTests(TestCase): "created", str((events_payload.get("items") or [{}])[0].get("event_type")) ) + def test_task_create_and_complete_tools(self): + create_payload = execute_tool( + "tasks.create", + { + "user_id": self.user.id, + "project_id": str(self.project.id), + "title": "Create via MCP", + "source_service": "xmpp", + "source_channel": "component.example.test", + "actor_identifier": "mcp-user", + }, + ) + task_payload = create_payload.get("task") or {} + self.assertEqual("Create via MCP", str(task_payload.get("title") or "")) + self.assertEqual("xmpp", str(task_payload.get("source_service") or "")) + + complete_payload = execute_tool( + "tasks.complete", + { + "user_id": self.user.id, + "task_id": str(task_payload.get("id") or ""), + "actor_identifier": "mcp-user", + }, + ) + completed_task = complete_payload.get("task") or {} + self.assertEqual( + "completed", str(completed_task.get("status_snapshot") or "") + ) + def test_memory_proposal_review_flow(self): propose_payload = execute_tool( "memory.propose", diff --git a/core/tests/test_presence_query_and_compose_context.py b/core/tests/test_presence_query_and_compose_context.py index 5fd9bf6..28b1c31 100644 --- a/core/tests/test_presence_query_and_compose_context.py +++ b/core/tests/test_presence_query_and_compose_context.py @@ -2,14 +2,19 @@ from __future__ import annotations from django.test import TestCase -from core.models import Person, PersonIdentifier, User +from core.clients import transport +from core.models import Person, PersonIdentifier, PlatformChatLink, User from core.presence import ( AvailabilitySignal, latest_state_for_people, record_native_signal, ) from core.presence.inference import now_ms -from core.views.compose import _compose_availability_payload, _context_base +from core.views.compose import ( + _compose_availability_payload, + _context_base, + _manual_contact_rows, +) class PresenceQueryAndComposeContextTests(TestCase): @@ -79,3 +84,84 @@ class PresenceQueryAndComposeContextTests(TestCase): self.assertEqual("whatsapp", str(slices[0].get("service"))) self.assertEqual("available", str(summary.get("state"))) self.assertTrue(bool(summary.get("is_cross_service"))) + + def test_context_base_preserves_native_signal_group_identifier(self): + PlatformChatLink.objects.create( + user=self.user, + service="signal", + chat_identifier="signal-group-123", + chat_name="Signal Group", + is_group=True, + ) + + base = _context_base( + user=self.user, + service="signal", + identifier="signal-group-123", + person=None, + ) + + self.assertTrue(bool(base["is_group"])) + self.assertEqual("signal-group-123", str(base["identifier"])) + + def test_manual_contact_rows_include_signal_groups(self): + PlatformChatLink.objects.create( + user=self.user, + service="signal", + chat_identifier="signal-group-123", + chat_name="Misinformation Club", + is_group=True, + ) + + rows = _manual_contact_rows(self.user) + match = next( + ( + row + for row in rows + if str(row.get("service")) == "signal" + and str(row.get("identifier")) == "signal-group-123" + ), + None, + ) + + self.assertIsNotNone(match) + self.assertEqual("Misinformation Club", str(match.get("detected_name") or "")) + + def test_manual_contact_rows_collapse_signal_group_aliases(self): + PlatformChatLink.objects.create( + user=self.user, + service="signal", + chat_identifier="group.signal-club", + chat_name="Misinformation Club", + is_group=True, + ) + transport.update_runtime_state( + "signal", + accounts=["+447700900001"], + groups=[ + { + "identifier": "group.signal-club", + "identifiers": [ + "group.signal-club", + "sEGA9F0HQ/eyLgmvKx23hha9Vp7mDRhpq23/roVSZbI=", + ], + "name": "Misinformation Club", + "id": "group.signal-club", + "internal_id": "sEGA9F0HQ/eyLgmvKx23hha9Vp7mDRhpq23/roVSZbI=", + } + ], + ) + + rows = [ + row + for row in _manual_contact_rows(self.user) + if str(row.get("service")) == "signal" + and str(row.get("detected_name")) == "Misinformation Club" + ] + + self.assertEqual(1, len(rows)) + self.assertEqual("group.signal-club", str(rows[0].get("identifier") or "")) + self.assertEqual( + ["sEGA9F0HQ/eyLgmvKx23hha9Vp7mDRhpq23/roVSZbI="], + rows[0].get("identifier_aliases"), + ) diff --git a/core/tests/test_settings_integrity.py b/core/tests/test_settings_integrity.py new file mode 100644 index 0000000..33e1533 --- /dev/null +++ b/core/tests/test_settings_integrity.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from django.test import RequestFactory, TestCase +from django.urls import resolve, reverse + +from core.context_processors import settings_hierarchy_nav +from core.security.capabilities import all_scope_keys +from core.models import CommandProfile, TaskProject, User + + +class SettingsIntegrityTests(TestCase): + def setUp(self): + self.user = User.objects.create_user( + username="settings-user", + email="settings@example.com", + password="x", + ) + self.client.force_login(self.user) + self.factory = RequestFactory() + + def test_permissions_page_shows_gateway_capabilities(self): + response = self.client.get(reverse("permission_settings")) + self.assertEqual(200, response.status_code) + self.assertContains(response, "Gateway contacts command") + self.assertContains(response, "Gateway help command") + self.assertContains(response, "Gateway whoami command") + for scope_key in all_scope_keys(configurable_only=True): + self.assertContains(response, scope_key) + + def test_capability_registry_excludes_removed_totp_scope(self): + self.assertNotIn("gateway.totp", all_scope_keys()) + + def test_codex_settings_receives_modules_settings_nav(self): + response = self.client.get(reverse("codex_settings")) + self.assertEqual(200, response.status_code) + settings_nav = response.context.get("settings_nav") + self.assertIsNotNone(settings_nav) + self.assertEqual("Modules", settings_nav["title"]) + labels = [str(item["label"]) for item in settings_nav["tabs"]] + self.assertIn("Commands", labels) + self.assertIn("Task Automation", labels) + + def test_business_plan_inbox_receives_modules_settings_nav(self): + response = self.client.get(reverse("business_plan_inbox")) + self.assertEqual(200, response.status_code) + settings_nav = response.context.get("settings_nav") + self.assertIsNotNone(settings_nav) + self.assertEqual("Modules", settings_nav["title"]) + + def test_tasks_settings_cross_links_commands_and_permissions(self): + TaskProject.objects.create(user=self.user, name="Integrity Project") + response = self.client.get(reverse("tasks_settings")) + self.assertEqual(200, response.status_code) + self.assertContains(response, "Task Automation") + self.assertContains(response, reverse("command_routing")) + self.assertContains(response, reverse("permission_settings")) + + def test_command_routing_cross_links_tasks_and_permissions(self): + CommandProfile.objects.create( + user=self.user, + slug="bp", + name="Business Plan", + enabled=True, + trigger_token=".bp", + reply_required=False, + exact_match_only=False, + ) + response = self.client.get(reverse("command_routing")) + self.assertEqual(200, response.status_code) + self.assertContains(response, reverse("tasks_settings")) + self.assertContains(response, reverse("permission_settings")) + + def test_settings_nav_includes_codex_approval_route(self): + request = self.factory.post(reverse("codex_approval")) + request.user = self.user + request.resolver_match = resolve(reverse("codex_approval")) + settings_nav = settings_hierarchy_nav(request)["settings_nav"] + self.assertEqual("Modules", settings_nav["title"]) + + def test_settings_nav_includes_translation_preview_route(self): + request = self.factory.post(reverse("translation_preview")) + request.user = self.user + request.resolver_match = resolve(reverse("translation_preview")) + settings_nav = settings_hierarchy_nav(request)["settings_nav"] + self.assertEqual("Modules", settings_nav["title"]) diff --git a/core/tests/test_signal_relink.py b/core/tests/test_signal_relink.py index d69cb8d..c1366e9 100644 --- a/core/tests/test_signal_relink.py +++ b/core/tests/test_signal_relink.py @@ -43,3 +43,22 @@ class SignalRelinkTests(TestCase): ) self.assertEqual(200, response.status_code) mock_unlink_account.assert_called_once_with("signal", "+447000000001") + + @patch("core.views.signal.transport.get_link_qr") + def test_signal_account_add_renders_notify_when_qr_fetch_fails(self, mock_get_link_qr): + mock_get_link_qr.side_effect = RuntimeError("timeout") + + response = self.client.post( + reverse( + "signal_account_add", + kwargs={"type": "modal"}, + ), + {"device": "My Device"}, + HTTP_HX_REQUEST="true", + HTTP_HX_TARGET="modals-here", + ) + + self.assertEqual(200, response.status_code) + self.assertContains(response, "modal is-active") + self.assertContains(response, "Signal QR link is unavailable right now") + self.assertContains(response, "timeout") diff --git a/core/tests/test_signal_unlink_fallback.py b/core/tests/test_signal_unlink_fallback.py index 01e9128..2bbafab 100644 --- a/core/tests/test_signal_unlink_fallback.py +++ b/core/tests/test_signal_unlink_fallback.py @@ -1,3 +1,4 @@ +import tempfile from unittest.mock import Mock, patch from django.test import TestCase @@ -6,6 +7,40 @@ from core.clients import transport class SignalUnlinkFallbackTests(TestCase): + def test_signal_wipe_uses_project_signal_cli_config_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + signal_root = os.path.join(tmpdir, "signal-cli-config") + os.makedirs(signal_root, exist_ok=True) + account_dir = os.path.join(signal_root, "account-data") + os.makedirs(account_dir, exist_ok=True) + keep_file = os.path.join(signal_root, "jsonrpc2.yml") + with open(keep_file, "w", encoding="utf-8") as handle: + handle.write("jsonrpc") + data_file = os.path.join(account_dir, "state.db") + with open(data_file, "w", encoding="utf-8") as handle: + handle.write("state") + + with patch.object(transport.settings, "BASE_DIR", tmpdir): + result = transport._wipe_signal_cli_local_state() + + self.assertTrue(result) + self.assertTrue(os.path.exists(keep_file)) + self.assertFalse(os.path.exists(account_dir)) + + @patch("requests.get") + def test_signal_list_accounts_uses_fast_timeout(self, mock_get): + ok_response = Mock() + ok_response.ok = True + ok_response.text = "[]" + mock_get.return_value = ok_response + + result = transport.list_accounts("signal") + + self.assertEqual([], result) + mock_get.assert_called_once() + _, kwargs = mock_get.call_args + self.assertEqual(5, int(kwargs.get("timeout") or 0)) + @patch("core.clients.transport._wipe_signal_cli_local_state") @patch("requests.delete") def test_signal_unlink_uses_rest_delete_when_available( @@ -40,3 +75,25 @@ class SignalUnlinkFallbackTests(TestCase): self.assertTrue(result) self.assertEqual(2, mock_delete.call_count) mock_wipe.assert_called_once() + + @patch("core.clients.transport.list_accounts") + @patch("core.clients.transport._wipe_signal_cli_local_state") + @patch("requests.delete") + def test_signal_unlink_returns_false_when_account_still_listed_after_wipe( + self, + mock_delete, + mock_wipe, + mock_list_accounts, + ): + bad_response = Mock() + bad_response.ok = False + mock_delete.return_value = bad_response + mock_wipe.return_value = True + mock_list_accounts.return_value = ["+447700900000"] + + result = transport.unlink_account("signal", "+447700900000") + + self.assertFalse(result) + self.assertEqual(2, mock_delete.call_count) + mock_wipe.assert_called_once() + mock_list_accounts.assert_called_once_with("signal") diff --git a/core/tests/test_tasks_pages_management.py b/core/tests/test_tasks_pages_management.py index fcb7fb4..37f8c77 100644 --- a/core/tests/test_tasks_pages_management.py +++ b/core/tests/test_tasks_pages_management.py @@ -251,6 +251,28 @@ class TasksPagesManagementTests(TestCase): self.assertEqual(200, response.status_code) self.assertContains(response, "Scope Person") + def test_tasks_hub_can_create_manual_task_without_chat_source(self): + project = TaskProject.objects.create(user=self.user, name="Manual Project") + response = self.client.post( + reverse("tasks_hub"), + { + "action": "task_create", + "project_id": str(project.id), + "title": "Manual web task", + "due_date": "2026-03-10", + "assignee_identifier": "@operator", + }, + follow=True, + ) + self.assertEqual(200, response.status_code) + task = DerivedTask.objects.get(user=self.user, project=project, title="Manual web task") + self.assertEqual("web", task.source_service) + self.assertEqual("@operator", task.assignee_identifier) + self.assertEqual("2026-03-10", task.due_date.isoformat()) + event = task.events.order_by("-created_at").first() + self.assertEqual("created", event.event_type) + self.assertEqual("web_ui", str((event.payload or {}).get("via") or "")) + def test_project_page_creator_column_links_to_compose(self): project = TaskProject.objects.create(user=self.user, name="Creator Link Test") session = ChatSession.objects.create(user=self.user, identifier=self.pid_signal) diff --git a/core/tests/test_whatsapp_reaction_and_recalc.py b/core/tests/test_whatsapp_reaction_and_recalc.py index 8767297..31e532a 100644 --- a/core/tests/test_whatsapp_reaction_and_recalc.py +++ b/core/tests/test_whatsapp_reaction_and_recalc.py @@ -7,7 +7,7 @@ from django.core.management import call_command from django.test import TestCase from core.clients.whatsapp import WhatsAppClient -from core.messaging import history +from core.messaging import history, media_bridge from core.models import ( ChatSession, ContactAvailabilityEvent, @@ -25,6 +25,16 @@ class _DummyXMPPClient: return None +class _DummyDownloadClient: + def __init__(self, payload: bytes): + self.payload = payload + self.calls = [] + + async def download_any(self, message): + self.calls.append(message) + return self.payload + + class _DummyUR: def __init__(self, loop): self.loop = loop @@ -122,6 +132,49 @@ class WhatsAppReactionHandlingTests(TestCase): self.assertEqual("offline", payload.get("presence")) self.assertTrue(int(payload.get("last_seen_ts") or 0) > 0) + def test_download_event_media_unwraps_device_sent_image(self): + downloader = _DummyDownloadClient(b"png-bytes") + self.client._client = downloader + event = { + "message": { + "deviceSentMessage": { + "message": { + "imageMessage": { + "caption": "wrapped image", + "mimetype": "image/png", + } + } + } + } + } + + attachments = async_to_sync(self.client._download_event_media)(event) + + self.assertEqual(1, len(attachments)) + self.assertEqual(1, len(downloader.calls)) + self.assertEqual( + {"imageMessage": {"caption": "wrapped image", "mimetype": "image/png"}}, + downloader.calls[0], + ) + blob = media_bridge.get_blob(attachments[0]["blob_key"]) + self.assertIsNotNone(blob) + self.assertEqual(b"png-bytes", blob["content"]) + self.assertEqual("image/png", blob["content_type"]) + + def test_message_text_unwraps_device_sent_caption(self): + text = self.client._message_text( + { + "deviceSentMessage": { + "message": { + "imageMessage": { + "caption": "caption from wrapper", + } + } + } + } + ) + self.assertEqual("caption from wrapper", text) + class RecalculateContactAvailabilityTests(TestCase): def setUp(self): diff --git a/core/tests/test_xmpp_approval_commands.py b/core/tests/test_xmpp_approval_commands.py index 47c2247..d72b821 100644 --- a/core/tests/test_xmpp_approval_commands.py +++ b/core/tests/test_xmpp_approval_commands.py @@ -5,7 +5,11 @@ from unittest.mock import MagicMock from asgiref.sync import async_to_sync from django.test import TestCase -from core.clients.xmpp import XMPPComponent +from core.gateway.builtin import ( + gateway_help_lines, + handle_approval_command, + handle_tasks_command, +) from core.models import ( CodexPermissionRequest, CodexRun, @@ -16,19 +20,6 @@ from core.models import ( ) -class _ApprovalProbe: - _resolve_request_provider = XMPPComponent._resolve_request_provider - _approval_event_prefix = XMPPComponent._approval_event_prefix - _APPROVAL_PROVIDER_COMMANDS = XMPPComponent._APPROVAL_PROVIDER_COMMANDS - _ACTION_TO_STATUS = XMPPComponent._ACTION_TO_STATUS - _apply_approval_decision = XMPPComponent._apply_approval_decision - _approval_list_pending = XMPPComponent._approval_list_pending - _approval_status = XMPPComponent._approval_status - _handle_approval_command = XMPPComponent._handle_approval_command - _gateway_help_lines = XMPPComponent._gateway_help_lines - _handle_tasks_command = XMPPComponent._handle_tasks_command - - class XMPPGatewayApprovalCommandTests(TestCase): def setUp(self): self.user = User.objects.create_user( @@ -43,7 +34,7 @@ class XMPPGatewayApprovalCommandTests(TestCase): epic=None, title="Approve me", source_service="xmpp", - source_channel="jews.zm.is", + source_channel="component.example.test", reference_code="77", status_snapshot="open", ) @@ -60,7 +51,7 @@ class XMPPGatewayApprovalCommandTests(TestCase): task=self.task, project=self.project, source_service="xmpp", - source_channel="jews.zm.is", + source_channel="component.example.test", status="waiting_approval", request_payload={ "action": "append_update", @@ -78,8 +69,7 @@ class XMPPGatewayApprovalCommandTests(TestCase): resume_payload={}, status="pending", ) - self.probe = _ApprovalProbe() - self.probe.log = MagicMock() + self.probe = MagicMock() def _run_command(self, text: str) -> list[str]: messages = [] @@ -87,11 +77,9 @@ class XMPPGatewayApprovalCommandTests(TestCase): def _sym(value): messages.append(str(value)) - handled = async_to_sync(XMPPComponent._handle_approval_command)( - self.probe, + handled = async_to_sync(handle_approval_command)( self.user, text, - "xmpp-approval-user@zm.is/mobile", _sym, ) self.assertTrue(handled) @@ -140,12 +128,11 @@ class XMPPGatewayTasksCommandTests(TestCase): epic=None, title="Ship CLI", source_service="xmpp", - source_channel="jews.zm.is", + source_channel="component.example.test", reference_code="12", status_snapshot="open", ) - self.probe = _ApprovalProbe() - self.probe.log = MagicMock() + self.probe = MagicMock() def _run_tasks(self, text: str) -> list[str]: messages = [] @@ -153,8 +140,7 @@ class XMPPGatewayTasksCommandTests(TestCase): def _sym(value): messages.append(str(value)) - handled = async_to_sync(XMPPComponent._handle_tasks_command)( - self.probe, + handled = async_to_sync(handle_tasks_command)( self.user, text, _sym, @@ -164,14 +150,18 @@ class XMPPGatewayTasksCommandTests(TestCase): return messages def test_help_contains_approval_and_tasks_sections(self): - lines = self.probe._gateway_help_lines() + lines = gateway_help_lines() text = "\n".join(lines) self.assertIn(".approval list-pending", text) self.assertIn(".tasks list", text) + self.assertIn(".tasks add", text) + self.assertIn(".l", text) def test_tasks_list_show_complete_and_undo(self): rows = self._run_tasks(".tasks list open 10") self.assertIn("#12", "\n".join(rows)) + rows = self._run_tasks(".l") + self.assertIn("#12", "\n".join(rows)) rows = self._run_tasks(".tasks show #12") self.assertIn("status: open", "\n".join(rows)) rows = self._run_tasks(".tasks complete #12") @@ -181,3 +171,24 @@ class XMPPGatewayTasksCommandTests(TestCase): rows = self._run_tasks(".tasks undo #12") self.assertIn("removed #12", "\n".join(rows)) self.assertFalse(DerivedTask.objects.filter(id=self.task.id).exists()) + + def test_tasks_add_creates_task_in_named_project(self): + rows = [] + handled = async_to_sync(handle_tasks_command)( + self.user, + ".tasks add Task Project :: Wire XMPP manual task create", + lambda value: rows.append(str(value)), + service="xmpp", + channel_identifier="component.example.test", + sender_identifier="operator@example.test", + ) + self.assertTrue(handled) + self.assertTrue(any("created #" in row.lower() for row in rows)) + created = DerivedTask.objects.filter( + user=self.user, + project=self.project, + title="Wire XMPP manual task create", + source_service="xmpp", + source_channel="component.example.test", + ).first() + self.assertIsNotNone(created) diff --git a/core/tests/test_xmpp_attachment_bridge.py b/core/tests/test_xmpp_attachment_bridge.py new file mode 100644 index 0000000..1edfb8d --- /dev/null +++ b/core/tests/test_xmpp_attachment_bridge.py @@ -0,0 +1,103 @@ +from unittest.mock import AsyncMock, patch + +from asgiref.sync import async_to_sync +from django.test import SimpleTestCase, TestCase, override_settings + +from core.clients import transport +from core.clients.xmpp import XMPPComponent, _resolve_person_from_xmpp_localpart +from core.models import Person, PersonIdentifier, User + + +class SignalAttachmentFetchTests(SimpleTestCase): + def test_signal_service_allows_direct_url_fetch(self): + response = AsyncMock() + response.status = 200 + response.headers = {"Content-Type": "image/png"} + response.read = AsyncMock(return_value=b"png-bytes") + + request_ctx = AsyncMock() + request_ctx.__aenter__.return_value = response + request_ctx.__aexit__.return_value = False + + session = AsyncMock() + session.get.return_value = request_ctx + + session_ctx = AsyncMock() + session_ctx.__aenter__.return_value = session + session_ctx.__aexit__.return_value = False + + with patch( + "core.clients.transport.aiohttp.ClientSession", + return_value=session_ctx, + ): + fetched = async_to_sync(transport.fetch_attachment)( + "signal", + { + "url": "https://example.com/file_share/demo.png", + "filename": "demo.png", + "content_type": "image/png", + }, + ) + + self.assertEqual(b"png-bytes", fetched["content"]) + self.assertEqual("image/png", fetched["content_type"]) + self.assertEqual("demo.png", fetched["filename"]) + self.assertEqual(9, fetched["size"]) + + +@override_settings( + XMPP_JID="component.example.test", + XMPP_USER_DOMAIN="example.test", +) +class XMPPContactJidTests(TestCase): + def _component(self): + return XMPPComponent( + ur=AsyncMock(), + jid="component.example.test", + secret="secret", + server="localhost", + port=5347, + ) + + def test_resolve_person_from_escaped_localpart(self): + user = User.objects.create_user(username="user", password="pw") + person = Person.objects.create(user=user, name="Misinformation Club") + + resolved = _resolve_person_from_xmpp_localpart( + user=user, + localpart_value=r"misinformation\20club", + ) + + self.assertEqual(person.id, resolved.id) + + def test_send_from_external_escapes_contact_jid(self): + user = User.objects.create_user(username="user2", password="pw") + person = Person.objects.create(user=user, name="Misinformation Club") + identifier = PersonIdentifier.objects.create( + user=user, + person=person, + service="signal", + identifier="group.example", + ) + component = self._component() + component.send_xmpp_message = AsyncMock(return_value="xmpp-id") + + with ( + patch("core.clients.xmpp.transport.record_bridge_mapping"), + patch("core.clients.xmpp.history.save_bridge_ref", new=AsyncMock()), + ): + async_to_sync(component.send_from_external)( + user, + identifier, + "hello", + False, + attachments=[], + source_ref={}, + ) + + component.send_xmpp_message.assert_awaited_once_with( + "user2@example.test", + r"misinformation\20club|signal@component.example.test", + "hello", + use_omemo_encryption=True, + ) diff --git a/core/tests/test_xmpp_carbons.py b/core/tests/test_xmpp_carbons.py new file mode 100644 index 0000000..253a9a2 --- /dev/null +++ b/core/tests/test_xmpp_carbons.py @@ -0,0 +1,165 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +from asgiref.sync import async_to_sync +from django.test import TestCase, override_settings +from slixmpp.plugins.xep_0356.permissions import MessagePermission + +from core.clients.xmpp import XMPPComponent +from core.models import Person, PersonIdentifier, User + + +@override_settings( + XMPP_JID="component.example.test", + XMPP_USER_DOMAIN="example.test", +) +class XMPPCarbonTests(TestCase): + def _component(self): + component = XMPPComponent( + ur=MagicMock(), + jid="component.example.test", + secret="secret", + server="localhost", + port=5347, + ) + component.log = MagicMock() + return component + + def test_build_privileged_outbound_message_targets_contact(self): + component = self._component() + + msg = component._build_privileged_outbound_message( + user_jid="user@example.test", + contact_jid="contact|signal@component.example.test", + body_text="hello from signal", + attachment_url="https://files.example.test/demo.png", + ) + + self.assertEqual("user@example.test", str(msg["from"])) + self.assertEqual("contact|signal@component.example.test", str(msg["to"])) + body = next( + (child for child in msg.xml if str(child.tag).endswith("body")), + None, + ) + self.assertIsNotNone(body) + self.assertEqual("hello from signal", body.text) + oob = msg.xml.find(".//{jabber:x:oob}url") + self.assertIsNotNone(oob) + self.assertEqual("https://files.example.test/demo.png", oob.text) + + def test_send_sent_carbon_copy_requires_outgoing_privilege(self): + component = self._component() + plugin = SimpleNamespace( + granted_privileges={"example.test": SimpleNamespace(message="none")}, + send_privileged_message=MagicMock(), + _make_privileged_message=MagicMock(), + ) + with ( + patch.object(component.plugin, "get", return_value=plugin), + patch.object(component, "_user_xmpp_domain", return_value="other.example.test"), + ): + sent = async_to_sync(component.send_sent_carbon_copy)( + user_jid="user@example.test", + contact_jid="contact|signal@component.example.test", + body_text="hello", + ) + + self.assertFalse(sent) + plugin.send_privileged_message.assert_not_called() + plugin._make_privileged_message.assert_not_called() + + def test_send_sent_carbon_copy_sends_privileged_message_when_allowed(self): + component = self._component() + plugin = SimpleNamespace( + granted_privileges={ + "example.test": SimpleNamespace(message=MessagePermission.OUTGOING) + }, + send_privileged_message=MagicMock(), + ) + with patch.object(component.plugin, "get", return_value=plugin): + sent = async_to_sync(component.send_sent_carbon_copy)( + user_jid="user@example.test", + contact_jid="contact|signal@component.example.test", + body_text="hello", + ) + + self.assertTrue(sent) + plugin.send_privileged_message.assert_called_once() + sent_message = plugin.send_privileged_message.call_args.args[0] + self.assertEqual("contact|signal@component.example.test", str(sent_message["to"])) + self.assertEqual("user@example.test", str(sent_message["from"])) + self.assertIsNotNone( + next( + (child for child in sent_message.xml if str(child.tag).endswith("body")), + None, + ) + ) + + def test_send_sent_carbon_copy_uses_configured_domain_without_advertisement(self): + component = self._component() + wrapped = MagicMock() + plugin = SimpleNamespace( + granted_privileges={}, + send_privileged_message=MagicMock(), + _make_privileged_message=MagicMock(return_value=wrapped), + ) + with patch.object(component.plugin, "get", return_value=plugin): + sent = async_to_sync(component.send_sent_carbon_copy)( + user_jid="user@example.test", + contact_jid="contact|signal@component.example.test", + body_text="hello", + ) + + self.assertTrue(sent) + plugin.send_privileged_message.assert_not_called() + plugin._make_privileged_message.assert_called_once() + wrapped.send.assert_called_once() + + def test_outgoing_relay_keeps_you_prefix_while_attempting_carbon_copy(self): + component = self._component() + user = User.objects.create_user(username="user", password="pw") + person = Person.objects.create(user=user, name="Contact") + identifier = PersonIdentifier.objects.create( + user=user, + person=person, + service="signal", + identifier="+15550000000", + ) + call_order = [] + + async def send_xmpp_message(*args, **kwargs): + call_order.append("fallback") + return "xmpp-message-id" + + async def send_sent_carbon_copy(*args, **kwargs): + call_order.append("carbon") + return True + + component.send_sent_carbon_copy = AsyncMock(side_effect=send_sent_carbon_copy) + component.send_xmpp_message = AsyncMock(side_effect=send_xmpp_message) + + with ( + patch("core.clients.xmpp.transport.record_bridge_mapping"), + patch("core.clients.xmpp.history.save_bridge_ref", new=AsyncMock()), + ): + async_to_sync(component.send_from_external)( + user, + identifier, + "hello", + True, + attachments=[], + source_ref={}, + ) + + component.send_sent_carbon_copy.assert_awaited_once_with( + user_jid="user@example.test", + contact_jid="contact|signal@component.example.test", + body_text="hello", + ) + component.send_xmpp_message.assert_awaited_once_with( + "user@example.test", + "contact|signal@component.example.test", + "YOU: hello", + use_omemo_encryption=True, + ) + self.assertEqual(["fallback", "carbon"], call_order) diff --git a/core/tests/test_xmpp_integration.py b/core/tests/test_xmpp_integration.py index 13e2f37..66b28b6 100644 --- a/core/tests/test_xmpp_integration.py +++ b/core/tests/test_xmpp_integration.py @@ -54,12 +54,12 @@ def _xmpp_c2s_port() -> int: def _xmpp_domain() -> str: - """The VirtualHost domain (zm.is), derived from XMPP_JID or XMPP_DOMAIN.""" + """The VirtualHost domain, derived from XMPP_JID or XMPP_DOMAIN.""" domain = getattr(settings, "XMPP_DOMAIN", None) if domain: return str(domain) jid = str(settings.XMPP_JID) - # Component JID is like "jews.zm.is" → parent domain is "zm.is" + # Component JIDs may be subdomains; derive the parent domain when needed. parts = jid.split(".") if len(parts) > 2: return ".".join(parts[1:]) @@ -389,7 +389,10 @@ class XMPPAuthBridgeTests(SimpleTestCase): """Auth bridge returns 0 (or error) for a request with a wrong XMPP_SECRET.""" _, host, port, path = self._parse_endpoint() # isuser command with wrong secret — should be rejected or return 0 - query = "?command=isuser%3Anonexistent%3Azm.is&secret=wrongsecret" + query = ( + f"?command=isuser%3Anonexistent%3A{urllib.parse.quote(_xmpp_domain())}" + "&secret=wrongsecret" + ) try: conn = http.client.HTTPConnection(host, port, timeout=5) conn.request("GET", path + query) @@ -410,7 +413,8 @@ class XMPPAuthBridgeTests(SimpleTestCase): secret = getattr(settings, "XMPP_SECRET", "") _, host, port, path = self._parse_endpoint() query = ( - f"?command=isuser%3Anonexistent%3Azm.is&secret={urllib.parse.quote(secret)}" + f"?command=isuser%3Anonexistent%3A{urllib.parse.quote(_xmpp_domain())}" + f"&secret={urllib.parse.quote(secret)}" ) try: conn = http.client.HTTPConnection(host, port, timeout=5) diff --git a/core/tests/test_xmpp_omemo_support.py b/core/tests/test_xmpp_omemo_support.py index ea9508b..f035c7f 100644 --- a/core/tests/test_xmpp_omemo_support.py +++ b/core/tests/test_xmpp_omemo_support.py @@ -9,7 +9,7 @@ from core.models import User, UserXmppOmemoState @override_settings( - XMPP_JID="jews.zm.is", + XMPP_JID="component.example.test", XMPP_SECRET="secret", XMPP_ADDRESS="127.0.0.1", XMPP_PORT=8888, @@ -51,14 +51,14 @@ class XMPPOmemoObservationPersistenceTests(TestCase): async_to_sync(XMPPComponent._record_sender_omemo_state)( xmpp_component, user, - sender_jid="xmpp-omemo-user@zm.is/mobile", - recipient_jid="jews.zm.is", + sender_jid="xmpp-omemo-user@example.test/mobile", + recipient_jid="component.example.test", message_stanza=SimpleNamespace(xml=stanza_xml), ) row = UserXmppOmemoState.objects.get(user=user) self.assertEqual("detected", row.status) self.assertEqual("sid:321,rid:654", row.latest_client_key) - self.assertEqual("jews.zm.is", row.last_target_jid) + self.assertEqual("component.example.test", row.last_target_jid) class XMPPOmemoEnforcementTests(TestCase): @@ -78,7 +78,7 @@ class XMPPOmemoEnforcementTests(TestCase): # Create a plaintext message stanza (no OMEMO encryption) stanza_xml = ET.fromstring( - "<message from='sender@example.com' to='jews.zm.is'>" + "<message from='sender@example.com' to='component.example.test'>" "<body>Hello, world!</body>" "</message>" ) @@ -125,7 +125,7 @@ class XMPPOmemoEnforcementTests(TestCase): # Create an OMEMO-encrypted message stanza stanza_xml = ET.fromstring( - "<message from='sender@example.com' to='jews.zm.is'>" + "<message from='sender@example.com' to='component.example.test'>" "<encrypted xmlns='eu.siacs.conversations.axolotl'>" "<header sid='77'><key rid='88'>x</key></header>" "</encrypted>" @@ -167,14 +167,14 @@ class XMPPOmemoDeviceDiscoveryTests(TestCase): # Create a mock XMPP component self.mock_component = MagicMock() self.mock_component.log = MagicMock() - self.mock_component.jid = "jews.zm.is" + self.mock_component.jid = "component.example.test" def test_gateway_publishes_device_list_to_pubsub(self): """Test that the gateway publishes its device list to PubSub (XEP-0060). This simulates the device discovery query that real XMPP clients perform. When a client wants to send an OMEMO message, it: - 1. Queries the PubSub node: pubsub.example.com/eu.siacs.conversations.axolotl/devices/jews.zm.is + 1. Queries the PubSub node: pubsub.example.com/eu.siacs.conversations.axolotl/devices/component.example.test 2. Expects to receive a device list with at least one device 3. Retrieves keys for those devices 4. Encrypts the message @@ -261,7 +261,7 @@ class XMPPOmemoDeviceDiscoveryTests(TestCase): """ # Simulate an OMEMO-encrypted message from a client device client_stanza = ET.fromstring( - "<message from='testuser@example.com/mobile' to='jews.zm.is'>" + "<message from='testuser@example.com/mobile' to='component.example.test'>" "<encrypted xmlns='eu.siacs.conversations.axolotl'>" "<header sid='12345' schemeVersion='2'>" # Device 12345 "<key rid='67890'>encrypted_payload_1</key>" # To recipient device 67890 @@ -289,7 +289,7 @@ class XMPPOmemoDeviceDiscoveryTests(TestCase): The OMEMO bootstrap must: 1. Initialize the session manager (which auto-creates devices) - 2. Publish device list to PubSub at: eu.siacs.conversations.axolotl/devices/jews.zm.is + 2. Publish device list to PubSub at: eu.siacs.conversations.axolotl/devices/component.example.test 3. Allow clients to discover and query those devices If PubSub is slow or unavailable, this times out and prevents @@ -309,15 +309,15 @@ class XMPPOmemoDeviceDiscoveryTests(TestCase): def test_component_jid_device_discovery(self): """Test that component JIDs (without user@) can publish OMEMO devices. - A key issue with components: they use JIDs like 'jews.zm.is' instead of - 'user@jews.zm.is'. This affects: - 1. Device list node path: eu.siacs.conversations.axolotl/devices/jews.zm.is + A key issue with components: they use JIDs like 'component.example.test' instead of + 'user@component.example.test'. This affects: + 1. Device list node path: eu.siacs.conversations.axolotl/devices/component.example.test 2. Device identity and trust establishment 3. How clients discover and encrypt to the component The OMEMO plugin must handle component JIDs correctly. """ - component_jid = "jews.zm.is" + component_jid = "component.example.test" # Component JID format (no user@ part) self.assertNotIn("@", component_jid) @@ -325,21 +325,21 @@ class XMPPOmemoDeviceDiscoveryTests(TestCase): # But PubSub device node still follows standard format pubsub_node = f"eu.siacs.conversations.axolotl/devices/{component_jid}" self.assertEqual( - "eu.siacs.conversations.axolotl/devices/jews.zm.is", pubsub_node + "eu.siacs.conversations.axolotl/devices/component.example.test", pubsub_node ) def test_gateway_accepts_presence_subscription_for_omemo(self): """Test that gateway auto-accepts presence subscriptions for OMEMO device discovery. - When a client subscribes to the gateway component (jews.zm.is) for OMEMO: - 1. Client sends: <presence type="subscribe" from="user@example.com" to="jews.zm.is"/> + When a client subscribes to the gateway component JID for OMEMO: + 1. Client sends: <presence type="subscribe" from="user@example.com" to="component.example.test"/> 2. Gateway should auto-accept and send presence availability 3. This allows the client to add the gateway to its roster 4. Client can then query PubSub for device lists """ # Simulate a client sending presence subscription to gateway client_jid = "testclient@example.com" - gateway_jid = "jews.zm.is" + gateway_jid = "component.example.test" # Create a mock XMPP component with the subscription handler mock_component = MagicMock() diff --git a/core/views/automation.py b/core/views/automation.py index 7981f24..b72de65 100644 --- a/core/views/automation.py +++ b/core/views/automation.py @@ -143,6 +143,7 @@ class CommandRoutingSettings(LoginRequiredMixin, View): "command_choices": ( ("bp", "Business Plan (bp)"), ("codex", "Codex (codex)"), + ("claude", "Claude (claude)"), ), "scope_service": scope_service, "scope_identifier": scope_identifier, @@ -165,21 +166,27 @@ class CommandRoutingSettings(LoginRequiredMixin, View): .lower() or "bp" ) + default_name = { + "bp": "Business Plan", + "codex": "Codex", + "claude": "Claude", + }.get(slug, "Business Plan") + default_trigger = { + "bp": ".bp", + "codex": ".codex", + "claude": ".claude", + }.get(slug, ".bp") profile, _ = CommandProfile.objects.get_or_create( user=request.user, slug=slug, defaults={ - "name": str( - request.POST.get("name") - or ("Codex" if slug == "codex" else "Business Plan") - ).strip() - or ("Codex" if slug == "codex" else "Business Plan"), + "name": str(request.POST.get("name") or default_name).strip() + or default_name, "enabled": True, "trigger_token": str( - request.POST.get("trigger_token") - or (".codex" if slug == "codex" else ".bp") + request.POST.get("trigger_token") or default_trigger ).strip() - or (".codex" if slug == "codex" else ".bp"), + or default_trigger, "template_text": str(request.POST.get("template_text") or ""), }, ) @@ -195,6 +202,10 @@ class CommandRoutingSettings(LoginRequiredMixin, View): profile.trigger_token = ".codex" profile.reply_required = False profile.exact_match_only = False + if slug == "claude": + profile.trigger_token = ".claude" + profile.reply_required = False + profile.exact_match_only = False profile.save( update_fields=[ "name", diff --git a/core/views/compose.py b/core/views/compose.py index 0dd1b27..6edf64a 100644 --- a/core/views/compose.py +++ b/core/views/compose.py @@ -7,7 +7,7 @@ import time from datetime import datetime from datetime import timezone as dt_timezone from difflib import SequenceMatcher -from urllib.parse import quote_plus, urlencode, urlparse +from urllib.parse import urlencode, urlparse from asgiref.sync import async_to_sync from django.conf import settings @@ -136,6 +136,13 @@ def _identifier_variants(service: str, identifier: str) -> list[str]: return variants +def _group_channel_identifier(service: str, group_link: PlatformChatLink, bare_id: str) -> str: + service_key = _default_service(service) + if service_key == "whatsapp": + return str(group_link.chat_jid or f"{bare_id}@g.us").strip() + return bare_id + + def _safe_limit(raw) -> int: try: value = int(raw or 40) @@ -424,7 +431,7 @@ def _extract_attachment_image_urls(blob) -> list[str]: direct_urls.append(normalized) urls.extend(direct_urls) blob_key = str(blob.get("blob_key") or "").strip() - # Prefer source-hosted URLs (for example share.zm.is) and use blob fallback only + # Prefer source-hosted URLs and use blob fallback only # when no usable direct URL exists. if blob_key and image_hint and not direct_urls: urls.append(f"/compose/media/blob/?key={quote_plus(blob_key)}") @@ -1783,6 +1790,11 @@ def _context_base(user, service, identifier, person): service=service, identifier__in=identifier_variants or [identifier], ).first() + if person_identifier is None and identifier and person is None: + person_identifier = PersonIdentifier.objects.filter( + user=user, + identifier__in=identifier_variants or [identifier], + ).first() if person_identifier: service = person_identifier.service @@ -1811,7 +1823,7 @@ def _context_base(user, service, identifier, person): return { "person_identifier": None, "service": service, - "identifier": f"{bare_id}@g.us", + "identifier": _group_channel_identifier(service, group_link, bare_id), "person": None, "is_group": True, "group_name": group_link.chat_name or bare_id, @@ -2426,6 +2438,63 @@ def _signal_identifier_shape(value: str) -> str: return "other" +def _preferred_signal_identifier(identifiers: list[str], *, is_group: bool) -> str: + cleaned = [] + for value in identifiers: + candidate = str(value or "").strip() + if candidate and candidate not in cleaned: + cleaned.append(candidate) + if not cleaned: + return "" + if is_group: + for candidate in cleaned: + if candidate.startswith("group."): + return candidate + return cleaned[0] + for candidate in cleaned: + if _signal_identifier_shape(candidate) == "phone": + return candidate + for candidate in cleaned: + if _signal_identifier_shape(candidate) == "uuid": + return candidate + return cleaned[0] + + +def _signal_runtime_alias_map() -> dict[str, set[str]]: + state = transport.get_runtime_state("signal") or {} + alias_map: dict[str, set[str]] = {} + for bucket_name in ("contacts", "groups"): + rows = state.get(bucket_name) or [] + if not isinstance(rows, list): + continue + for item in rows: + if not isinstance(item, dict): + continue + identifiers = [] + candidates = item.get("identifiers") + if isinstance(candidates, list): + identifiers.extend(candidates) + identifiers.extend( + [ + item.get("identifier"), + item.get("number"), + item.get("uuid"), + item.get("id"), + item.get("internal_id"), + ] + ) + unique = [] + for value in identifiers: + candidate = str(value or "").strip() + if candidate and candidate not in unique: + unique.append(candidate) + if len(unique) < 2: + continue + for identifier in unique: + alias_map[identifier] = {value for value in unique if value != identifier} + return alias_map + + def _manual_contact_rows(user): rows = [] seen = set() @@ -2493,6 +2562,8 @@ def _manual_contact_rows(user): ) for row in identifiers: + if _default_service(row.service) == "signal": + continue add_row( service=row.service, identifier=row.identifier, @@ -2500,6 +2571,27 @@ def _manual_contact_rows(user): source="linked", ) + group_links = ( + PlatformChatLink.objects.filter(user=user, is_group=True) + .order_by("service", "chat_name", "chat_identifier") + ) + for link in group_links: + if _default_service(link.service) == "signal": + continue + group_identifier = _group_channel_identifier( + str(link.service or "").strip(), + link, + str(link.chat_identifier or "").strip(), + ) + if not group_identifier: + continue + add_row( + service=link.service, + identifier=group_identifier, + source=f"{_default_service(link.service)}_group", + detected_name=str(link.chat_name or "").strip(), + ) + signal_links = { str(row.identifier): row for row in ( @@ -2508,6 +2600,163 @@ def _manual_contact_rows(user): .order_by("id") ) } + signal_state = transport.get_runtime_state("signal") or {} + signal_accounts = [ + str(value or "").strip() + for value in (signal_state.get("accounts") or []) + if str(value or "").strip() + ] + signal_account_set = set(signal_accounts) + signal_entities = {} + signal_alias_index = {} + + def _signal_entity_key(identifiers_list: list[str], *, is_group: bool) -> str: + preferred = _preferred_signal_identifier(identifiers_list, is_group=is_group) + if is_group: + return f"group:{preferred}" + for candidate in identifiers_list: + if _signal_identifier_shape(candidate) == "uuid": + return f"contact:uuid:{candidate.lower()}" + for candidate in identifiers_list: + if _signal_identifier_shape(candidate) == "phone": + return f"contact:phone:{candidate}" + return f"contact:other:{preferred.lower()}" + + def _resolve_signal_entity_key(candidate: str) -> str: + cleaned = str(candidate or "").strip() + if not cleaned: + return "" + for variant in _identifier_variants("signal", cleaned): + entity_key = signal_alias_index.get(variant) + if entity_key: + return entity_key + return "" + + def _register_signal_entity( + *, + identifiers_list, + is_group: bool, + detected_name="", + person=None, + source="signal_runtime", + ): + unique_identifiers = [] + for value in identifiers_list or []: + cleaned = str(value or "").strip() + if ( + not cleaned + or cleaned in unique_identifiers + or cleaned in signal_account_set + ): + continue + unique_identifiers.append(cleaned) + if not unique_identifiers: + return None + entity_key = "" + for identifier in unique_identifiers: + entity_key = _resolve_signal_entity_key(identifier) + if entity_key: + break + if not entity_key: + entity_key = _signal_entity_key(unique_identifiers, is_group=is_group) + entity = signal_entities.get(entity_key) + if entity is None: + entity = { + "is_group": bool(is_group), + "identifiers": [], + "detected_name": _clean_detected_name(detected_name or ""), + "person": person, + "sources": set(), + } + signal_entities[entity_key] = entity + for identifier in unique_identifiers: + if identifier not in entity["identifiers"]: + entity["identifiers"].append(identifier) + for variant in _identifier_variants("signal", identifier): + signal_alias_index[variant] = entity_key + cleaned_name = _clean_detected_name(detected_name or "") + if cleaned_name and not entity["detected_name"]: + entity["detected_name"] = cleaned_name + if person is not None and entity.get("person") is None: + entity["person"] = person + entity["sources"].add(str(source or "").strip() or "signal_runtime") + return entity + + for row in signal_links.values(): + _register_signal_entity( + identifiers_list=[row.identifier], + is_group=False, + detected_name=(str(row.person.name or "").strip() if row.person else ""), + person=row.person, + source="linked", + ) + + signal_group_links = ( + PlatformChatLink.objects.filter(user=user, service="signal", is_group=True) + .order_by("chat_name", "chat_identifier") + ) + for link in signal_group_links: + group_identifier = _group_channel_identifier( + "signal", + link, + str(link.chat_identifier or "").strip(), + ) + if not group_identifier: + continue + _register_signal_entity( + identifiers_list=[group_identifier], + is_group=True, + detected_name=str(link.chat_name or "").strip(), + source="signal_group", + ) + + signal_contacts = signal_state.get("contacts") or [] + if isinstance(signal_contacts, list): + for item in signal_contacts: + if not isinstance(item, dict): + continue + candidate_identifiers = item.get("identifiers") + if not isinstance(candidate_identifiers, list): + candidate_identifiers = [ + item.get("identifier"), + item.get("number"), + item.get("uuid"), + ] + linked = None + for candidate in candidate_identifiers: + cleaned = str(candidate or "").strip() + if not cleaned: + continue + linked = signal_links.get(cleaned) + if linked is not None: + break + _register_signal_entity( + identifiers_list=candidate_identifiers, + is_group=False, + detected_name=str(item.get("name") or "").strip(), + person=(linked.person if linked else None), + source="signal_runtime", + ) + + signal_groups = signal_state.get("groups") or [] + if isinstance(signal_groups, list): + for item in signal_groups: + if not isinstance(item, dict): + continue + candidate_identifiers = item.get("identifiers") + if not isinstance(candidate_identifiers, list): + candidate_identifiers = [ + item.get("identifier"), + item.get("id"), + item.get("internal_id"), + ] + _register_signal_entity( + identifiers_list=candidate_identifiers, + is_group=True, + detected_name=str(item.get("name") or "").strip(), + source="signal_group_raw", + ) + signal_chats = Chat.objects.all().order_by("-id")[:500] for chat in signal_chats: uuid_candidate = str(chat.source_uuid or "").strip() @@ -2517,20 +2766,45 @@ def _manual_contact_rows(user): fallback_linked = signal_links.get(uuid_candidate) if fallback_linked is None and number_candidate: fallback_linked = signal_links.get(number_candidate) - for candidate in (uuid_candidate, number_candidate): - if not candidate: - continue - linked = signal_links.get(candidate) or fallback_linked - add_row( - service="signal", - identifier=candidate, - person=(linked.person if linked else None), - source="signal_chat", - account=str(chat.account or ""), - detected_name=_clean_detected_name( - chat.source_name or chat.account or "" - ), - ) + linked = fallback_linked + if linked is None: + for candidate in (uuid_candidate, number_candidate): + linked = signal_links.get(candidate) + if linked is not None: + break + _register_signal_entity( + identifiers_list=[uuid_candidate, number_candidate], + is_group=False, + detected_name=_clean_detected_name(chat.source_name or chat.account or ""), + person=(linked.person if linked else None), + source="signal_chat", + ) + + for entity in signal_entities.values(): + entity_identifiers = list(entity.get("identifiers") or []) + identifier_value = _preferred_signal_identifier( + entity_identifiers, + is_group=bool(entity.get("is_group")), + ) + if not identifier_value: + continue + add_row( + service="signal", + identifier=identifier_value, + person=entity.get("person"), + source=",".join(sorted(entity.get("sources") or {"signal_runtime"})), + account=entity.get("detected_name") or "", + detected_name=entity.get("detected_name") or "", + ) + if rows: + rows[-1]["identifier_aliases"] = [ + candidate + for candidate in entity_identifiers + if str(candidate or "").strip() and candidate != identifier_value + ] + rows[-1]["identifier_search"] = " ".join( + [rows[-1]["identifier"]] + rows[-1]["identifier_aliases"] + ).strip() whatsapp_links = { str(row.identifier): row @@ -3225,8 +3499,11 @@ class ComposeContactMatch(LoginRequiredMixin, View): value = str(identifier or "").strip() if not value: return set() - source_shape = _signal_identifier_shape(value) companions = set() + runtime_aliases = _signal_runtime_alias_map() + for variant in _identifier_variants("signal", value): + companions.update(runtime_aliases.get(variant) or set()) + source_shape = _signal_identifier_shape(value) signal_rows = Chat.objects.filter(source_uuid=value) | Chat.objects.filter( source_number=value ) diff --git a/core/views/signal.py b/core/views/signal.py index 24cb45f..40c30fc 100644 --- a/core/views/signal.py +++ b/core/views/signal.py @@ -5,6 +5,7 @@ import requests from django.conf import settings from django.contrib import messages from django.db.models import Q +from django.http import HttpResponse from django.shortcuts import render from django.urls import reverse from django.views import View @@ -141,7 +142,12 @@ class SignalAccountUnlink(SuperUserRequiredMixin, View): else: messages.error( request, - "Signal relink failed to clear current device state. Try relink again.", + ( + "Signal relink could not verify that the current device state was " + "cleared. The account may still be linked in signal-cli-rest-api. " + "Try relink again, and if it still fails, restart the Signal service " + "before requesting a new QR code." + ), ) else: messages.warning(request, "No Signal account selected to relink.") @@ -324,15 +330,16 @@ class SignalChatsList(SuperUserRequiredMixin, ObjectList): group_id = str(link.chat_identifier or "").strip() if not group_id: continue + query = urlencode({"service": "signal", "identifier": group_id}) rows.append( { "chat": None, - "compose_page_url": "", - "compose_widget_url": "", + "compose_page_url": f"{reverse('compose_page')}?{query}", + "compose_widget_url": f"{reverse('compose_widget')}?{query}", "ai_url": reverse("ai_workspace"), "person_name": "", "manual_icon_class": "fa-solid fa-users", - "can_compose": False, + "can_compose": True, "match_url": "", "is_group": True, "name": link.chat_name or group_id, @@ -412,7 +419,21 @@ class SignalAccountAdd(SuperUserRequiredMixin, CustomObjectRead): def get_object(self, **kwargs): form_args = self.request.POST.dict() device_name = form_args["device"] - image_bytes = transport.get_link_qr(self.service, device_name) + try: + image_bytes = transport.get_link_qr(self.service, device_name) + except Exception as exc: + return render( + self.request, + "mixins/wm/modal.html", + { + "window_content": "mixins/partials/notify.html", + "message": ( + "Signal QR link is unavailable right now. " + f"signal-cli-rest-api did not return a QR in time: {exc}" + ), + "class": "danger", + }, + ) base64_image = transport.image_bytes_to_base64(image_bytes) return base64_image diff --git a/core/views/system.py b/core/views/system.py index a19d5cd..1e65025 100644 --- a/core/views/system.py +++ b/core/views/system.py @@ -40,6 +40,14 @@ from core.models import ( WorkspaceConversation, WorkspaceMetricSnapshot, ) +from core.security.capabilities import ( + CAPABILITY_SCOPES, +) +from core.security.capabilities import GLOBAL_SCOPE_KEY as COMMAND_GLOBAL_SCOPE_KEY +from core.security.capabilities import GROUP_LABELS as CAPABILITY_GROUP_LABELS +from core.security.capabilities import ( + scope_record, +) from core.transports.capabilities import capability_snapshot from core.views.manage.permissions import SuperUserRequiredMixin @@ -528,7 +536,7 @@ class SecurityPage(LoginRequiredMixin, View): template_name = "pages/security.html" page_mode = "encryption" - GLOBAL_SCOPE_KEY = "global.override" + GLOBAL_SCOPE_KEY = COMMAND_GLOBAL_SCOPE_KEY # Allowed Services list used by both Global Scope Override and local scopes. # Keep this in sync with the UI text on the Security page. POLICY_SERVICES = ["xmpp", "whatsapp", "signal", "instagram", "web"] @@ -541,47 +549,7 @@ class SecurityPage(LoginRequiredMixin, View): "require_omemo", "require_trusted_fingerprint", ) - POLICY_SCOPES = [ - ( - "gateway.tasks", - "Gateway .tasks commands", - "Handles .tasks list/show/complete/undo over gateway channels.", - ), - ( - "gateway.approval", - "Gateway approval commands", - "Handles .approval/.codex/.claude approve/deny over gateway channels.", - ), - ( - "gateway.totp", - "Gateway TOTP enrollment", - "Controls TOTP enrollment/status commands over gateway channels.", - ), - ( - "tasks.submit", - "Task submissions from chat", - "Controls automatic task creation from inbound messages.", - ), - ( - "tasks.commands", - "Task command verbs (.task/.undo/.epic)", - "Controls explicit task command verbs.", - ), - ( - "command.bp", - "Business plan command", - "Controls Business Plan command execution.", - ), - ("command.codex", "Codex command", "Controls Codex command execution."), - ("command.claude", "Claude command", "Controls Claude command execution."), - ] - POLICY_GROUP_LABELS = { - "gateway": "Gateway", - "tasks": "Tasks", - "command": "Commands", - "agentic": "Agentic", - "other": "Other", - } + POLICY_GROUP_LABELS = CAPABILITY_GROUP_LABELS def _show_encryption(self) -> bool: return str(getattr(self, "page_mode", "encryption")).strip().lower() in { @@ -774,8 +742,10 @@ class SecurityPage(LoginRequiredMixin, View): ) } payload = [] - for scope_key, label, description in self.POLICY_SCOPES: - key = str(scope_key or "").strip().lower() + for scope in CAPABILITY_SCOPES: + if not bool(scope.configurable): + continue + key = str(scope.key or "").strip().lower() item = rows.get(key) raw_allowed_services = [ str(value or "").strip().lower() @@ -797,8 +767,8 @@ class SecurityPage(LoginRequiredMixin, View): payload.append( { "scope_key": key, - "label": label, - "description": description, + "label": scope.label, + "description": scope.description, "enabled": self._apply_global_override( bool(getattr(item, "enabled", True)), global_overrides["scope_enabled"], @@ -827,38 +797,20 @@ class SecurityPage(LoginRequiredMixin, View): return payload def _scope_group_key(self, scope_key: str) -> str: - key = str(scope_key or "").strip().lower() - if key in {"tasks.commands", "gateway.tasks"}: - return "tasks" - if key in {"command.codex", "command.claude"}: - return "agentic" - if key.startswith("gateway."): - return "command" - if key.startswith("tasks."): - if key == "tasks.submit": - return "tasks" - return "command" - if key.startswith("command."): - return "command" - if ".commands" in key: - return "command" - if ".approval" in key: - return "command" - if ".totp" in key: - return "command" - if ".task" in key: - return "tasks" - return "other" + row = scope_record(scope_key) + return row.group if row is not None else "other" def _grouped_scope_rows(self, request): rows = self._scope_rows(request) - grouped: dict[str, list[dict]] = {key: [] for key in self.POLICY_GROUP_LABELS} + grouped: dict[str, list[dict]] = { + key: [] for key in self.POLICY_GROUP_LABELS.keys() + } for row in rows: group_key = self._scope_group_key(row.get("scope_key")) grouped.setdefault(group_key, []) grouped[group_key].append(row) payload = [] - for group_key in ("tasks", "command", "agentic", "other"): + for group_key in ("gateway", "tasks", "command", "agentic", "other"): items = grouped.get(group_key) or [] if not items: continue @@ -875,6 +827,10 @@ class SecurityPage(LoginRequiredMixin, View): row = self._security_settings(request) if str(request.POST.get("encryption_settings_submit") or "").strip() == "1": row.require_omemo = _to_bool(request.POST.get("require_omemo"), False) + row.encrypt_component_messages_with_omemo = _to_bool( + request.POST.get("encrypt_component_messages_with_omemo"), + True, + ) row.encrypt_contact_messages_with_omemo = _to_bool( request.POST.get("encrypt_contact_messages_with_omemo"), False, @@ -882,6 +838,7 @@ class SecurityPage(LoginRequiredMixin, View): row.save( update_fields=[ "require_omemo", + "encrypt_component_messages_with_omemo", "encrypt_contact_messages_with_omemo", "updated_at", ] diff --git a/core/views/tasks.py b/core/views/tasks.py index bea0db8..4fc8dce 100644 --- a/core/views/tasks.py +++ b/core/views/tasks.py @@ -1,5 +1,6 @@ from __future__ import annotations +import datetime import hashlib import json from urllib.parse import urlencode @@ -40,6 +41,7 @@ from core.tasks.chat_defaults import ( ) from core.tasks.codex_approval import queue_codex_event_with_pre_approval from core.tasks.codex_support import resolve_external_chat_id +from core.tasks.engine import create_task_record_and_sync from core.tasks.providers import get_provider @@ -828,6 +830,9 @@ class TasksHub(LoginRequiredMixin, View): return { "projects": projects, "project_choices": all_projects, + "epic_choices": TaskEpic.objects.filter( + project__user=request.user + ).select_related("project").order_by("project__name", "name"), "tasks": tasks, "scope": scope, "person_identifier_rows": person_identifier_rows, @@ -875,6 +880,60 @@ class TasksHub(LoginRequiredMixin, View): return redirect(f"{reverse('tasks_hub')}?{urlencode(query)}") return redirect("tasks_hub") + if action == "task_create": + project = get_object_or_404( + TaskProject, + user=request.user, + id=request.POST.get("project_id"), + ) + epic = None + epic_id = str(request.POST.get("epic_id") or "").strip() + if epic_id: + epic = get_object_or_404(TaskEpic, id=epic_id, project=project) + title = str(request.POST.get("title") or "").strip() + if not title: + messages.error(request, "Task title is required.") + return redirect("tasks_hub") + scope = self._scope(request) + source_service = str(scope.get("service") or "").strip().lower() or "web" + source_channel = str(scope.get("identifier") or "").strip() + due_raw = str(request.POST.get("due_date") or "").strip() + due_date = None + if due_raw: + try: + due_date = datetime.date.fromisoformat(due_raw) + except Exception: + messages.error(request, "Due date must be YYYY-MM-DD.") + return redirect("tasks_hub") + task, _event = async_to_sync(create_task_record_and_sync)( + user=request.user, + project=project, + epic=epic, + title=title, + source_service=source_service, + source_channel=source_channel, + actor_identifier=str(request.user.username or request.user.id), + due_date=due_date, + assignee_identifier=str( + request.POST.get("assignee_identifier") or "" + ).strip(), + immutable_payload={ + "source": "tasks_hub_manual_create", + "person_id": scope["person_id"], + "service": source_service, + "identifier": source_channel, + }, + event_payload={ + "source": "tasks_hub_manual_create", + "via": "web_ui", + }, + ) + messages.success( + request, + f"Created task #{task.reference_code} in '{project.name}'.", + ) + return redirect("tasks_task", task_id=str(task.id)) + if action == "project_map_identifier": project = get_object_or_404( TaskProject, diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index f710f11..a241092 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -20,13 +20,6 @@ reload-on-as=512 reload-on-rss=256 vacuum=1 home=/venv -processes=4 +processes=2 threads=2 -log-level=debug - -# Autoreload on code changes (graceful reload) -py-autoreload=1 -# In the container the repository is mounted at /code -# point autoreload at the actual in-container paths -py-autoreload-on-edit=/code/core -py-autoreload-on-edit=/code/app +log-level=warning diff --git a/scripts/quadlet/manage.sh b/scripts/quadlet/manage.sh index 3c92c7c..6765b01 100755 --- a/scripts/quadlet/manage.sh +++ b/scripts/quadlet/manage.sh @@ -114,12 +114,14 @@ run_worker_container() { local cmd="$2" local with_uwsgi="${3:-0}" local with_whatsapp="${4:-0}" + local cpu_limit="${5:-0.5}" rm_if_exists "$name" local args=( --replace --name "$name" --pod "$POD_NAME" + --cpus "$cpu_limit" --user "$(id -u):$(id -g)" --env-file "$STACK_ENV" --env "SIGNAL_HTTP_URL=http://127.0.0.1:8080" @@ -147,6 +149,7 @@ run_oneshot_container() { --replace --name "$name" --pod "$POD_NAME" + --cpus "0.5" --user "$(id -u):$(id -g)" --env-file "$STACK_ENV" --env "SIGNAL_HTTP_URL=http://127.0.0.1:8080" @@ -223,6 +226,7 @@ start_stack() { --replace \ --name "$REDIS_CONTAINER" \ --pod "$POD_NAME" \ + --cpus "0.1" \ -v "$REPO_DIR/docker/redis.conf:/etc/redis.conf:ro" \ -v "$REDIS_DATA_DIR:/data" \ -v "$VRUN_DIR:/var/run" \ @@ -233,15 +237,17 @@ start_stack() { --replace \ --name "$SIGNAL_CONTAINER" \ --pod "$POD_NAME" \ - -e MODE=json-rpc \ + --cpus "0.2" \ + -e MODE=native \ -v "$ROOT_DIR/signal-cli-config:/home/.local/share/signal-cli" \ - docker.io/bbernhard/signal-cli-rest-api:latest >/dev/null + localhost/gia-signal:latest >/dev/null if [[ "$PROSODY_ENABLED" == "true" ]]; then podman run -d \ --replace \ --name "$PROSODY_CONTAINER" \ --pod "$POD_NAME" \ + --cpus "0.2" \ --env-file "$STACK_ENV" \ --user "$PROSODY_RUN_USER" \ --entrypoint prosody \ @@ -259,11 +265,11 @@ start_stack() { run_oneshot_container "$MIGRATION_CONTAINER" ". /venv/bin/activate && python manage.py migrate --noinput" run_oneshot_container "$COLLECTSTATIC_CONTAINER" ". /venv/bin/activate && python manage.py collectstatic --noinput" - run_worker_container "$APP_CONTAINER" "if [ \"\$OPERATION\" = \"uwsgi\" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi" 1 1 - run_worker_container "$ASGI_CONTAINER" "rm -f /var/run/asgi-gia.sock && . /venv/bin/activate && python -m pip install --disable-pip-version-check -q uvicorn && python -m uvicorn app.asgi:application --uds /var/run/asgi-gia.sock --workers 1" 0 1 - run_worker_container "$UR_CONTAINER" ". /venv/bin/activate && python manage.py ur" 1 1 - run_worker_container "$SCHED_CONTAINER" ". /venv/bin/activate && python manage.py scheduling" 1 0 - run_worker_container "$CODEX_WORKER_CONTAINER" ". /venv/bin/activate && python manage.py codex_worker" 1 0 + run_worker_container "$APP_CONTAINER" "if [ \"\$OPERATION\" = \"uwsgi\" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi" 1 1 0.5 + run_worker_container "$ASGI_CONTAINER" "rm -f /var/run/asgi-gia.sock && . /venv/bin/activate && python -m pip install --disable-pip-version-check -q uvicorn && python -m uvicorn app.asgi:application --uds /var/run/asgi-gia.sock --workers 1" 0 1 0.3 + run_worker_container "$UR_CONTAINER" ". /venv/bin/activate && python manage.py ur" 1 1 0.5 + run_worker_container "$SCHED_CONTAINER" ". /venv/bin/activate && python manage.py scheduling" 1 0 0.3 + run_worker_container "$CODEX_WORKER_CONTAINER" ". /venv/bin/activate && python manage.py codex_worker" 1 0 0.4 } render_units() { diff --git a/scripts/quadlet/render_units.py b/scripts/quadlet/render_units.py index 9fa2fe8..ffeeaa4 100755 --- a/scripts/quadlet/render_units.py +++ b/scripts/quadlet/render_units.py @@ -186,7 +186,7 @@ Image=docker.io/bbernhard/signal-cli-rest-api:latest ContainerName={with_stack('signal')} Pod={pod_ref} Volume={signal_cli_dir}:/home/.local/share/signal-cli -Environment=MODE=json-rpc +Environment=MODE=native [Service] Restart=always diff --git a/stack.env.example b/stack.env.example index 68e2bf8..d8deb09 100644 --- a/stack.env.example +++ b/stack.env.example @@ -35,11 +35,16 @@ XMPP_USER_DOMAIN=example.com XMPP_PORT=8888 # Auto-generated if empty by Prosody startup helpers. XMPP_SECRET= +# XEP-0363 upload service (used by clients + relay attachment upload). +XMPP_UPLOAD_SERVICE=example.com +# Optional legacy alias consumed by app fallback: +# XMPP_UPLOAD_JID=upload.example.com +XMPP_UPLOAD_BASE_URL=https://share.example.com/file_share # Directory for OMEMO key storage. Defaults to <BASE_DIR>/xmpp_omemo_data if unset. # XMPP_OMEMO_DATA_DIR=./.podman/gia_xmpp_omemo_data # Optional Prosody container storage/config paths used by utilities/prosody/manage_prosody_container.sh -PROSODY_IMAGE=docker.io/prosody/prosody:latest +PROSODY_IMAGE=docker.io/prosody/prosody:trunk QUADLET_PROSODY_CONFIG_FILE=./utilities/prosody/prosody.cfg.lua QUADLET_PROSODY_CERTS_DIR=./.podman/gia_prosody_certs QUADLET_PROSODY_DATA_DIR=./.podman/gia_prosody_data diff --git a/utilities/memory/manage_manticore_container.sh b/utilities/memory/manage_manticore_container.sh index fd9f097..dca3e38 100755 --- a/utilities/memory/manage_manticore_container.sh +++ b/utilities/memory/manage_manticore_container.sh @@ -41,7 +41,7 @@ up() { -p "${MANTICORE_SPHINX_PORT}:9312" \ -v "$MANTICORE_DATA_DIR:/var/lib/manticore" \ -v "$MANTICORE_LOG_DIR:/var/log/manticore" \ - -v "$MANTICORE_CONFIG_FILE:/etc/manticoresearch/manticore.conf:ro" \ + -v "$MANTICORE_CONFIG_FILE:/etc/manticoresearch/manticore.conf" \ docker.io/manticoresearch/manticore:latest >/dev/null echo "Started $MANTICORE_CONTAINER" } diff --git a/utilities/prosody/modules/mod_privilege.lua b/utilities/prosody/modules/mod_privilege.lua new file mode 100644 index 0000000..9a527d9 --- /dev/null +++ b/utilities/prosody/modules/mod_privilege.lua @@ -0,0 +1,121 @@ +local st = require "prosody.util.stanza"; +local jid_bare = require "prosody.util.jid".bare; + +local xmlns_privilege = "urn:xmpp:privilege:2"; +local xmlns_forward = "urn:xmpp:forward:0"; +local xmlns_carbons = "urn:xmpp:carbons:2"; +local bare_sessions = prosody.bare_sessions; + +local allowed_component_jid = module:get_option_string("privileged_entity_jid"); + +module:log("info", "mod_privilege loaded for host=%s privileged_entity_jid=%s", module.host or "?", allowed_component_jid or ""); + +local function iter_sessions(user_sessions) + if not user_sessions or not user_sessions.sessions then + return function() return nil end; + end + return pairs(user_sessions.sessions); +end + +local function send_privilege_advertisement(session) + if not allowed_component_jid or allowed_component_jid == "" then + module:log("debug", "Privilege advertisement skipped: no privileged_entity_jid configured"); + return; + end + if session.host ~= allowed_component_jid then + module:log("debug", "Privilege advertisement skipped: session host %s != %s", session.host or "?", allowed_component_jid); + return; + end + local msg = st.message({ + from = module.host; + to = session.host; + type = "normal"; + }); + msg:tag("privilege", { xmlns = xmlns_privilege }) + :tag("perm", { access = "message", type = "outgoing" }):up() + :up(); + session.send(msg); + module:log("info", "Advertised outgoing message privilege to %s", session.host); +end + +local function unwrap_forwarded_message(stanza) + local privilege = stanza:get_child("privilege", xmlns_privilege); + if not privilege then + return nil; + end + local forwarded = privilege:get_child("forwarded", xmlns_forward); + if not forwarded then + return nil; + end + for _, tag in ipairs(forwarded.tags or {}) do + if tag.name == "message" then + return tag; + end + end + return nil; +end + +local function is_carbon_delivery(inner) + if not inner then + return false; + end + return inner:get_child("sent", xmlns_carbons) ~= nil + or inner:get_child("received", xmlns_carbons) ~= nil; +end + +local function deliver_carbon_to_user_sessions(bare_jid, inner) + local user_sessions = bare_sessions[bare_jid]; + if not user_sessions then + module:log("debug", "Privilege carbon skipped for offline user %s", bare_jid); + return true; + end + local delivered = false; + for _, session in iter_sessions(user_sessions) do + if session.full_jid and session.send then + local copy = st.clone(inner); + copy.attr.to = session.full_jid; + session.send(copy); + delivered = true; + end + end + module:log("debug", "Privilege carbon delivered user=%s delivered=%s", bare_jid, tostring(delivered)); + return true; +end + +local function handle_privileged_carbon(event) + local origin, stanza = event.origin, event.stanza; + if not origin or origin.type ~= "component" then + return nil; + end + if allowed_component_jid and allowed_component_jid ~= "" and origin.host ~= allowed_component_jid then + module:log("debug", "Ignoring privileged message from unexpected component %s", origin.host or "?"); + return nil; + end + + local inner = unwrap_forwarded_message(stanza); + if not is_carbon_delivery(inner) then + module:log("debug", "Ignoring privileged message without carbon payload from %s", origin.host or "?"); + return nil; + end + + local bare_to = jid_bare(inner.attr.to); + local bare_from = jid_bare(inner.attr.from); + if not bare_to or bare_to == "" or bare_to ~= bare_from then + module:log("warn", "Rejected malformed privileged carbon from %s", origin.host or "?"); + return true; + end + if bare_to:match("@(.+)$") ~= module.host then + module:log("debug", "Ignoring privileged carbon for remote host %s", bare_to); + return nil; + end + + return deliver_carbon_to_user_sessions(bare_to, inner); +end + +module:hook_global("component-authenticated", function(event) + module:log("info", "component-authenticated for %s", event.session and event.session.host or "?"); + send_privilege_advertisement(event.session); +end); + +module:hook("pre-message/host", handle_privileged_carbon, 10); +module:hook("message/host", handle_privileged_carbon, 10); diff --git a/utilities/prosody/modules/mod_privileged_carbons.lua b/utilities/prosody/modules/mod_privileged_carbons.lua new file mode 100644 index 0000000..3102b6b --- /dev/null +++ b/utilities/prosody/modules/mod_privileged_carbons.lua @@ -0,0 +1,138 @@ +local st = require "prosody.util.stanza"; +local jid_bare = require "prosody.util.jid".bare; + +local xmlns_privilege = "urn:xmpp:privilege:2"; +local xmlns_forward = "urn:xmpp:forward:0"; +local xmlns_carbons = "urn:xmpp:carbons:2"; +local bare_sessions = prosody.bare_sessions; + +local allowed_component_jid = module:get_option_string("privileged_entity_jid"); + +module:log("info", "mod_privileged_carbons loaded for host=%s privileged_entity_jid=%s", module.host or "?", allowed_component_jid or ""); + +local function iter_sessions(user_sessions) + if not user_sessions or not user_sessions.sessions then + return function() return nil end; + end + return pairs(user_sessions.sessions); +end + +local function unwrap_forwarded_message(stanza) + local privilege = stanza:get_child("privilege", xmlns_privilege); + if not privilege then + return nil; + end + local forwarded = privilege:get_child("forwarded", xmlns_forward); + if not forwarded then + return nil; + end + for _, tag in ipairs(forwarded.tags or {}) do + if tag.name == "message" then + return tag; + end + end + return nil; +end + +local function is_carbon_delivery(inner) + if not inner then + return false; + end + return inner:get_child("sent", xmlns_carbons) ~= nil + or inner:get_child("received", xmlns_carbons) ~= nil; +end + +local function build_sent_carbon(inner, user_bare) + local function rebuild_stanza(node) + if type(node) ~= "table" or not node.name then + return node; + end + local attr = {}; + for k, v in pairs(node.attr or {}) do + attr[k] = v; + end + local rebuilt = st.stanza(node.name, attr); + for _, child in ipairs(node) do + if type(child) == "table" and child.name then + rebuilt:add_direct_child(rebuild_stanza(child)); + elseif type(child) == "string" then + rebuilt:add_direct_child(child); + end + end + return rebuilt; + end + local copy = rebuild_stanza(inner); + local function normalize_client_ns(node) + if not node then + return; + end + if node.attr then + if node.attr.xmlns == nil or node.attr.xmlns == "jabber:component:accept" then + node.attr.xmlns = "jabber:client"; + end + end + if node.tags then + for _, child in ipairs(node.tags) do + normalize_client_ns(child); + end + end + end + normalize_client_ns(copy); + return st.message({ from = user_bare, type = inner.attr.type or "chat", xmlns = "jabber:client" }) + :tag("sent", { xmlns = xmlns_carbons }) + :tag("forwarded", { xmlns = xmlns_forward }) + :add_child(copy):reset(); +end + +local function deliver_carbon_to_user_sessions(bare_jid, inner) + local user_sessions = bare_sessions[bare_jid]; + if not user_sessions then + module:log("debug", "Privileged carbon skipped for offline user %s", bare_jid); + return true; + end + local carbon = build_sent_carbon(inner, bare_jid); + local delivered = false; + for _, session in iter_sessions(user_sessions) do + if session.full_jid and session.send then + local copy = st.clone(carbon); + copy.attr.xmlns = "jabber:client"; + copy.attr.to = session.full_jid; + session.rawsend(tostring(copy)); + delivered = true; + end + end + module:log("info", "Privileged carbon delivered user=%s delivered=%s", bare_jid, tostring(delivered)); + return true; +end + +local function handle_privileged_carbon(event) + local origin, stanza = event.origin, event.stanza; + if not origin or origin.type ~= "component" then + return nil; + end + if allowed_component_jid and allowed_component_jid ~= "" and origin.host ~= allowed_component_jid then + module:log("debug", "Ignoring privileged message from unexpected component %s", origin.host or "?"); + return nil; + end + + local inner = unwrap_forwarded_message(stanza); + if not inner then + module:log("debug", "Ignoring privileged message without forwarded payload from %s", origin.host or "?"); + return nil; + end + + local bare_from = jid_bare(inner.attr.from); + if not bare_from or bare_from == "" then + module:log("warn", "Rejected malformed privileged carbon from %s", origin.host or "?"); + return true; + end + if bare_from:match("@(.+)$") ~= module.host then + module:log("debug", "Ignoring privileged carbon for remote host %s", bare_from); + return nil; + end + + return deliver_carbon_to_user_sessions(bare_from, inner); +end + +module:hook("pre-message/host", handle_privileged_carbon, 10); +module:hook("message/host", handle_privileged_carbon, 10); diff --git a/utilities/prosody/prosody.cfg.lua b/utilities/prosody/prosody.cfg.lua index 7c4b2f2..dabb5d8 100644 --- a/utilities/prosody/prosody.cfg.lua +++ b/utilities/prosody/prosody.cfg.lua @@ -1,5 +1,7 @@ local env = os.getenv local xmpp_secret = env("XMPP_SECRET") or "" +local xmpp_user_domain = env("XMPP_USER_DOMAIN") or "example.com" +local xmpp_upload_base_url = env("XMPP_UPLOAD_BASE_URL") or "https://share.example.com/file_share" if xmpp_secret == "" then error("XMPP_SECRET is required for Prosody component authentication") @@ -19,6 +21,7 @@ modules_enabled = { "saslauth"; "tls"; "blocklist"; + "privileged_carbons"; "carbons"; "dialback"; "limits"; @@ -34,6 +37,7 @@ modules_enabled = { "admin_adhoc"; "announce"; "http"; + "http_file_share"; } s2s_secure_auth = true @@ -66,7 +70,10 @@ http_external_url = "https://share.example.com/" VirtualHost "example.com" authentication = "gia" + http_host = "share.example.com" external_auth_command = "/code/utilities/prosody/auth_django.sh" + privileged_entity_jid = env("XMPP_JID") or "jews.example.com" + http_file_share_size_limit = 104857600 ssl = { key = "/etc/prosody/certs/cert.pem"; certificate = "/etc/prosody/certs/cert.pem"; diff --git a/utilities/signal/Containerfile b/utilities/signal/Containerfile new file mode 100644 index 0000000..4e2cd3f --- /dev/null +++ b/utilities/signal/Containerfile @@ -0,0 +1,85 @@ +# Custom signal-cli-rest-api image with signal-cli 0.14.1 +# +# signal-cli 0.14.1 ships pre-built as two standalone binaries: +# signal-cli-<VER>-Linux-client.tar.gz → single file "signal-cli-client" (JVM) +# signal-cli-<VER>-Linux-native.tar.gz → single file "signal-cli" (native/GraalVM) +# +# We pull the REST API Go binaries from the upstream bbernhard image and +# layer in the 0.14.1 native binary. + +ARG SIGNAL_CLI_VERSION=0.14.1 +ARG BBERNHARD_TAG=latest + +# ── Stage 1: REST API binaries from upstream ─────────────────────────────── +FROM docker.io/bbernhard/signal-cli-rest-api:${BBERNHARD_TAG} AS restapi + +# ── Stage 2: runtime image ───────────────────────────────────────────────── +FROM ubuntu:noble + +ARG SIGNAL_CLI_VERSION + +ENV GIN_MODE=release +ENV PORT=8080 +ENV SIGNAL_CLI_CONFIG_DIR=/home/.local/share/signal-cli +ENV SIGNAL_CLI_UID=1000 +ENV SIGNAL_CLI_GID=1000 +ENV SIGNAL_CLI_CHOWN_ON_STARTUP=true +ENV SIGNAL_CLI_REST_API_PLUGIN_SHARED_OBJ_DIR=/usr/bin/ +ENV LANG=en_US.UTF-8 + +# Runtime deps (openjdk-21 for JVM fallback, supervisor for json-rpc mode) +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + util-linux supervisor openjdk-21-jre wget curl locales \ + && sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ + && dpkg-reconfigure --frontend=noninteractive locales \ + && update-locale LANG=en_US.UTF-8 \ + && rm -rf /var/lib/apt/lists/* + +# Copy REST API binaries from upstream image +COPY --from=restapi /usr/bin/signal-cli-rest-api /usr/bin/signal-cli-rest-api +COPY --from=restapi /usr/bin/jsonrpc2-helper /usr/bin/jsonrpc2-helper +COPY --from=restapi /usr/bin/signal-cli-rest-api_plugin_loader.so /usr/bin/signal-cli-rest-api_plugin_loader.so +COPY --from=restapi /entrypoint.sh /entrypoint.sh +COPY --from=restapi /etc/supervisor /etc/supervisor + +# Download signal-cli 0.14.1 binaries. +# The tarballs each contain a single file (no subdirectory). +# native tarball → file named "signal-cli" +# client tarball → file named "signal-cli-client" +RUN mkdir -p /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin \ + \ + # Native binary (GraalVM compiled, no JVM needed) + && wget -q "https://github.com/AsamK/signal-cli/releases/download/v${SIGNAL_CLI_VERSION}/signal-cli-${SIGNAL_CLI_VERSION}-Linux-native.tar.gz" \ + -O /opt/signal-cli-native.tar.gz \ + && tar xzf /opt/signal-cli-native.tar.gz -C /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/ \ + && mv /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli \ + /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli-native \ + && chmod +x /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli-native \ + && rm /opt/signal-cli-native.tar.gz \ + \ + # JVM client (used when MODE != native, and as "signal-cli" wrapper) + && wget -q "https://github.com/AsamK/signal-cli/releases/download/v${SIGNAL_CLI_VERSION}/signal-cli-${SIGNAL_CLI_VERSION}-Linux-client.tar.gz" \ + -O /opt/signal-cli-client.tar.gz \ + && tar xzf /opt/signal-cli-client.tar.gz -C /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/ \ + && mv /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli-client \ + /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli \ + && chmod +x /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli \ + && rm /opt/signal-cli-client.tar.gz \ + \ + # Symlinks to /usr/bin (expected by signal-cli-rest-api) + && ln -sf /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli /usr/bin/signal-cli \ + && ln -sf /opt/signal-cli-${SIGNAL_CLI_VERSION}/bin/signal-cli-native /usr/bin/signal-cli-native + +# User + directories (mirror upstream; remove default ubuntu user first) +RUN userdel ubuntu -r 2>/dev/null || true \ + && groupadd -g 1000 signal-api \ + && useradd --no-log-init -M -d /home -s /bin/bash -u 1000 -g 1000 signal-api \ + && mkdir -p /home/.local/share/signal-cli /signal-cli-config + +EXPOSE ${PORT} + +ENTRYPOINT ["/entrypoint.sh"] + +HEALTHCHECK --interval=20s --timeout=10s --retries=3 \ + CMD curl -f http://localhost:${PORT}/v1/health || exit 1 diff --git a/vendor/django-crud-mixins/README.md b/vendor/django-crud-mixins/README.md index efcf327..00a45d0 100644 --- a/vendor/django-crud-mixins/README.md +++ b/vendor/django-crud-mixins/README.md @@ -33,12 +33,12 @@ Designed for **HTMX**, **Gridstack.js**, and **Django ORM** with built-in **acce Add to your `requirements.txt`: ```shell -git+https://git.zm.is/XF/django-crud-mixins +git+https://git.example.invalid/vendor/django-crud-mixins ``` Or install via pip: ```shell -pip install git+https://git.zm.is/XF/django-crud-mixins +pip install git+https://git.example.invalid/vendor/django-crud-mixins ``` ## 🔧 Usage diff --git a/vendor/django-crud-mixins/setup.cfg b/vendor/django-crud-mixins/setup.cfg index 6304fdd..09e3224 100644 --- a/vendor/django-crud-mixins/setup.cfg +++ b/vendor/django-crud-mixins/setup.cfg @@ -2,8 +2,8 @@ name = django-crud-mixins version = 1.0.3 author = Mark Veidemanis -author_email = m@zm.is -url = https://git.zm.is/XF/django-crud-mixins +author_email = maintainer@example.test +url = https://git.example.invalid/vendor/django-crud-mixins description = CRUD mixins for Django class-based views. long_description = file: README.md long_description_content_type = text/markdown @@ -25,4 +25,4 @@ install_requires = [options.package_data] mixins = templates/mixins/*, README.md -* = README.md \ No newline at end of file +* = README.md