XF/GIA
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Fully implement WhatsApp, Signal and XMPP multiplexing

Browse Source
This commit is contained in:
Mark Veidemanis
2026-02-16 19:19:32 +00:00
parent 3f82c27ab9
commit 658ab10647
9 changed files with 659 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
core/clients/signal.py
View File

@@ -1,5 +1,6 @@
import asyncio
import json
import time
from urllib.parse import quote_plus, urlparse
import aiohttp
@@ -472,6 +473,14 @@ class HandleMessage(Command):
outgoing=is_from_bot,
)
stored_messages.add(message_key)
# Notify unified router to ensure service context is preserved
await self.ur.message_received(
self.service,
identifier=identifier,
text=message_text,
ts=ts,
payload=msg,
)
# TODO: Permission checks
manips = await sync_to_async(list)(Manipulation.objects.filter(enabled=True))
@@ -595,6 +604,7 @@ class HandleMessage(Command):
class SignalClient(ClientBase):
def __init__(self, ur, *args, **kwargs):
super().__init__(ur, *args, **kwargs)
self._stopping = False
signal_number = str(getattr(settings, "SIGNAL_NUMBER", "")).strip()
self.client = NewSignalBot(
ur,
@@ -606,9 +616,121 @@ class SignalClient(ClientBase):
)
self.client.register(HandleMessage(self.ur, self.service))
self._command_task = None
async def _drain_runtime_commands(self):
"""Process queued runtime commands (e.g., web UI sends via composite router)."""
from core.clients import transport
# Process a small burst each loop to keep sends responsive.
for _ in range(5):
command = transport.pop_runtime_command(self.service)
if not command:
return
await self._execute_runtime_command(command)
async def _execute_runtime_command(self, command):
"""Execute a single runtime command like send_message_raw."""
from core.clients import transport
command_id = str((command or {}).get("id") or "").strip()
action = str((command or {}).get("action") or "").strip()
payload = dict((command or {}).get("payload") or {})
if not command_id:
return
if action == "send_message_raw":
recipient = str(payload.get("recipient") or "").strip()
text = payload.get("text")
attachments = payload.get("attachments") or []
try:
result = await signalapi.send_message_raw(
recipient_uuid=recipient,
text=text,
attachments=attachments,
)
if result is False or result is None:
raise RuntimeError("signal_send_failed")
transport.set_runtime_command_result(
self.service,
command_id,
{
"ok": True,
"timestamp": int(result)
if isinstance(result, int)
else int(time.time() * 1000),
},
)
except Exception as exc:
self.log.error(f"send_message_raw failed: {exc}", exc_info=True)
transport.set_runtime_command_result(
self.service,
command_id,
{
"ok": False,
"error": str(exc),
},
)
return
if action == "notify_xmpp_sent":
person_identifier_id = str(payload.get("person_identifier_id") or "").strip()
text = str(payload.get("text") or "")
if not person_identifier_id:
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": "missing_person_identifier_id"},
)
return
try:
identifier = await sync_to_async(
lambda: PersonIdentifier.objects.filter(id=person_identifier_id).select_related("user", "person").first()
)()
if identifier is None:
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": "person_identifier_not_found"},
)
return
await self.ur.xmpp.client.send_from_external(
identifier.user,
identifier,
text,
True,
attachments=[],
)
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": True, "timestamp": int(time.time() * 1000)},
)
except Exception as exc:
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": str(exc)},
)
return
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": f"unsupported_action:{action or '-'}"},
)
async def _command_loop(self):
"""Background task to periodically drain queued commands."""
while not self._stopping:
try:
await self._drain_runtime_commands()
except Exception as exc:
self.log.warning(f"Command loop error: {exc}")
await asyncio.sleep(1)
def start(self):
self.log.info("Signal client starting...")
self.client._event_loop = self.loop
# Start background command processing loop
if not self._command_task or self._command_task.done():
self._command_task = self.loop.create_task(self._command_loop())
self.client.start()

28
core/clients/transport.py
View File

@@ -44,6 +44,10 @@ def _runtime_command_cancel_key(service: str, command_id: str) -> str:
return f"gia:service:command-cancel:{_service_key(service)}:{command_id}"
def _runtime_command_meta_key(service: str, command_id: str) -> str:
return f"gia:service:command-meta:{_service_key(service)}:{command_id}"
def _gateway_base(service: str) -> str:
key = f"{service.upper()}_HTTP_URL"
default = f"http://{service}:8080"
@@ -110,6 +114,14 @@ def enqueue_runtime_command(
if len(queued) > 200:
queued = queued[-200:]
cache.set(key, queued, timeout=_RUNTIME_COMMANDS_TTL)
cache.set(
_runtime_command_meta_key(service_key, command_id),
{
"created_at": int(command.get("created_at") or int(time.time())),
"action": str(command.get("action") or ""),
},
timeout=_RUNTIME_COMMANDS_TTL,
)
return command_id
@@ -132,6 +144,7 @@ def set_runtime_command_result(
payload = dict(result or {})
payload.setdefault("completed_at", int(time.time()))
cache.set(result_key, payload, timeout=_RUNTIME_COMMAND_RESULT_TTL)
cache.delete(_runtime_command_meta_key(service_key, command_id))
def cancel_runtime_command(service: str, command_id: str):
@@ -142,9 +155,24 @@ def cancel_runtime_command(service: str, command_id: str):
payload = {"ok": False, "error": "cancelled", "completed_at": int(time.time())}
cache.set(result_key, payload, timeout=_RUNTIME_COMMAND_RESULT_TTL)
cache.set(cancel_key, True, timeout=60)
cache.delete(_runtime_command_meta_key(service_key, command_id))
return True
def runtime_command_age_seconds(service: str, command_id: str) -> float | None:
service_key = _service_key(service)
meta = cache.get(_runtime_command_meta_key(service_key, command_id))
if not isinstance(meta, dict):
return None
try:
created_at = int(meta.get("created_at") or 0)
except Exception:
created_at = 0
if created_at <= 0:
return None
return max(0.0, time.time() - created_at)
def cancel_runtime_commands_for_recipient(service: str, recipient: str) -> list[str]:
"""Cancel any queued runtime commands for the given recipient and return their ids."""
service_key = _service_key(service)

215
core/clients/whatsapp.py
View File

@@ -1,4 +1,5 @@
import asyncio
import inspect
import logging
import os
import re
@@ -42,6 +43,7 @@ class WhatsAppClient(ClientBase):
self._qr_handler_registered = False
self._qr_handler_supported = False
self._event_hook_callable = False
self._last_send_error = ""
self.enabled = bool(
str(getattr(settings, "WHATSAPP_ENABLED", "false")).lower()
@@ -658,6 +660,17 @@ class WhatsAppClient(ClientBase):
return await value
return value
async def _call_client_method(self, method, *args, timeout: float | None = None):
if method is None:
return None
if inspect.iscoroutinefunction(method):
coro = method(*args)
else:
coro = asyncio.to_thread(method, *args)
if timeout and timeout > 0:
return await asyncio.wait_for(coro, timeout=timeout)
return await coro
async def _drain_runtime_commands(self):
# Process a small burst each loop to keep sends responsive but avoid starvation.
for _ in range(5):
@@ -672,18 +685,27 @@ class WhatsAppClient(ClientBase):
payload = dict((command or {}).get("payload") or {})
if not command_id:
return
self.log.info(
"whatsapp runtime command start: id=%s action=%s",
command_id,
action,
)
if action == "send_message_raw":
recipient = str(payload.get("recipient") or "").strip()
text = payload.get("text")
attachments = payload.get("attachments") or []
send_timeout_s = 18.0
try:
# Include command_id so send_message_raw can observe cancel requests
result = await self.send_message_raw(
recipient=recipient,
text=text,
attachments=attachments,
command_id=command_id,
result = await asyncio.wait_for(
self.send_message_raw(
recipient=recipient,
text=text,
attachments=attachments,
command_id=command_id,
),
timeout=send_timeout_s,
)
if result is not False and result is not None:
transport.set_runtime_command_result(
@@ -696,15 +718,45 @@ class WhatsAppClient(ClientBase):
else int(time.time() * 1000),
},
)
self.log.info(
"whatsapp runtime command ok: id=%s action=%s",
command_id,
action,
)
return
transport.set_runtime_command_result(
self.service,
command_id,
{
"ok": False,
"error": "runtime_send_failed",
"error": str(
getattr(self, "_last_send_error", "")
or "runtime_send_failed"
),
},
)
self.log.warning(
"whatsapp runtime command failed: id=%s action=%s error=%s",
command_id,
action,
str(getattr(self, "_last_send_error", "") or "runtime_send_failed"),
)
return
except asyncio.TimeoutError:
transport.set_runtime_command_result(
self.service,
command_id,
{
"ok": False,
"error": f"runtime_send_timeout:{int(send_timeout_s)}s",
},
)
self.log.warning(
"whatsapp runtime command timeout: id=%s action=%s timeout=%ss",
command_id,
action,
int(send_timeout_s),
)
return
except Exception as exc:
transport.set_runtime_command_result(
@@ -715,6 +767,12 @@ class WhatsAppClient(ClientBase):
"error": str(exc),
},
)
self.log.warning(
"whatsapp runtime command exception: id=%s action=%s error=%s",
command_id,
action,
exc,
)
return
if action == "force_history_sync":
@@ -741,6 +799,48 @@ class WhatsAppClient(ClientBase):
)
return
if action == "notify_xmpp_sent":
person_identifier_id = str(payload.get("person_identifier_id") or "").strip()
text = str(payload.get("text") or "")
if not person_identifier_id:
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": "missing_person_identifier_id"},
)
return
try:
identifier = await sync_to_async(
lambda: PersonIdentifier.objects.filter(id=person_identifier_id).select_related("user", "person").first()
)()
if identifier is None:
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": "person_identifier_not_found"},
)
return
await self.ur.xmpp.client.send_from_external(
identifier.user,
identifier,
text,
True,
attachments=[],
)
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": True, "timestamp": int(time.time() * 1000)},
)
return
except Exception as exc:
transport.set_runtime_command_result(
self.service,
command_id,
{"ok": False, "error": str(exc)},
)
return
transport.set_runtime_command_result(
self.service,
command_id,
@@ -2338,27 +2438,43 @@ class WhatsAppClient(ClientBase):
async def send_message_raw(
self, recipient, text=None, attachments=None, command_id: str | None = None
):
self._last_send_error = ""
if not self._client:
return False
if self._build_jid is None:
self._last_send_error = "client_missing"
return False
jid_str = self._to_jid(recipient)
if not jid_str:
self._last_send_error = "recipient_invalid"
return False
# Convert string JID to actual JID object that neonize expects
# Prefer direct JID string for sends to avoid Neonize usync/device-list
# lookups that can stall on some runtime sessions.
jid = jid_str
jid_obj = None
try:
jid = self._build_jid(jid_str)
# Verify it's a proper JID object with SerializeToString method
if not hasattr(jid, "SerializeToString"):
self.log.error(
"whatsapp build_jid returned non-JID object: type=%s repr=%s",
type(jid).__name__,
repr(jid)[:100],
)
return False
if self._build_jid is not None:
maybe_jid = None
if "@" in jid_str:
local_part, server_part = jid_str.split("@", 1)
try:
maybe_jid = self._build_jid(local_part, server_part)
except TypeError:
maybe_jid = self._build_jid(jid_str)
else:
maybe_jid = self._build_jid(jid_str)
if hasattr(maybe_jid, "SerializeToString"):
jid_obj = maybe_jid
else:
self.log.warning(
"whatsapp build_jid returned non-JID object, falling back to string: type=%s repr=%s",
type(maybe_jid).__name__,
repr(maybe_jid)[:100],
)
except Exception as exc:
self.log.warning("whatsapp failed to build JID from %s: %s", jid_str, exc)
return False
self.log.warning(
"whatsapp failed to build JID from %s, falling back to string: %s",
jid_str,
exc,
)
if not self._connected and hasattr(self._client, "connect"):
try:
await self._maybe_await(self._client.connect())
@@ -2370,6 +2486,7 @@ class WhatsAppClient(ClientBase):
last_error="",
)
except Exception as exc:
self._last_send_error = f"reconnect_failed:{exc}"
self._publish_state(
connected=False,
last_event="send_reconnect_failed",
@@ -2388,24 +2505,25 @@ class WhatsAppClient(ClientBase):
).lower()
data = payload.get("content") or b""
filename = payload.get("filename") or "attachment.bin"
attachment_target = jid_obj if jid_obj is not None else jid
try:
if mime.startswith("image/") and hasattr(self._client, "send_image"):
response = await self._maybe_await(
self._client.send_image(jid, data, caption="")
self._client.send_image(attachment_target, data, caption="")
)
elif mime.startswith("video/") and hasattr(self._client, "send_video"):
response = await self._maybe_await(
self._client.send_video(jid, data, caption="")
self._client.send_video(attachment_target, data, caption="")
)
elif mime.startswith("audio/") and hasattr(self._client, "send_audio"):
response = await self._maybe_await(
self._client.send_audio(jid, data)
self._client.send_audio(attachment_target, data)
)
elif hasattr(self._client, "send_document"):
response = await self._maybe_await(
self._client.send_document(
jid,
attachment_target,
data,
filename=filename,
mimetype=mime,
@@ -2435,22 +2553,27 @@ class WhatsAppClient(ClientBase):
except Exception:
cancel_key = None
for attempt in range(5): # Increased from 3 to 5 attempts
for attempt in range(2):
# Check for a cancellation marker set by transport.cancel_runtime_command
try:
if cancel_key and cache.get(cancel_key):
self.log.info("whatsapp send cancelled via cancel marker")
self._last_send_error = "cancelled"
return False
except Exception:
pass
try:
send_target = jid_obj if jid_obj is not None else jid
# Log what we're about to send for debugging
if getattr(settings, "WHATSAPP_DEBUG", False):
self.log.debug(
f"send_message attempt {attempt+1}: jid={jid} text_type={type(text).__name__} text_len={len(text)}"
f"send_message attempt {attempt+1}: target_type={type(send_target).__name__} text_type={type(text).__name__} text_len={len(text)}"
)
response = await self._maybe_await(
self._client.send_message(jid, text)
response = await self._call_client_method(
getattr(self._client, "send_message", None),
send_target,
text,
timeout=9.0,
)
sent_any = True
last_error = None
@@ -2462,9 +2585,31 @@ class WhatsAppClient(ClientBase):
)
last_error = exc
error_text = str(last_error or "").lower()
is_transient = "usync query" in error_text or "timed out" in error_text
if is_transient and attempt < 4: # Updated to match new attempt range
error_text = (
f"{type(last_error).__name__}:{repr(last_error)}"
if last_error is not None
else ""
).lower()
is_transient = (
"usync" in error_text
or "timed out" in error_text
or "timeout" in error_text
or "device list" in error_text
or "serializetostring" in error_text
or not error_text.strip()
)
if is_transient and attempt < 1:
# If runtime rejected string target, try to build protobuf JID for retry.
if jid_obj is None and self._build_jid is not None and "@" in jid_str:
local_part, server_part = jid_str.split("@", 1)
try:
maybe_retry_jid = self._build_jid(local_part, server_part)
except TypeError:
maybe_retry_jid = self._build_jid(jid_str)
except Exception:
maybe_retry_jid = None
if hasattr(maybe_retry_jid, "SerializeToString"):
jid_obj = maybe_retry_jid
if hasattr(self._client, "connect"):
try:
await self._maybe_await(self._client.connect())
@@ -2482,7 +2627,7 @@ class WhatsAppClient(ClientBase):
)
# Sleep but wake earlier if cancelled: poll small intervals
# Increase backoff time for device list queries
total_sleep = 1.5 * (attempt + 1)
total_sleep = 0.8 * (attempt + 1)
slept = 0.0
while slept < total_sleep:
try:
@@ -2490,6 +2635,7 @@ class WhatsAppClient(ClientBase):
self.log.info(
"whatsapp send cancelled during retry backoff"
)
self._last_send_error = "cancelled"
return False
except Exception:
pass
@@ -2499,6 +2645,9 @@ class WhatsAppClient(ClientBase):
break
if last_error is not None and not sent_any:
self.log.warning("whatsapp text send failed: %s", last_error)
self._last_send_error = (
f"text_send_failed:{type(last_error).__name__}:{repr(last_error)}"
)
return False
sent_ts = max(
sent_ts,
@@ -2506,7 +2655,9 @@ class WhatsAppClient(ClientBase):
)
if not sent_any:
self._last_send_error = "no_payload_sent"
return False
self._last_send_error = ""
return sent_ts or int(time.time() * 1000)
async def start_typing(self, identifier):