XF/GIA
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix all integrations

Browse Source
This commit is contained in:
Mark Veidemanis
2026-03-08 22:08:55 +00:00
parent bca4d6898f
commit acedc01e83
58 changed files with 4120 additions and 960 deletions
Download Patch File Download Diff File Expand all files Collapse all files

523
core/clients/signal.py
View File

@@ -280,6 +280,43 @@ def _extract_signal_delete(envelope):
}
def _extract_signal_group_identifiers(payload: dict) -> list[str]:
envelope = payload.get("envelope") or {}
if not isinstance(envelope, dict):
return []
candidates = []
paths = [
("group",),
("groupV2",),
("dataMessage", "groupInfo"),
("dataMessage", "groupV2"),
("syncMessage", "sentMessage", "groupInfo"),
("syncMessage", "sentMessage", "groupV2"),
("syncMessage", "sentMessage", "message", "groupInfo"),
("syncMessage", "sentMessage", "message", "groupV2"),
]
key_names = ("id", "groupId", "groupID", "internal_id", "masterKey")
for path in paths:
node = _get_nested(envelope, path)
if isinstance(node, str):
candidates.append(node)
continue
if not isinstance(node, dict):
continue
for key_name in key_names:
value = str(node.get(key_name) or "").strip()
if value:
candidates.append(value)
unique = []
for value in candidates:
cleaned = str(value or "").strip()
if cleaned and cleaned not in unique:
unique.append(cleaned)
return unique
def _extract_signal_text(raw_payload, default_text=""):
text = str(default_text or "").strip()
if text:
@@ -332,6 +369,22 @@ def _identifier_candidates(*values):
return out
def _dedupe_person_identifiers(rows):
deduped = []
seen = set()
for row in rows or []:
key = (
int(getattr(row, "user_id", 0) or 0),
str(getattr(row, "person_id", "") or ""),
str(getattr(row, "service", "") or "").strip().lower(),
)
if key in seen:
continue
seen.add(key)
deduped.append(row)
return deduped
def _digits_only(value):
return re.sub(r"[^0-9]", "", str(value or "").strip())
@@ -762,6 +815,31 @@ class HandleMessage(Command):
log.warning("Signal reaction relay to XMPP failed: %s", exc)
return
if reply_to_self and str(text or "").strip().startswith("."):
responded_user_ids = set()
for identifier in identifiers:
if identifier.user_id in responded_user_ids:
continue
responded_user_ids.add(identifier.user_id)
gateway_replies = await self.ur.xmpp.client.execute_gateway_command(
sender_user=identifier.user,
body=text,
service=self.service,
channel_identifier=str(identifier.identifier or ""),
sender_identifier=str(identifier.identifier or ""),
local_message=None,
message_meta={
"signal": {
"source_uuid": str(effective_source_uuid or ""),
"source_number": str(effective_source_number or ""),
"reply_to_self": True,
}
},
)
for line in gateway_replies:
await c.send(f"[>] {line}")
return
# Handle attachments across multiple Signal payload variants.
attachment_list = _extract_attachments(raw)
xmpp_attachments = []
@@ -1087,6 +1165,130 @@ class SignalClient(ClientBase):
self.client.register(HandleMessage(self.ur, self.service))
self._command_task = None
self._raw_receive_task = None
self._catalog_refresh_task = None
async def _signal_api_get_list(
self, session: aiohttp.ClientSession, path: str
) -> list[dict]:
base = str(getattr(settings, "SIGNAL_HTTP_URL", "http://signal:8080")).rstrip(
"/"
)
try:
async with session.get(f"{base}{path}") as response:
if response.status != 200:
body = str(await response.text()).strip()[:300]
self.log.warning(
"signal catalog fetch failed path=%s status=%s body=%s",
path,
response.status,
body or "-",
)
return []
payload = await response.json(content_type=None)
except Exception as exc:
self.log.warning("signal catalog fetch failed path=%s error=%s", path, exc)
return []
return payload if isinstance(payload, list) else []
async def _refresh_runtime_catalog(self) -> None:
signal_number = str(getattr(settings, "SIGNAL_NUMBER", "") or "").strip()
if not signal_number:
return
encoded_account = quote_plus(signal_number)
timeout = aiohttp.ClientTimeout(total=20)
async with aiohttp.ClientSession(timeout=timeout) as session:
identities = await self._signal_api_get_list(
session, f"/v1/identities/{encoded_account}"
)
groups = await self._signal_api_get_list(
session, f"/v1/groups/{encoded_account}"
)
account_digits = _digits_only(signal_number)
contact_rows = []
seen_contacts = set()
for identity in identities:
if not isinstance(identity, dict):
continue
number = str(identity.get("number") or "").strip()
uuid = str(identity.get("uuid") or "").strip()
if account_digits and number and _digits_only(number) == account_digits:
continue
identifiers = []
for candidate in (number, uuid):
cleaned = str(candidate or "").strip()
if cleaned and cleaned not in identifiers:
identifiers.append(cleaned)
if not identifiers:
continue
key = (
f"uuid:{uuid.lower()}"
if uuid
else f"phone:{_digits_only(number) or number}"
)
if key in seen_contacts:
continue
seen_contacts.add(key)
contact_rows.append(
{
"identifier": number or uuid,
"identifiers": identifiers,
"name": str(identity.get("name") or "").strip(),
"number": number,
"uuid": uuid,
}
)
group_rows = []
seen_groups = set()
for group in groups:
if not isinstance(group, dict):
continue
group_id = str(group.get("id") or "").strip()
internal_id = str(group.get("internal_id") or "").strip()
identifiers = []
for candidate in (group_id, internal_id):
cleaned = str(candidate or "").strip()
if cleaned and cleaned not in identifiers:
identifiers.append(cleaned)
if not identifiers:
continue
key = group_id or internal_id
if key in seen_groups:
continue
seen_groups.add(key)
group_rows.append(
{
"identifier": group_id or internal_id,
"identifiers": identifiers,
"name": str(group.get("name") or "").strip(),
"id": group_id,
"internal_id": internal_id,
}
)
transport.update_runtime_state(
self.service,
accounts=[signal_number],
contacts=contact_rows,
groups=group_rows,
catalog_refreshed_at=int(time.time()),
catalog_error="",
)
async def _refresh_runtime_catalog_safe(self) -> None:
try:
await self._refresh_runtime_catalog()
except asyncio.CancelledError:
raise
except Exception as exc:
transport.update_runtime_state(
self.service,
catalog_error=str(exc).strip()[:300],
catalog_refreshed_at=int(time.time()),
)
self.log.warning("signal catalog refresh failed: %s", exc)
async def _drain_runtime_commands(self):
"""Process queued runtime commands (e.g., web UI sends via composite router)."""
@@ -1237,7 +1439,104 @@ class SignalClient(ClientBase):
self.log.warning(f"Command loop error: {exc}")
await asyncio.sleep(1)
async def _resolve_signal_identifiers(self, source_uuid: str, source_number: str):
async def _resolve_signal_group_identifiers(self, group_candidates: list[str]):
unique_candidates = []
for value in group_candidates or []:
cleaned = str(value or "").strip()
if cleaned and cleaned not in unique_candidates:
unique_candidates.append(cleaned)
if not unique_candidates:
return []
runtime_groups = transport.get_runtime_state(self.service).get("groups") or []
expanded = list(unique_candidates)
for item in runtime_groups:
if not isinstance(item, dict):
continue
identifiers = []
for candidate in item.get("identifiers") or []:
cleaned = str(candidate or "").strip()
if cleaned:
identifiers.append(cleaned)
if not identifiers:
continue
if not any(candidate in identifiers for candidate in unique_candidates):
continue
for candidate in identifiers:
if candidate not in expanded:
expanded.append(candidate)
exact_identifiers = await sync_to_async(list)(
PersonIdentifier.objects.filter(
service=self.service,
identifier__in=expanded,
).select_related("user", "person")
)
if exact_identifiers:
return exact_identifiers
bare_candidates = []
for candidate in expanded:
bare = str(candidate or "").strip().split("@", 1)[0].strip()
if bare and bare not in bare_candidates:
bare_candidates.append(bare)
links = await sync_to_async(list)(
PlatformChatLink.objects.filter(
service=self.service,
is_group=True,
chat_identifier__in=bare_candidates,
)
.select_related("user", "person_identifier", "person")
.order_by("id")
)
if not links:
return []
results = []
seen_ids = set()
for link in links:
if link.person_identifier_id:
if link.person_identifier_id not in seen_ids:
seen_ids.add(link.person_identifier_id)
results.append(link.person_identifier)
continue
if not link.person_id:
continue
group_pi = await sync_to_async(
lambda: PersonIdentifier.objects.filter(
user=link.user,
person=link.person,
service=self.service,
identifier__in=expanded,
)
.select_related("user", "person")
.first()
)()
if group_pi is None:
group_pi = await sync_to_async(
lambda: PersonIdentifier.objects.filter(
user=link.user,
person=link.person,
service=self.service,
)
.select_related("user", "person")
.first()
)()
if group_pi is not None and group_pi.id not in seen_ids:
seen_ids.add(group_pi.id)
results.append(group_pi)
return results
async def _resolve_signal_identifiers(
self,
source_uuid: str,
source_number: str,
group_candidates: list[str] | None = None,
):
group_rows = await self._resolve_signal_group_identifiers(group_candidates or [])
if group_rows:
return _dedupe_person_identifiers(group_rows)
candidates = _identifier_candidates(source_uuid, source_number)
if not candidates:
return []
@@ -1248,7 +1547,7 @@ class SignalClient(ClientBase):
)
)
if identifiers:
return identifiers
return _dedupe_person_identifiers(identifiers)
candidate_digits = {_digits_only(value) for value in candidates}
candidate_digits = {value for value in candidate_digits if value}
if not candidate_digits:
@@ -1256,11 +1555,13 @@ class SignalClient(ClientBase):
rows = await sync_to_async(list)(
PersonIdentifier.objects.filter(service=self.service).select_related("user")
)
return [
row
for row in rows
if _digits_only(getattr(row, "identifier", "")) in candidate_digits
]
return _dedupe_person_identifiers(
[
row
for row in rows
if _digits_only(getattr(row, "identifier", "")) in candidate_digits
]
)
async def _auto_link_single_user_signal_identifier(
self, source_uuid: str, source_number: str
@@ -1301,7 +1602,123 @@ class SignalClient(ClientBase):
fallback_identifier,
int(owner.id),
)
return [pi]
return _dedupe_person_identifiers([pi])
async def _build_xmpp_relay_attachments(self, payload: dict):
attachment_list = _extract_attachments(payload)
xmpp_attachments = []
compose_media_urls = []
if not attachment_list:
return xmpp_attachments, compose_media_urls
fetched_attachments = await asyncio.gather(
*[signalapi.fetch_signal_attachment(att["id"]) for att in attachment_list]
)
for fetched, att in zip(fetched_attachments, attachment_list):
if not fetched:
self.log.warning(
"signal raw attachment fetch failed attachment_id=%s", att["id"]
)
continue
xmpp_attachments.append(
{
"content": fetched["content"],
"content_type": fetched["content_type"],
"filename": fetched["filename"],
"size": fetched["size"],
}
)
blob_key = media_bridge.put_blob(
service="signal",
content=fetched["content"],
filename=fetched["filename"],
content_type=fetched["content_type"],
)
if blob_key:
compose_media_urls.append(
f"/compose/media/blob/?key={quote_plus(str(blob_key))}"
)
return xmpp_attachments, compose_media_urls
async def _relay_signal_inbound_to_xmpp(
self,
*,
identifiers,
relay_text,
xmpp_attachments,
compose_media_urls,
source_uuid,
source_number,
ts,
):
resolved_text_by_session = {}
for identifier in identifiers:
user = identifier.user
session_key = (identifier.user.id, identifier.person.id)
mutate_manips = await sync_to_async(list)(
Manipulation.objects.filter(
group__people=identifier.person,
user=identifier.user,
mode="mutate",
filter_enabled=True,
enabled=True,
)
)
if mutate_manips:
uploaded_urls = []
for manip in mutate_manips:
prompt = replies.generate_mutate_reply_prompt(
relay_text,
None,
manip,
None,
)
result = await ai.run_prompt(
prompt,
manip.ai,
operation="signal_mutate",
)
uploaded_urls = await self.ur.xmpp.client.send_from_external(
user,
identifier,
result,
False,
attachments=xmpp_attachments,
source_ref={
"upstream_message_id": "",
"upstream_author": str(
source_uuid or source_number or ""
),
"upstream_ts": int(ts or 0),
},
)
resolved_text = relay_text
if (not resolved_text) and uploaded_urls:
resolved_text = "\n".join(uploaded_urls)
elif (not resolved_text) and compose_media_urls:
resolved_text = "\n".join(compose_media_urls)
resolved_text_by_session[session_key] = resolved_text
continue
uploaded_urls = await self.ur.xmpp.client.send_from_external(
user,
identifier,
relay_text,
False,
attachments=xmpp_attachments,
source_ref={
"upstream_message_id": "",
"upstream_author": str(source_uuid or source_number or ""),
"upstream_ts": int(ts or 0),
},
)
resolved_text = relay_text
if (not resolved_text) and uploaded_urls:
resolved_text = "\n".join(uploaded_urls)
elif (not resolved_text) and compose_media_urls:
resolved_text = "\n".join(compose_media_urls)
resolved_text_by_session[session_key] = resolved_text
return resolved_text_by_session
async def _process_raw_inbound_event(self, raw_message: str):
try:
@@ -1361,6 +1778,15 @@ class SignalClient(ClientBase):
envelope = payload.get("envelope") or {}
if not isinstance(envelope, dict):
return
group_candidates = _extract_signal_group_identifiers(payload)
preferred_group_id = ""
for candidate in group_candidates:
cleaned = str(candidate or "").strip()
if cleaned.startswith("group."):
preferred_group_id = cleaned
break
if not preferred_group_id and group_candidates:
preferred_group_id = str(group_candidates[0] or "").strip()
sync_sent_message = _get_nested(envelope, ("syncMessage", "sentMessage")) or {}
if isinstance(sync_sent_message, dict) and sync_sent_message:
raw_text = sync_sent_message.get("message")
@@ -1395,6 +1821,7 @@ class SignalClient(ClientBase):
identifiers = await self._resolve_signal_identifiers(
destination_uuid,
destination_number,
group_candidates,
)
if not identifiers:
identifiers = await self._auto_link_single_user_signal_identifier(
@@ -1532,7 +1959,12 @@ class SignalClient(ClientBase):
or str(payload.get("account") or "").strip()
or "self"
)
source_chat_id = destination_number or destination_uuid or sender_key
source_chat_id = (
preferred_group_id
or destination_number
or destination_uuid
or sender_key
)
reply_ref = reply_sync.extract_reply_ref(self.service, payload)
for identifier in identifiers:
session = await history.get_chat_session(
@@ -1599,7 +2031,11 @@ class SignalClient(ClientBase):
):
return
identifiers = await self._resolve_signal_identifiers(source_uuid, source_number)
identifiers = await self._resolve_signal_identifiers(
source_uuid,
source_number,
group_candidates,
)
reaction_payload = _extract_signal_reaction(envelope)
edit_payload = _extract_signal_edit(envelope)
delete_payload = _extract_signal_delete(envelope)
@@ -1620,6 +2056,7 @@ class SignalClient(ClientBase):
identifiers = await self._resolve_signal_identifiers(
destination_uuid,
destination_number,
group_candidates,
)
if not identifiers:
identifiers = await self._auto_link_single_user_signal_identifier(
@@ -1730,7 +2167,13 @@ class SignalClient(ClientBase):
text = _extract_signal_text(
payload, str(data_message.get("message") or "").strip()
)
if not text:
relay_text = text
xmpp_attachments, compose_media_urls = await self._build_xmpp_relay_attachments(
payload
)
if xmpp_attachments and _is_compose_blob_only_text(relay_text):
relay_text = ""
if not relay_text and not xmpp_attachments:
return
ts_raw = (
@@ -1753,8 +2196,17 @@ class SignalClient(ClientBase):
or source_number
or (identifiers[0].identifier if identifiers else "")
)
source_chat_id = source_number or source_uuid or sender_key
source_chat_id = preferred_group_id or source_number or source_uuid or sender_key
reply_ref = reply_sync.extract_reply_ref(self.service, payload)
resolved_text_by_session = await self._relay_signal_inbound_to_xmpp(
identifiers=identifiers,
relay_text=relay_text,
xmpp_attachments=xmpp_attachments,
compose_media_urls=compose_media_urls,
source_uuid=source_uuid,
source_number=source_number,
ts=ts,
)
for identifier in identifiers:
session = await history.get_chat_session(identifier.user, identifier)
@@ -1773,10 +2225,14 @@ class SignalClient(ClientBase):
)()
if exists:
continue
message_text = resolved_text_by_session.get(
(identifier.user.id, identifier.person.id),
relay_text if relay_text else "\n".join(compose_media_urls),
)
local_message = await history.store_message(
session=session,
sender=sender_key,
text=text,
text=message_text,
ts=ts,
outgoing=False,
source_service=self.service,
@@ -1792,7 +2248,7 @@ class SignalClient(ClientBase):
await self.ur.message_received(
self.service,
identifier=identifier,
text=text,
text=message_text,
ts=ts,
payload=payload,
local_message=local_message,
@@ -1809,16 +2265,45 @@ class SignalClient(ClientBase):
if not signal_number:
return
uri = f"ws://{SIGNAL_URL}/v1/receive/{signal_number}"
poll_uri = f"http://{SIGNAL_URL}/v1/receive/{signal_number}"
use_http_polling = False
while not self._stopping:
try:
transport.update_runtime_state(self.service, accounts=[signal_number])
if use_http_polling:
timeout = aiohttp.ClientTimeout(total=15)
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.get(poll_uri) as response:
response.raise_for_status()
payload = await response.json(content_type=None)
if isinstance(payload, dict):
payload = [payload]
if not isinstance(payload, list):
payload = []
for item in payload:
if not isinstance(item, dict):
continue
await self._process_raw_inbound_event(json.dumps(item))
continue
async with websockets.connect(uri, ping_interval=None) as websocket:
async for raw_message in websocket:
await self._process_raw_inbound_event(raw_message)
except asyncio.CancelledError:
raise
except Exception as exc:
if (
not use_http_polling
and "server rejected WebSocket connection: HTTP 200" in str(exc)
):
self.log.info(
"signal raw-receive switching to HTTP polling for %s",
signal_number,
)
use_http_polling = True
continue
self.log.warning("signal raw-receive loop error: %s", exc)
await asyncio.sleep(2)
transport.update_runtime_state(self.service, accounts=[])
def start(self):
self.log.info("Signal client starting...")
@@ -1828,6 +2313,10 @@ class SignalClient(ClientBase):
self._command_task = self.loop.create_task(self._command_loop())
if not self._raw_receive_task or self._raw_receive_task.done():
self._raw_receive_task = self.loop.create_task(self._raw_receive_loop())
# Use direct websocket receive loop as primary ingestion path.
# signalbot's internal receive consumer can compete for the same stream
# and starve inbound events in this deployment, so we keep it disabled.
if not self._catalog_refresh_task or self._catalog_refresh_task.done():
self._catalog_refresh_task = self.loop.create_task(
self._refresh_runtime_catalog_safe()
)
# Keep signalbot's internal receive consumer disabled to avoid competing
# consumers. The raw loop adapts between websocket and HTTP polling
# depending on the deployed signal-cli-rest-api behavior.