Harden security
This commit is contained in:
@@ -16,6 +16,22 @@ SIGNAL_UUID_PATTERN = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _safe_parse_send_response(payload_value) -> int | bool:
|
||||
payload = payload_value
|
||||
if isinstance(payload_value, str):
|
||||
try:
|
||||
payload = orjson.loads(payload_value)
|
||||
except orjson.JSONDecodeError:
|
||||
return False
|
||||
if not isinstance(payload, dict):
|
||||
return False
|
||||
try:
|
||||
ts = payload.get("timestamp")
|
||||
return int(ts) if ts else False
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def normalize_signal_recipient(recipient: str) -> str:
|
||||
raw = str(recipient or "").strip()
|
||||
if not raw:
|
||||
@@ -395,8 +411,8 @@ def send_message_raw_sync(recipient_uuid, text=None, attachments=None):
|
||||
response.status_code == status.HTTP_201_CREATED
|
||||
): # Signal server returns 201 on success
|
||||
try:
|
||||
ts = orjson.loads(response.text).get("timestamp", None)
|
||||
return ts if ts else False
|
||||
except orjson.JSONDecodeError:
|
||||
return False
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
payload = {}
|
||||
return _safe_parse_send_response(payload)
|
||||
return False # If response status is not 201
|
||||
|
||||
@@ -17,6 +17,10 @@ from django.core.cache import cache
|
||||
|
||||
from core.clients import signalapi
|
||||
from core.messaging import media_bridge
|
||||
from core.security.attachments import (
|
||||
validate_attachment_metadata,
|
||||
validate_attachment_url,
|
||||
)
|
||||
from core.transports.capabilities import supports, unsupported_reason
|
||||
from core.util import logs
|
||||
|
||||
@@ -665,17 +669,21 @@ async def _normalize_gateway_attachment(service: str, row: dict, session):
|
||||
if isinstance(content, memoryview):
|
||||
content = content.tobytes()
|
||||
if isinstance(content, bytes):
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=normalized.get("filename") or "attachment.bin",
|
||||
content_type=normalized.get("content_type") or "application/octet-stream",
|
||||
size=normalized.get("size") or len(content),
|
||||
)
|
||||
blob_key = media_bridge.put_blob(
|
||||
service=service,
|
||||
content=content,
|
||||
filename=normalized.get("filename") or "attachment.bin",
|
||||
content_type=normalized.get("content_type") or "application/octet-stream",
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
return {
|
||||
"blob_key": blob_key,
|
||||
"filename": normalized.get("filename") or "attachment.bin",
|
||||
"content_type": normalized.get("content_type")
|
||||
or "application/octet-stream",
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size": normalized.get("size") or len(content),
|
||||
}
|
||||
|
||||
@@ -685,33 +693,39 @@ async def _normalize_gateway_attachment(service: str, row: dict, session):
|
||||
source_url = normalized.get("url")
|
||||
if source_url:
|
||||
try:
|
||||
async with session.get(source_url) as response:
|
||||
safe_url = validate_attachment_url(source_url)
|
||||
async with session.get(safe_url) as response:
|
||||
if response.status == 200:
|
||||
payload = await response.read()
|
||||
blob_key = media_bridge.put_blob(
|
||||
service=service,
|
||||
content=payload,
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=normalized.get("filename")
|
||||
or source_url.rstrip("/").split("/")[-1]
|
||||
or safe_url.rstrip("/").split("/")[-1]
|
||||
or "attachment.bin",
|
||||
content_type=normalized.get("content_type")
|
||||
or response.headers.get(
|
||||
"Content-Type", "application/octet-stream"
|
||||
),
|
||||
size=normalized.get("size") or len(payload),
|
||||
)
|
||||
blob_key = media_bridge.put_blob(
|
||||
service=service,
|
||||
content=payload,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
return {
|
||||
"blob_key": blob_key,
|
||||
"filename": normalized.get("filename")
|
||||
or source_url.rstrip("/").split("/")[-1]
|
||||
or "attachment.bin",
|
||||
"content_type": normalized.get("content_type")
|
||||
or response.headers.get(
|
||||
"Content-Type", "application/octet-stream"
|
||||
),
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size": normalized.get("size") or len(payload),
|
||||
}
|
||||
except Exception:
|
||||
log.warning("%s attachment fetch failed for %s", service, source_url)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"%s attachment fetch failed for %s: %s",
|
||||
service,
|
||||
source_url,
|
||||
exc,
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
@@ -1074,21 +1088,27 @@ async def fetch_attachment(service: str, attachment_ref: dict):
|
||||
if blob_key:
|
||||
return media_bridge.get_blob(blob_key)
|
||||
if direct_url:
|
||||
safe_url = validate_attachment_url(direct_url)
|
||||
timeout = aiohttp.ClientTimeout(total=20)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(direct_url) as response:
|
||||
async with session.get(safe_url) as response:
|
||||
if response.status != 200:
|
||||
return None
|
||||
content = await response.read()
|
||||
return {
|
||||
"content": content,
|
||||
"content_type": response.headers.get(
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=attachment_ref.get("filename")
|
||||
or safe_url.rstrip("/").split("/")[-1]
|
||||
or "attachment.bin",
|
||||
content_type=response.headers.get(
|
||||
"Content-Type",
|
||||
attachment_ref.get("content_type", "application/octet-stream"),
|
||||
),
|
||||
"filename": attachment_ref.get("filename")
|
||||
or direct_url.rstrip("/").split("/")[-1]
|
||||
or "attachment.bin",
|
||||
size=len(content),
|
||||
)
|
||||
return {
|
||||
"content": content,
|
||||
"content_type": content_type,
|
||||
"filename": filename,
|
||||
"size": len(content),
|
||||
}
|
||||
return None
|
||||
|
||||
@@ -17,6 +17,10 @@ from django.core.cache import cache
|
||||
from core.clients import ClientBase, transport
|
||||
from core.messaging import history, media_bridge, reply_sync
|
||||
from core.models import Message, PersonIdentifier, PlatformChatLink
|
||||
from core.security.attachments import (
|
||||
validate_attachment_metadata,
|
||||
validate_attachment_url,
|
||||
)
|
||||
|
||||
try:
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
@@ -3141,31 +3145,42 @@ class WhatsAppClient(ClientBase):
|
||||
if isinstance(content, memoryview):
|
||||
content = content.tobytes()
|
||||
if isinstance(content, bytes):
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=(attachment or {}).get("filename") or "attachment.bin",
|
||||
content_type=(attachment or {}).get("content_type")
|
||||
or "application/octet-stream",
|
||||
size=len(content),
|
||||
)
|
||||
return {
|
||||
"content": content,
|
||||
"filename": (attachment or {}).get("filename") or "attachment.bin",
|
||||
"content_type": (attachment or {}).get("content_type")
|
||||
or "application/octet-stream",
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size": len(content),
|
||||
}
|
||||
|
||||
url = (attachment or {}).get("url")
|
||||
if url:
|
||||
safe_url = validate_attachment_url(url)
|
||||
timeout = aiohttp.ClientTimeout(total=20)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(url) as response:
|
||||
async with session.get(safe_url) as response:
|
||||
if response.status != 200:
|
||||
return None
|
||||
payload = await response.read()
|
||||
return {
|
||||
"content": payload,
|
||||
"filename": (attachment or {}).get("filename")
|
||||
or url.rstrip("/").split("/")[-1]
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=(attachment or {}).get("filename")
|
||||
or safe_url.rstrip("/").split("/")[-1]
|
||||
or "attachment.bin",
|
||||
"content_type": (attachment or {}).get("content_type")
|
||||
content_type=(attachment or {}).get("content_type")
|
||||
or response.headers.get(
|
||||
"Content-Type", "application/octet-stream"
|
||||
),
|
||||
size=len(payload),
|
||||
)
|
||||
return {
|
||||
"content": payload,
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size": len(payload),
|
||||
}
|
||||
return None
|
||||
@@ -3320,11 +3335,19 @@ class WhatsAppClient(ClientBase):
|
||||
payload = await self._fetch_attachment_payload(attachment)
|
||||
if not payload:
|
||||
continue
|
||||
mime = str(
|
||||
payload.get("content_type") or "application/octet-stream"
|
||||
).lower()
|
||||
data = payload.get("content") or b""
|
||||
filename = payload.get("filename") or "attachment.bin"
|
||||
try:
|
||||
filename, mime = validate_attachment_metadata(
|
||||
filename=payload.get("filename") or "attachment.bin",
|
||||
content_type=payload.get("content_type")
|
||||
or "application/octet-stream",
|
||||
size=payload.get("size")
|
||||
or (len(data) if isinstance(data, (bytes, bytearray)) else 0),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.log.warning("whatsapp blocked attachment: %s", exc)
|
||||
continue
|
||||
mime = str(mime).lower()
|
||||
attachment_target = jid_obj if jid_obj is not None else jid
|
||||
send_method = "document"
|
||||
if mime.startswith("image/") and hasattr(self._client, "send_image"):
|
||||
@@ -3372,7 +3395,7 @@ class WhatsAppClient(ClientBase):
|
||||
sent_ts,
|
||||
self._normalize_timestamp(self._pluck(response, "Timestamp") or 0),
|
||||
)
|
||||
await _record_bridge(response, sent_ts, body_hint=filename)
|
||||
await _record_bridge(response, sent_ts, body_hint="attachment")
|
||||
sent_any = True
|
||||
if getattr(settings, "WHATSAPP_DEBUG", False):
|
||||
self.log.debug(
|
||||
|
||||
@@ -30,6 +30,10 @@ from core.models import (
|
||||
User,
|
||||
WorkspaceConversation,
|
||||
)
|
||||
from core.security.attachments import (
|
||||
validate_attachment_metadata,
|
||||
validate_attachment_url,
|
||||
)
|
||||
from core.util import logs
|
||||
|
||||
URL_PATTERN = re.compile(r"https?://[^\s<>'\"\\]+")
|
||||
@@ -49,9 +53,8 @@ def _filename_from_url(url_value):
|
||||
|
||||
|
||||
def _content_type_from_filename_or_url(url_value, default="application/octet-stream"):
|
||||
filename = _filename_from_url(url_value)
|
||||
guessed, _ = mimetypes.guess_type(filename)
|
||||
return guessed or default
|
||||
_ = url_value
|
||||
return str(default or "application/octet-stream")
|
||||
|
||||
|
||||
def _extract_xml_attachment_urls(message_stanza):
|
||||
@@ -1013,13 +1016,21 @@ class XMPPComponent(ComponentXMPP):
|
||||
url_value = _clean_url(att.attrib.get("url"))
|
||||
if not url_value:
|
||||
continue
|
||||
try:
|
||||
safe_url = validate_attachment_url(url_value)
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=att.attrib.get("filename") or _filename_from_url(safe_url),
|
||||
content_type=att.attrib.get("content_type")
|
||||
or "application/octet-stream",
|
||||
)
|
||||
except Exception as exc:
|
||||
self.log.warning("xmpp dropped unsafe attachment url=%s: %s", url_value, exc)
|
||||
continue
|
||||
attachments.append(
|
||||
{
|
||||
"url": url_value,
|
||||
"filename": att.attrib.get("filename")
|
||||
or _filename_from_url(url_value),
|
||||
"content_type": att.attrib.get("content_type")
|
||||
or "application/octet-stream",
|
||||
"url": safe_url,
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1028,11 +1039,19 @@ class XMPPComponent(ComponentXMPP):
|
||||
url_value = _clean_url(oob.text)
|
||||
if not url_value:
|
||||
continue
|
||||
guessed_content_type = _content_type_from_filename_or_url(url_value)
|
||||
try:
|
||||
safe_url = validate_attachment_url(url_value)
|
||||
filename, guessed_content_type = validate_attachment_metadata(
|
||||
filename=_filename_from_url(safe_url),
|
||||
content_type=_content_type_from_filename_or_url(safe_url),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.log.warning("xmpp dropped unsafe oob url=%s: %s", url_value, exc)
|
||||
continue
|
||||
attachments.append(
|
||||
{
|
||||
"url": url_value,
|
||||
"filename": _filename_from_url(url_value),
|
||||
"url": safe_url,
|
||||
"filename": filename,
|
||||
"content_type": guessed_content_type,
|
||||
}
|
||||
)
|
||||
@@ -1043,11 +1062,19 @@ class XMPPComponent(ComponentXMPP):
|
||||
for url_value in extracted_urls:
|
||||
if url_value in existing_urls:
|
||||
continue
|
||||
guessed_content_type = _content_type_from_filename_or_url(url_value)
|
||||
try:
|
||||
safe_url = validate_attachment_url(url_value)
|
||||
filename, guessed_content_type = validate_attachment_metadata(
|
||||
filename=_filename_from_url(safe_url),
|
||||
content_type=_content_type_from_filename_or_url(safe_url),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.log.warning("xmpp dropped extracted unsafe url=%s: %s", url_value, exc)
|
||||
continue
|
||||
attachments.append(
|
||||
{
|
||||
"url": url_value,
|
||||
"filename": _filename_from_url(url_value),
|
||||
"url": safe_url,
|
||||
"filename": filename,
|
||||
"content_type": guessed_content_type,
|
||||
}
|
||||
)
|
||||
@@ -1397,7 +1424,16 @@ class XMPPComponent(ComponentXMPP):
|
||||
async def upload_and_send(self, att, upload_slot, recipient_jid, sender_jid):
|
||||
"""Uploads a file and immediately sends the corresponding XMPP message."""
|
||||
upload_url, put_url, auth_header = upload_slot
|
||||
headers = {"Content-Type": att["content_type"]}
|
||||
try:
|
||||
filename, content_type = validate_attachment_metadata(
|
||||
filename=att.get("filename"),
|
||||
content_type=att.get("content_type"),
|
||||
size=att.get("size"),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.log.warning("xmpp blocked outbound attachment: %s", exc)
|
||||
return None
|
||||
headers = {"Content-Type": content_type}
|
||||
if auth_header:
|
||||
headers["Authorization"] = auth_header
|
||||
|
||||
@@ -1412,7 +1448,7 @@ class XMPPComponent(ComponentXMPP):
|
||||
)
|
||||
return None
|
||||
self.log.debug(
|
||||
"Successfully uploaded %s to %s", att["filename"], upload_url
|
||||
"Successfully uploaded %s to %s", filename, upload_url
|
||||
)
|
||||
|
||||
# Send XMPP message immediately after successful upload
|
||||
|
||||
@@ -140,7 +140,7 @@ class ManticoreMemorySearchBackend(BaseMemorySearchBackend):
|
||||
|
||||
def __init__(self):
|
||||
self.base_url = str(
|
||||
getattr(settings, "MANTICORE_HTTP_URL", "http://127.0.0.1:9308")
|
||||
getattr(settings, "MANTICORE_HTTP_URL", "http://localhost:9308")
|
||||
).rstrip("/")
|
||||
self.table = str(
|
||||
getattr(settings, "MANTICORE_MEMORY_TABLE", "gia_memory_items")
|
||||
|
||||
2
core/security/__init__.py
Normal file
2
core/security/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""Security helpers shared across transport adapters."""
|
||||
|
||||
169
core/security/attachments.py
Normal file
169
core/security/attachments.py
Normal file
@@ -0,0 +1,169 @@
|
||||
import ipaddress
|
||||
import os
|
||||
import socket
|
||||
from fnmatch import fnmatch
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from core.util import logs
|
||||
|
||||
log = logs.get_logger("security.attachments")
|
||||
|
||||
_DEFAULT_ALLOWED_MIME_PATTERNS = (
|
||||
"image/*",
|
||||
"video/*",
|
||||
"audio/*",
|
||||
"text/plain",
|
||||
"application/pdf",
|
||||
"application/zip",
|
||||
"application/json",
|
||||
"application/msword",
|
||||
"application/vnd.*",
|
||||
)
|
||||
|
||||
_DEFAULT_BLOCKED_MIME_TYPES = {
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"application/javascript",
|
||||
"text/javascript",
|
||||
"application/x-sh",
|
||||
"application/x-msdownload",
|
||||
"application/x-dosexec",
|
||||
"application/x-executable",
|
||||
}
|
||||
|
||||
_BLOCKED_FILENAME_EXTENSIONS = {
|
||||
".exe",
|
||||
".dll",
|
||||
".bat",
|
||||
".cmd",
|
||||
".msi",
|
||||
".sh",
|
||||
".ps1",
|
||||
".jar",
|
||||
".hta",
|
||||
}
|
||||
|
||||
|
||||
def _strip_content_type(value: str | None) -> str:
|
||||
raw = str(value or "").strip().lower()
|
||||
if not raw:
|
||||
return ""
|
||||
return raw.split(";", 1)[0].strip()
|
||||
|
||||
|
||||
def normalize_filename(value: str | None, fallback: str = "attachment.bin") -> str:
|
||||
name = os.path.basename(str(value or "").strip())
|
||||
name = name.replace("\x00", "").strip()
|
||||
return name or fallback
|
||||
|
||||
|
||||
def normalized_content_type(
|
||||
value: str | None,
|
||||
*,
|
||||
fallback: str = "application/octet-stream",
|
||||
) -> str:
|
||||
clean = _strip_content_type(value)
|
||||
return clean or fallback
|
||||
|
||||
|
||||
def _allowed_mime_patterns() -> tuple[str, ...]:
|
||||
configured = getattr(settings, "ATTACHMENT_ALLOWED_MIME_TYPES", None)
|
||||
if isinstance(configured, (list, tuple)):
|
||||
patterns = tuple(str(item or "").strip().lower() for item in configured if item)
|
||||
if patterns:
|
||||
return patterns
|
||||
return _DEFAULT_ALLOWED_MIME_PATTERNS
|
||||
|
||||
|
||||
def _blocked_mime_types() -> set[str]:
|
||||
configured = getattr(settings, "ATTACHMENT_BLOCKED_MIME_TYPES", None)
|
||||
if isinstance(configured, (list, tuple, set)):
|
||||
values = {str(item or "").strip().lower() for item in configured if item}
|
||||
if values:
|
||||
return values
|
||||
return set(_DEFAULT_BLOCKED_MIME_TYPES)
|
||||
|
||||
|
||||
def validate_attachment_metadata(
|
||||
*,
|
||||
filename: str | None,
|
||||
content_type: str | None,
|
||||
size: int | None = None,
|
||||
) -> tuple[str, str]:
|
||||
normalized_name = normalize_filename(filename)
|
||||
normalized_type = normalized_content_type(content_type)
|
||||
_, ext = os.path.splitext(normalized_name.lower())
|
||||
if ext in _BLOCKED_FILENAME_EXTENSIONS:
|
||||
raise ValueError(f"blocked_filename_extension:{ext}")
|
||||
if normalized_type in _blocked_mime_types():
|
||||
raise ValueError(f"blocked_mime_type:{normalized_type}")
|
||||
|
||||
allow_unmatched = bool(getattr(settings, "ATTACHMENT_ALLOW_UNKNOWN_MIME", False))
|
||||
if not any(fnmatch(normalized_type, pattern) for pattern in _allowed_mime_patterns()):
|
||||
if not allow_unmatched:
|
||||
raise ValueError(f"unsupported_mime_type:{normalized_type}")
|
||||
|
||||
max_bytes = int(getattr(settings, "ATTACHMENT_MAX_BYTES", 25 * 1024 * 1024) or 0)
|
||||
if size and max_bytes > 0 and int(size) > max_bytes:
|
||||
raise ValueError(f"attachment_too_large:{size}>{max_bytes}")
|
||||
return normalized_name, normalized_type
|
||||
|
||||
|
||||
def _host_is_private(hostname: str) -> bool:
|
||||
if not hostname:
|
||||
return True
|
||||
lower = hostname.strip().lower()
|
||||
if lower in {"localhost", "localhost.localdomain"} or lower.endswith(".local"):
|
||||
return True
|
||||
try:
|
||||
addr = ipaddress.ip_address(lower)
|
||||
return (
|
||||
addr.is_private
|
||||
or addr.is_loopback
|
||||
or addr.is_link_local
|
||||
or addr.is_multicast
|
||||
or addr.is_reserved
|
||||
or addr.is_unspecified
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
infos = socket.getaddrinfo(lower, None)
|
||||
except Exception:
|
||||
return True
|
||||
for info in infos:
|
||||
ip = info[4][0]
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return True
|
||||
if (
|
||||
addr.is_private
|
||||
or addr.is_loopback
|
||||
or addr.is_link_local
|
||||
or addr.is_multicast
|
||||
or addr.is_reserved
|
||||
or addr.is_unspecified
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def validate_attachment_url(url_value: str | None) -> str:
|
||||
url_text = str(url_value or "").strip()
|
||||
parsed = urlparse(url_text)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
raise ValueError("unsupported_url_scheme")
|
||||
if not parsed.netloc:
|
||||
raise ValueError("attachment_url_missing_host")
|
||||
if parsed.username or parsed.password:
|
||||
raise ValueError("attachment_url_embedded_credentials")
|
||||
|
||||
allow_private = bool(getattr(settings, "ATTACHMENT_ALLOW_PRIVATE_URLS", False))
|
||||
host = str(parsed.hostname or "").strip()
|
||||
if not allow_private and _host_is_private(host):
|
||||
raise ValueError(f"attachment_private_host_blocked:{host or '-'}")
|
||||
return url_text
|
||||
36
core/tests/test_attachment_security.py
Normal file
36
core/tests/test_attachment_security.py
Normal file
@@ -0,0 +1,36 @@
|
||||
from django.test import SimpleTestCase, override_settings
|
||||
|
||||
from core.security.attachments import (
|
||||
validate_attachment_metadata,
|
||||
validate_attachment_url,
|
||||
)
|
||||
|
||||
|
||||
class AttachmentSecurityTests(SimpleTestCase):
|
||||
def test_blocks_html_payload(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_attachment_metadata(
|
||||
filename="payload.html",
|
||||
content_type="text/html",
|
||||
size=32,
|
||||
)
|
||||
|
||||
@override_settings(ATTACHMENT_MAX_BYTES=10)
|
||||
def test_blocks_oversized_payload(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_attachment_metadata(
|
||||
filename="dump.bin",
|
||||
content_type="application/octet-stream",
|
||||
size=32,
|
||||
)
|
||||
|
||||
def test_blocks_private_url_by_default(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_attachment_url("http://localhost/internal")
|
||||
|
||||
@override_settings(ATTACHMENT_ALLOW_PRIVATE_URLS=True)
|
||||
def test_allows_private_url_when_explicitly_enabled(self):
|
||||
self.assertEqual(
|
||||
"http://localhost/internal",
|
||||
validate_attachment_url("http://localhost/internal"),
|
||||
)
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timezone as dt_timezone
|
||||
from decimal import Decimal, InvalidOperation
|
||||
import re
|
||||
from typing import Any, Callable
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -20,6 +21,51 @@ from mixins.views import ObjectList
|
||||
|
||||
from core.models import Group, Manipulation, Message, Person, PersonIdentifier, Persona
|
||||
|
||||
_QUERY_MAX_LEN = 400
|
||||
_QUERY_ALLOWED_PATTERN = re.compile(r"[\w\s@\-\+\.:,#/]+", re.UNICODE)
|
||||
|
||||
|
||||
def _sanitize_search_query(value: str) -> str:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
trimmed = raw[:_QUERY_MAX_LEN]
|
||||
cleaned = "".join(_QUERY_ALLOWED_PATTERN.findall(trimmed)).strip()
|
||||
return cleaned
|
||||
|
||||
|
||||
def _safe_page_number(value: Any) -> int:
|
||||
try:
|
||||
page_value = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 1
|
||||
return max(1, page_value)
|
||||
|
||||
|
||||
def _safe_query_param(request, key: str, default: str = "") -> str:
|
||||
raw = request.GET.get(key, default)
|
||||
return str(raw or default).strip()
|
||||
|
||||
|
||||
def _sanitize_query_state(raw: dict[str, Any]) -> dict[str, str]:
|
||||
cleaned: dict[str, str] = {}
|
||||
for key, value in (raw or {}).items():
|
||||
key_text = str(key or "").strip()
|
||||
if not key_text or len(key_text) > 80:
|
||||
continue
|
||||
value_text = str(value or "").strip()
|
||||
if not value_text:
|
||||
continue
|
||||
if key_text in {"q", "query"}:
|
||||
value_text = _sanitize_search_query(value_text)
|
||||
elif key_text == "page":
|
||||
value_text = str(_safe_page_number(value_text))
|
||||
else:
|
||||
value_text = value_text[:200]
|
||||
if value_text:
|
||||
cleaned[key_text] = value_text
|
||||
return cleaned
|
||||
|
||||
|
||||
def _context_type(request_type: str) -> str:
|
||||
return "modal" if request_type == "page" else request_type
|
||||
@@ -561,12 +607,14 @@ class OSINTListBase(ObjectList):
|
||||
return lookups
|
||||
|
||||
def _query_dict(self) -> dict[str, Any]:
|
||||
return {k: v for k, v in self.request.GET.items() if v not in {"", None}}
|
||||
return _sanitize_query_state(
|
||||
{k: v for k, v in self.request.GET.items() if v not in {"", None}}
|
||||
)
|
||||
|
||||
def _apply_list_search(
|
||||
self, queryset: models.QuerySet, scope: OsintScopeConfig
|
||||
) -> models.QuerySet:
|
||||
query = self.request.GET.get("q", "").strip()
|
||||
query = _sanitize_search_query(self.request.GET.get("q", ""))
|
||||
if not query:
|
||||
return queryset
|
||||
|
||||
@@ -721,14 +769,16 @@ class OSINTListBase(ObjectList):
|
||||
}
|
||||
|
||||
if page_obj.has_previous():
|
||||
previous_page = _safe_page_number(page_obj.previous_page_number())
|
||||
pagination["previous_url"] = _url_with_query(
|
||||
list_url,
|
||||
_merge_query(query_state, page=page_obj.previous_page_number()),
|
||||
{"page": previous_page},
|
||||
)
|
||||
if page_obj.has_next():
|
||||
next_page = _safe_page_number(page_obj.next_page_number())
|
||||
pagination["next_url"] = _url_with_query(
|
||||
list_url,
|
||||
_merge_query(query_state, page=page_obj.next_page_number()),
|
||||
{"page": next_page},
|
||||
)
|
||||
|
||||
for entry in page_obj.paginator.get_elided_page_range(page_obj.number):
|
||||
@@ -742,7 +792,7 @@ class OSINTListBase(ObjectList):
|
||||
"current": entry == page_obj.number,
|
||||
"url": _url_with_query(
|
||||
list_url,
|
||||
_merge_query(query_state, page=entry),
|
||||
{"page": _safe_page_number(entry)},
|
||||
),
|
||||
}
|
||||
)
|
||||
@@ -834,7 +884,7 @@ class OSINTSearch(LoginRequiredMixin, View):
|
||||
S - Size, I - Index, Q - Query, T - Tags, S - Source, R - Ranges,
|
||||
S - Sort, S - Sentiment, A - Annotate, D - Dedup, R - Reverse.
|
||||
"""
|
||||
query = str(request.GET.get("q") or "").strip()
|
||||
query = _sanitize_search_query(_safe_query_param(request, "q", ""))
|
||||
tags = tuple(
|
||||
token[4:].strip()
|
||||
for token in query.split()
|
||||
@@ -845,15 +895,16 @@ class OSINTSearch(LoginRequiredMixin, View):
|
||||
index=self._scope_key(request.GET.get("scope")),
|
||||
query=query,
|
||||
tags=tags,
|
||||
source=str(request.GET.get("source") or "all").strip().lower() or "all",
|
||||
date_from=str(request.GET.get("date_from") or "").strip(),
|
||||
date_to=str(request.GET.get("date_to") or "").strip(),
|
||||
sort_mode=str(request.GET.get("sort_mode") or "relevance").strip().lower(),
|
||||
sentiment_min=str(request.GET.get("sentiment_min") or "").strip(),
|
||||
sentiment_max=str(request.GET.get("sentiment_max") or "").strip(),
|
||||
annotate=str(request.GET.get("annotate") or "1").strip() not in {"0", "false", "off"},
|
||||
dedup=str(request.GET.get("dedup") or "").strip() in {"1", "true", "on"},
|
||||
reverse=str(request.GET.get("reverse") or "").strip() in {"1", "true", "on"},
|
||||
source=_safe_query_param(request, "source", "all").lower() or "all",
|
||||
date_from=_safe_query_param(request, "date_from", ""),
|
||||
date_to=_safe_query_param(request, "date_to", ""),
|
||||
sort_mode=_safe_query_param(request, "sort_mode", "relevance").lower(),
|
||||
sentiment_min=_safe_query_param(request, "sentiment_min", ""),
|
||||
sentiment_max=_safe_query_param(request, "sentiment_max", ""),
|
||||
annotate=_safe_query_param(request, "annotate", "1")
|
||||
not in {"0", "false", "off"},
|
||||
dedup=_safe_query_param(request, "dedup", "") in {"1", "true", "on"},
|
||||
reverse=_safe_query_param(request, "reverse", "") in {"1", "true", "on"},
|
||||
)
|
||||
|
||||
def _parse_date_boundaries(self, plan: "OSINTSearch.SearchPlan") -> tuple[datetime | None, datetime | None]:
|
||||
@@ -1069,7 +1120,9 @@ class OSINTSearch(LoginRequiredMixin, View):
|
||||
return "all"
|
||||
|
||||
def _query_state(self, request) -> dict[str, Any]:
|
||||
return {k: v for k, v in request.GET.items() if v not in {None, ""}}
|
||||
return _sanitize_query_state(
|
||||
{k: v for k, v in request.GET.items() if v not in {None, ""}}
|
||||
)
|
||||
|
||||
def _apply_common_filters(
|
||||
self,
|
||||
@@ -1359,14 +1412,16 @@ class OSINTSearch(LoginRequiredMixin, View):
|
||||
}
|
||||
|
||||
if page_obj.has_previous():
|
||||
previous_page = _safe_page_number(page_obj.previous_page_number())
|
||||
pagination["previous_url"] = _url_with_query(
|
||||
list_url,
|
||||
_merge_query(query_state, page=page_obj.previous_page_number()),
|
||||
{"page": previous_page},
|
||||
)
|
||||
if page_obj.has_next():
|
||||
next_page = _safe_page_number(page_obj.next_page_number())
|
||||
pagination["next_url"] = _url_with_query(
|
||||
list_url,
|
||||
_merge_query(query_state, page=page_obj.next_page_number()),
|
||||
{"page": next_page},
|
||||
)
|
||||
|
||||
for entry in page_obj.paginator.get_elided_page_range(page_obj.number):
|
||||
@@ -1380,7 +1435,7 @@ class OSINTSearch(LoginRequiredMixin, View):
|
||||
"current": entry == page_obj.number,
|
||||
"url": _url_with_query(
|
||||
list_url,
|
||||
_merge_query(query_state, page=entry),
|
||||
{"page": _safe_page_number(entry)},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -17,6 +17,28 @@ from core.presence import latest_state_for_people
|
||||
from core.views.manage.permissions import SuperUserRequiredMixin
|
||||
|
||||
|
||||
def _safe_json_list(text_value):
|
||||
try:
|
||||
payload = orjson.loads(text_value)
|
||||
except orjson.JSONDecodeError:
|
||||
return []
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
|
||||
def _sanitize_signal_rows(rows):
|
||||
safe_rows = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
safe_row = {}
|
||||
for key, value in row.items():
|
||||
if isinstance(key, str) and len(key) <= 100:
|
||||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||
safe_row[key] = value
|
||||
safe_rows.append(safe_row)
|
||||
return safe_rows
|
||||
|
||||
|
||||
class CustomObjectRead(ObjectRead):
|
||||
def post(self, request, *args, **kwargs):
|
||||
self.request = request
|
||||
@@ -171,21 +193,28 @@ class SignalContactsList(SuperUserRequiredMixin, ObjectList):
|
||||
list_url_args = ["type", "pk"]
|
||||
|
||||
def get_queryset(self, *args, **kwargs):
|
||||
# url = signal:8080/v1/accounts
|
||||
# /v1/configuration/{number}/settings
|
||||
# /v1/identities/{number}
|
||||
# /v1/contacts/{number}
|
||||
# response = requests.get(
|
||||
# f"http://signal:8080/v1/configuration/{self.kwargs['pk']}/settings"
|
||||
# )
|
||||
# config = orjson.loads(response.text)
|
||||
|
||||
base = getattr(settings, "SIGNAL_HTTP_URL", "http://signal:8080").rstrip("/")
|
||||
response = requests.get(f"{base}/v1/identities/{self.kwargs['pk']}")
|
||||
identities = orjson.loads(response.text)
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{base}/v1/identities/{self.kwargs['pk']}", timeout=15
|
||||
)
|
||||
response.raise_for_status()
|
||||
identities = _sanitize_signal_rows(response.json() or [])
|
||||
except requests.RequestException:
|
||||
identities = []
|
||||
except ValueError:
|
||||
identities = []
|
||||
|
||||
response = requests.get(f"{base}/v1/contacts/{self.kwargs['pk']}")
|
||||
contacts = orjson.loads(response.text)
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{base}/v1/contacts/{self.kwargs['pk']}", timeout=15
|
||||
)
|
||||
response.raise_for_status()
|
||||
contacts = _sanitize_signal_rows(response.json() or [])
|
||||
except requests.RequestException:
|
||||
contacts = []
|
||||
except ValueError:
|
||||
contacts = []
|
||||
|
||||
# add identities to contacts
|
||||
for contact in contacts:
|
||||
|
||||
Reference in New Issue
Block a user