from __future__ import annotations from dataclasses import dataclass, field from datetime import date, datetime, timezone as dt_timezone from decimal import Decimal, InvalidOperation import re from typing import Any, Callable from urllib.parse import urlencode from django.contrib.auth.mixins import LoginRequiredMixin from django.core.exceptions import FieldDoesNotExist from django.core.paginator import Paginator from django.db import models from django.db.models import Q from django.http import HttpResponseBadRequest from django.shortcuts import render from django.urls import reverse from django.utils import timezone from django.views import View from mixins.views import ObjectList from core.models import Group, Manipulation, Message, Person, PersonIdentifier, Persona _QUERY_MAX_LEN = 400 _QUERY_ALLOWED_PATTERN = re.compile(r"[\w\s@\-\+\.:,#/]+", re.UNICODE) def _sanitize_search_query(value: str) -> str: raw = str(value or "").strip() if not raw: return "" trimmed = raw[:_QUERY_MAX_LEN] cleaned = "".join(_QUERY_ALLOWED_PATTERN.findall(trimmed)).strip() return cleaned def _safe_page_number(value: Any) -> int: try: page_value = int(value) except (TypeError, ValueError): return 1 return max(1, page_value) def _safe_query_param(request, key: str, default: str = "") -> str: raw = request.GET.get(key, default) return str(raw or default).strip() def _sanitize_query_state(raw: dict[str, Any]) -> dict[str, str]: cleaned: dict[str, str] = {} for key, value in (raw or {}).items(): key_text = str(key or "").strip() if not key_text or len(key_text) > 80: continue value_text = str(value or "").strip() if not value_text: continue if key_text in {"q", "query"}: value_text = _sanitize_search_query(value_text) elif key_text == "page": value_text = str(_safe_page_number(value_text)) else: value_text = value_text[:200] if value_text: cleaned[key_text] = value_text return cleaned def _context_type(request_type: str) -> str: return "modal" if request_type == "page" else request_type def _parse_bool(raw: str) -> bool | None: lowered = raw.strip().lower() truthy = {"1", "true", "yes", "y", "on", "enabled"} falsy = {"0", "false", "no", "n", "off", "disabled"} if lowered in truthy: return True if lowered in falsy: return False return None def _preferred_related_text_field(model: type[models.Model]) -> str | None: preferred = ("name", "alias", "title", "identifier", "model") model_fields = {f.name: f for f in model._meta.get_fields()} for candidate in preferred: field = model_fields.get(candidate) if isinstance(field, (models.CharField, models.TextField)): return candidate return None def _column_field_name(column: "OsintColumn") -> str: if column.search_lookup: return str(column.search_lookup).split("__", 1)[0] if column.sort_field: return str(column.sort_field).split("__", 1)[0] return str(column.key) def _safe_icon_class(raw: str | None, default: str) -> str: icon_class = str(raw or "").strip() if not icon_class: return default cleaned_parts = [] for part in icon_class.split(): if not part: continue if all(ch.isalnum() or ch in {"-", "_"} for ch in part): cleaned_parts.append(part) if not cleaned_parts: return default return " ".join(cleaned_parts) def _url_with_query(base_url: str, query: dict[str, Any]) -> str: params = {} for key, value in query.items(): if value is None: continue value_str = str(value).strip() if value_str == "": continue params[key] = value_str if not params: return base_url return f"{base_url}?{urlencode(params)}" def _merge_query(current_query: dict[str, Any], **updates: Any) -> dict[str, Any]: merged = dict(current_query) for key, value in updates.items(): if value is None or str(value).strip() == "": merged.pop(key, None) continue merged[key] = value return merged @dataclass(frozen=True) class OsintColumn: key: str label: str accessor: Callable[[Any], Any] sort_field: str | None = None search_lookup: str | None = None kind: str = "text" @dataclass(frozen=True) class OsintScopeConfig: key: str title: str model: type[models.Model] list_url_name: str update_url_name: str delete_url_name: str default_sort: str columns: tuple[OsintColumn, ...] select_related: tuple[str, ...] = () prefetch_related: tuple[str, ...] = () delete_label: Callable[[Any], str] = str extra_actions: Callable[[Any, str], list[dict[str, Any]]] = field( default_factory=lambda: (lambda _item, _type: []) ) search_lookups: tuple[str, ...] = () def _person_extra_actions(item: Person, _request_type: str) -> list[dict[str, Any]]: return [ { "mode": "link", "url": reverse( "person_identifiers", kwargs={"type": "page", "person": item.id}, ), "icon": "fa-solid fa-eye", "title": "Identifiers", } ] OSINT_SCOPES: dict[str, OsintScopeConfig] = { "people": OsintScopeConfig( key="people", title="People", model=Person, list_url_name="people", update_url_name="person_update", delete_url_name="person_delete", default_sort="name", columns=( OsintColumn( key="id", label="ID", accessor=lambda item: item.id, sort_field="id", search_lookup="id__icontains", kind="id_copy", ), OsintColumn( key="name", label="Name", accessor=lambda item: item.name, sort_field="name", search_lookup="name__icontains", ), OsintColumn( key="sentiment", label="Sentiment", accessor=lambda item: item.sentiment, sort_field="sentiment", ), OsintColumn( key="timezone", label="Timezone", accessor=lambda item: item.timezone, sort_field="timezone", search_lookup="timezone__icontains", ), OsintColumn( key="last_interaction", label="Last Interaction", accessor=lambda item: item.last_interaction, sort_field="last_interaction", kind="datetime", ), ), delete_label=lambda item: item.name, extra_actions=_person_extra_actions, search_lookups=( "id__icontains", "name__icontains", "summary__icontains", "profile__icontains", "revealed__icontains", "likes__icontains", "dislikes__icontains", "timezone__icontains", ), ), "groups": OsintScopeConfig( key="groups", title="Groups", model=Group, list_url_name="groups", update_url_name="group_update", delete_url_name="group_delete", default_sort="name", columns=( OsintColumn( key="id", label="ID", accessor=lambda item: item.id, sort_field="id", search_lookup="id__icontains", kind="id_copy", ), OsintColumn( key="name", label="Name", accessor=lambda item: item.name, sort_field="name", search_lookup="name__icontains", ), OsintColumn( key="people_count", label="People", accessor=lambda item: item.people.count(), kind="number", ), ), prefetch_related=("people",), delete_label=lambda item: item.name, search_lookups=( "id__icontains", "name__icontains", "people__name__icontains", ), ), "personas": OsintScopeConfig( key="personas", title="Personas", model=Persona, list_url_name="personas", update_url_name="persona_update", delete_url_name="persona_delete", default_sort="alias", columns=( OsintColumn( key="id", label="ID", accessor=lambda item: item.id, sort_field="id", search_lookup="id__icontains", kind="id_copy", ), OsintColumn( key="alias", label="Alias", accessor=lambda item: item.alias, sort_field="alias", search_lookup="alias__icontains", ), OsintColumn( key="mbti", label="MBTI", accessor=lambda item: item.mbti, sort_field="mbti", search_lookup="mbti__icontains", ), OsintColumn( key="mbti_identity", label="MBTI Identity", accessor=lambda item: item.mbti_identity, sort_field="mbti_identity", ), OsintColumn( key="humor_style", label="Humor Style", accessor=lambda item: item.humor_style, sort_field="humor_style", search_lookup="humor_style__icontains", ), OsintColumn( key="tone", label="Tone", accessor=lambda item: item.tone, sort_field="tone", search_lookup="tone__icontains", ), OsintColumn( key="trust", label="Trust", accessor=lambda item: item.trust, sort_field="trust", ), OsintColumn( key="adaptability", label="Adaptability", accessor=lambda item: item.adaptability, sort_field="adaptability", ), ), delete_label=lambda item: item.alias or str(item.id), search_lookups=( "id__icontains", "alias__icontains", "mbti__icontains", "humor_style__icontains", "tone__icontains", "communication_style__icontains", "core_values__icontains", "inner_story__icontains", "likes__icontains", "dislikes__icontains", ), ), "manipulations": OsintScopeConfig( key="manipulations", title="Manipulations", model=Manipulation, list_url_name="manipulations", update_url_name="manipulation_update", delete_url_name="manipulation_delete", default_sort="name", columns=( OsintColumn( key="id", label="ID", accessor=lambda item: item.id, sort_field="id", search_lookup="id__icontains", kind="id_copy", ), OsintColumn( key="name", label="Name", accessor=lambda item: item.name, sort_field="name", search_lookup="name__icontains", ), OsintColumn( key="group", label="Group", accessor=lambda item: item.group, sort_field="group__name", search_lookup="group__name__icontains", ), OsintColumn( key="ai", label="AI", accessor=lambda item: item.ai, sort_field="ai__model", search_lookup="ai__model__icontains", ), OsintColumn( key="persona", label="Persona", accessor=lambda item: item.persona, sort_field="persona__alias", search_lookup="persona__alias__icontains", ), OsintColumn( key="enabled", label="Enabled", accessor=lambda item: item.enabled, sort_field="enabled", kind="bool", ), OsintColumn( key="mode", label="Mode", accessor=lambda item: item.mode, sort_field="mode", search_lookup="mode__icontains", ), OsintColumn( key="filter_enabled", label="Filter", accessor=lambda item: item.filter_enabled, sort_field="filter_enabled", kind="bool", ), ), select_related=("group", "ai", "persona"), delete_label=lambda item: item.name, search_lookups=( "id__icontains", "name__icontains", "mode__icontains", "group__name__icontains", "ai__model__icontains", "persona__alias__icontains", ), ), "identifiers": OsintScopeConfig( key="identifiers", title="Identifiers", model=PersonIdentifier, list_url_name="identifiers", update_url_name="identifier_update", delete_url_name="identifier_delete", default_sort="identifier", columns=( OsintColumn( key="id", label="ID", accessor=lambda item: item.id, sort_field="id", search_lookup="id__icontains", kind="id_copy", ), OsintColumn( key="person", label="Person", accessor=lambda item: item.person.name if item.person_id else "", sort_field="person__name", search_lookup="person__name__icontains", ), OsintColumn( key="service", label="Service", accessor=lambda item: item.service, sort_field="service", search_lookup="service__icontains", ), OsintColumn( key="identifier", label="Identifier", accessor=lambda item: item.identifier, sort_field="identifier", search_lookup="identifier__icontains", ), ), select_related=("person",), delete_label=lambda item: item.identifier, search_lookups=( "id__icontains", "person__name__icontains", "service__icontains", "identifier__icontains", ), ), "messages": OsintScopeConfig( key="messages", title="Messages", model=Message, list_url_name="sessions", update_url_name="session_update", delete_url_name="session_delete", default_sort="ts", columns=( OsintColumn( key="id", label="ID", accessor=lambda item: item.id, sort_field="id", search_lookup="id__icontains", kind="id_copy", ), OsintColumn( key="service", label="Service", accessor=lambda item: item.source_service or "", sort_field="source_service", search_lookup="source_service__icontains", ), OsintColumn( key="chat", label="Chat", accessor=lambda item: { "display": "", "copy": item.source_chat_id or "", }, sort_field="source_chat_id", search_lookup="source_chat_id__icontains", kind="chat_ref", ), OsintColumn( key="sender", label="Sender", accessor=lambda item: item.custom_author or item.sender_uuid or "", sort_field="sender_uuid", search_lookup="sender_uuid__icontains", ), OsintColumn( key="text", label="Text", accessor=lambda item: item.text or "", search_lookup="text__icontains", ), OsintColumn( key="ts", label="Timestamp", accessor=lambda item: datetime.fromtimestamp(item.ts / 1000.0), sort_field="ts", kind="datetime", ), ), search_lookups=( "id__icontains", "text__icontains", "source_service__icontains", "source_chat_id__icontains", "sender_uuid__icontains", "custom_author__icontains", "source_message_id__icontains", ), ), } OSINT_SCOPE_ICONS: dict[str, str] = { "all": "fa-solid fa-globe", "people": "fa-solid fa-user-group", "groups": "fa-solid fa-users", "personas": "fa-solid fa-masks-theater", "manipulations": "fa-solid fa-sliders", "identifiers": "fa-solid fa-id-card", "messages": "fa-solid fa-message", } class OSINTListBase(ObjectList): list_template = "partials/osint/list-table.html" paginate_by = 20 osint_scope = "" def get_scope(self) -> OsintScopeConfig: if self.osint_scope not in OSINT_SCOPES: raise ValueError(f"Unknown OSINT scope: {self.osint_scope}") return OSINT_SCOPES[self.osint_scope] def _list_url(self) -> str: list_url_args = {} for arg in self.list_url_args: if arg in self.kwargs: list_url_args[arg] = self.kwargs[arg] return reverse(self.list_url_name, kwargs=list_url_args) def _active_sort(self, scope: OsintScopeConfig) -> tuple[str, str]: direction = self.request.GET.get("dir", "asc").lower() if direction not in {"asc", "desc"}: direction = "asc" allowed = {col.sort_field for col in scope.columns if col.sort_field} sort_field = self.request.GET.get("sort") if sort_field not in allowed: sort_field = scope.default_sort return sort_field, direction def get_ordering(self): scope = self.get_scope() sort_field, direction = self._active_sort(scope) if not sort_field: return None if direction == "desc": return f"-{sort_field}" return sort_field def _search_lookups(self, scope: OsintScopeConfig) -> dict[str, str]: lookups = {} for column in scope.columns: if column.search_lookup: lookups[column.key] = column.search_lookup return lookups def _query_dict(self) -> dict[str, Any]: return _sanitize_query_state( {k: v for k, v in self.request.GET.items() if v not in {"", None}} ) def _apply_list_search( self, queryset: models.QuerySet, scope: OsintScopeConfig ) -> models.QuerySet: query = _sanitize_search_query(self.request.GET.get("q", "")) if not query: return queryset lookups_by_field = self._search_lookups(scope) selected_field = self.request.GET.get("field", "__all__").strip() if selected_field in lookups_by_field: selected_lookups = [lookups_by_field[selected_field]] else: selected_lookups = list(scope.search_lookups) or list( lookups_by_field.values() ) if not selected_lookups: return queryset condition = Q() for lookup in selected_lookups: condition |= Q(**{lookup: query}) queryset = queryset.filter(condition) if any("__" in lookup for lookup in selected_lookups): queryset = queryset.distinct() return queryset def get_queryset(self, **kwargs): queryset = super().get_queryset(**kwargs) scope = self.get_scope() if scope.select_related: queryset = queryset.select_related(*scope.select_related) if scope.prefetch_related: queryset = queryset.prefetch_related(*scope.prefetch_related) return self._apply_list_search(queryset, scope) def _build_column_context( self, scope: OsintScopeConfig, list_url: str, query_state: dict[str, Any], ) -> list[dict[str, Any]]: sort_field, direction = self._active_sort(scope) columns = [] for column in scope.columns: if not column.sort_field: columns.append( { "key": column.key, "field_name": _column_field_name(column), "label": column.label, "sortable": False, "kind": column.kind, } ) continue is_sorted = sort_field == column.sort_field next_direction = "desc" if is_sorted and direction == "asc" else "asc" sort_query = _merge_query( query_state, sort=column.sort_field, dir=next_direction, page=1, ) columns.append( { "key": column.key, "field_name": _column_field_name(column), "label": column.label, "sortable": True, "kind": column.kind, "is_sorted": is_sorted, "is_desc": is_sorted and direction == "desc", "sort_url": _url_with_query(list_url, sort_query), } ) return columns def _build_rows( self, scope: OsintScopeConfig, object_list: list[Any], request_type: str, ) -> list[dict[str, Any]]: context_type = _context_type(request_type) update_type = "window" if request_type == "widget" else context_type update_target = ( "#windows-here" if update_type == "window" else f"#{update_type}s-here" ) rows = [] for item in object_list: row = {"id": str(item.pk), "cells": [], "actions": []} for column in scope.columns: row["cells"].append( { "kind": column.kind, "value": column.accessor(item), } ) update_url = reverse( scope.update_url_name, kwargs={"type": update_type, "pk": item.pk}, ) delete_url = reverse( scope.delete_url_name, kwargs={"type": context_type, "pk": item.pk}, ) row["actions"].append( { "mode": "hx-get", "url": update_url, "target": update_target, "icon": "fa-solid fa-pencil", "title": "Edit", } ) row["actions"].append( { "mode": "hx-delete", "url": delete_url, "target": "#modals-here", "icon": "fa-solid fa-xmark", "title": "Delete", "confirm": ( "Are you sure you wish to delete " f"{scope.delete_label(item)}?" ), } ) row["actions"].extend(scope.extra_actions(item, context_type)) rows.append(row) return rows def _build_pagination( self, page_obj: Any, list_url: str, query_state: dict[str, Any], ) -> dict[str, Any]: if page_obj is None: return {"enabled": False} pagination = { "enabled": page_obj.paginator.num_pages > 1, "count": page_obj.paginator.count, "current": page_obj.number, "total": page_obj.paginator.num_pages, "has_previous": page_obj.has_previous(), "has_next": page_obj.has_next(), "previous_url": None, "next_url": None, "pages": [], } if page_obj.has_previous(): previous_page = _safe_page_number(page_obj.previous_page_number()) pagination["previous_url"] = _url_with_query( list_url, {"page": previous_page}, ) if page_obj.has_next(): next_page = _safe_page_number(page_obj.next_page_number()) pagination["next_url"] = _url_with_query( list_url, {"page": next_page}, ) for entry in page_obj.paginator.get_elided_page_range(page_obj.number): if entry == "…": pagination["pages"].append({"ellipsis": True}) continue pagination["pages"].append( { "ellipsis": False, "number": entry, "current": entry == page_obj.number, "url": _url_with_query( list_url, {"page": _safe_page_number(entry)}, ), } ) return pagination def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) scope = self.get_scope() request_type = self.kwargs.get("type", "modal") list_url = self._list_url() query_state = self._query_dict() query_url = _url_with_query(list_url, query_state) field_options = [{"value": "__all__", "label": "All"}] field_options.extend( [ {"value": key, "label": column.label} for key, column in ( (column.key, column) for column in scope.columns if column.search_lookup ) ] ) context["osint_scope"] = scope.key context["osint_title"] = scope.title context["osint_table_id"] = f"{self.context_object_name}-table" context["osint_event_name"] = f"{self.context_object_name_singular}Event" context["osint_refresh_url"] = query_url or list_url context["osint_columns"] = self._build_column_context( scope, list_url, query_state, ) context["osint_rows"] = self._build_rows( scope, list(context["object_list"]), request_type, ) context["osint_pagination"] = self._build_pagination( context.get("page_obj"), list_url, query_state, ) context["osint_search_query"] = self.request.GET.get("q", "") context["osint_search_field"] = self.request.GET.get("field", "__all__") context["osint_search_fields"] = field_options context["osint_show_search"] = True context["osint_show_actions"] = True context["osint_search_url"] = list_url context["osint_result_count"] = context["osint_pagination"].get("count", 0) context["widget_icon"] = _safe_icon_class( self.request.GET.get("widget_icon"), OSINT_SCOPE_ICONS.get(scope.key, "fa-solid fa-arrows-minimize"), ) return context class OSINTSearch(LoginRequiredMixin, View): allowed_types = {"page", "widget"} page_template = "pages/osint-search.html" widget_template = "mixins/wm/widget.html" panel_template = "partials/osint/search-panel.html" result_template = "partials/results_table.html" per_page_default = 20 per_page_max = 100 all_scope_keys = ("people", "identifiers", "messages") @dataclass(frozen=True) class SearchPlan: size: int index: str query: str tags: tuple[str, ...] source: str date_from: str date_to: str sort_mode: str sentiment_min: str sentiment_max: str annotate: bool dedup: bool reverse: bool def _prepare_siqtsrss_adr(self, request) -> "OSINTSearch.SearchPlan": """ Parse search controls following the Neptune-style SIQTSRSS/ADR flow. S - Size, I - Index, Q - Query, T - Tags, S - Source, R - Ranges, S - Sort, S - Sentiment, A - Annotate, D - Dedup, R - Reverse. """ query = _sanitize_search_query(_safe_query_param(request, "q", "")) tags = tuple( token[4:].strip() for token in query.split() if token.lower().startswith("tag:") ) return self.SearchPlan( size=self._per_page(request.GET.get("per_page")), index=self._scope_key(request.GET.get("scope")), query=query, tags=tags, source=_safe_query_param(request, "source", "all").lower() or "all", date_from=_safe_query_param(request, "date_from", ""), date_to=_safe_query_param(request, "date_to", ""), sort_mode=_safe_query_param(request, "sort_mode", "relevance").lower(), sentiment_min=_safe_query_param(request, "sentiment_min", ""), sentiment_max=_safe_query_param(request, "sentiment_max", ""), annotate=_safe_query_param(request, "annotate", "1") not in {"0", "false", "off"}, dedup=_safe_query_param(request, "dedup", "") in {"1", "true", "on"}, reverse=_safe_query_param(request, "reverse", "") in {"1", "true", "on"}, ) def _parse_date_boundaries(self, plan: "OSINTSearch.SearchPlan") -> tuple[datetime | None, datetime | None]: parsed_from = None parsed_to = None if plan.date_from: try: parsed_from = datetime.fromisoformat(plan.date_from) except ValueError: parsed_from = None if plan.date_to: try: parsed_to = datetime.fromisoformat(plan.date_to) except ValueError: parsed_to = None if parsed_to is not None: parsed_to = parsed_to.replace(hour=23, minute=59, second=59, microsecond=999999) return parsed_from, parsed_to def _score_hit(self, query: str, primary: str, secondary: str) -> int: if not query: return 0 needle = query.lower() return primary.lower().count(needle) * 3 + secondary.lower().count(needle) def _identifier_exact_boost(self, query: str, identifier: str) -> int: needle = str(query or "").strip().lower() hay = str(identifier or "").strip().lower() if not needle or not hay: return 0 if hay == needle: return 120 if hay.startswith(needle): return 45 if needle in hay: return 15 return 0 def _message_recency_boost(self, stamp: datetime | None) -> int: if stamp is None: return 0 now_ts = timezone.now() if timezone.is_naive(stamp): stamp = timezone.make_aware(stamp, dt_timezone.utc) age = now_ts - stamp if age.days < 1: return 40 if age.days < 7: return 25 if age.days < 30: return 12 if age.days < 90: return 6 return 0 def _snippet(self, text: str, query: str, max_len: int = 180) -> str: value = str(text or "").strip() if not value: return "" if not query: return value[:max_len] lower = value.lower() needle = query.lower() idx = lower.find(needle) if idx < 0: return value[:max_len] start = max(idx - 40, 0) end = min(idx + len(needle) + 90, len(value)) snippet = value[start:end] if start > 0: snippet = "…" + snippet if end < len(value): snippet = snippet + "…" return snippet def _field_options(self, model_cls: type[models.Model]) -> list[dict[str, str]]: options = [] for model_field in model_cls._meta.get_fields(): # Skip reverse/accessor relations (e.g. ManyToManyRel) that are not # directly searchable as user-facing fields in this selector. if model_field.auto_created and not model_field.concrete: continue if model_field.name == "user": continue label = getattr( model_field, "verbose_name", str(model_field.name).replace("_", " "), ) options.append( { "value": model_field.name, "label": str(label).title(), } ) options.sort(key=lambda item: item["label"]) return options def _field_q( self, model_cls: type[models.Model], field_name: str, query: str, ) -> tuple[Q | None, bool]: try: field = model_cls._meta.get_field(field_name) except FieldDoesNotExist: return None, False if isinstance(field, (models.CharField, models.TextField, models.UUIDField)): return Q(**{f"{field_name}__icontains": query}), False if isinstance(field, models.BooleanField): parsed = _parse_bool(query) if parsed is None: return None, False return Q(**{field_name: parsed}), False if isinstance(field, models.IntegerField): try: return Q(**{field_name: int(query)}), False except ValueError: return None, False if isinstance(field, (models.FloatField, models.DecimalField)): try: value = Decimal(query) except InvalidOperation: return None, False return Q(**{field_name: value}), False if isinstance(field, models.DateField): try: parsed_date = date.fromisoformat(query) except ValueError: return None, False return Q(**{field_name: parsed_date}), False if isinstance(field, models.DateTimeField): try: parsed_dt = datetime.fromisoformat(query) except ValueError: try: parsed_date = date.fromisoformat(query) except ValueError: return None, False return Q(**{f"{field_name}__date": parsed_date}), False return Q(**{field_name: parsed_dt}), False if isinstance(field, models.ForeignKey): related_text_field = _preferred_related_text_field(field.related_model) if related_text_field: return ( Q(**{f"{field_name}__{related_text_field}__icontains": query}), False, ) return Q(**{f"{field_name}__id__icontains": query}), False if isinstance(field, models.ManyToManyField): related_text_field = _preferred_related_text_field(field.related_model) if related_text_field: return ( Q(**{f"{field_name}__{related_text_field}__icontains": query}), True, ) return Q(**{f"{field_name}__id__icontains": query}), True return None, False def _search_queryset( self, queryset: models.QuerySet, model_cls: type[models.Model], query: str, field_name: str, field_options: list[dict[str, str]], ) -> models.QuerySet: if not query: return queryset if field_name != "__all__": field_q, use_distinct = self._field_q(model_cls, field_name, query) if field_q is None: return queryset.none() queryset = queryset.filter(field_q) return queryset.distinct() if use_distinct else queryset condition = Q() use_distinct = False for option in field_options: field_q, field_distinct = self._field_q( model_cls, option["value"], query, ) if field_q is None: continue condition |= field_q use_distinct = use_distinct or field_distinct if not condition.children: return queryset.none() queryset = queryset.filter(condition) return queryset.distinct() if use_distinct else queryset def _per_page(self, raw_value: str | None) -> int: if not raw_value: return self.per_page_default try: value = int(raw_value) except ValueError: return self.per_page_default if value < 1: return self.per_page_default return min(value, self.per_page_max) def _scope_key(self, raw_scope: str | None) -> str: if raw_scope == "all": return "all" if raw_scope in OSINT_SCOPES: return raw_scope return "all" def _query_state(self, request) -> dict[str, Any]: return _sanitize_query_state( {k: v for k, v in request.GET.items() if v not in {None, ""}} ) def _apply_common_filters( self, queryset: models.QuerySet, scope_key: str, plan: "OSINTSearch.SearchPlan", ) -> models.QuerySet: date_from, date_to = self._parse_date_boundaries(plan) if plan.source and plan.source != "all": if scope_key == "messages": queryset = queryset.filter(source_service=plan.source) elif scope_key == "identifiers": queryset = queryset.filter(service=plan.source) elif scope_key == "people": queryset = queryset.filter(personidentifier__service=plan.source).distinct() if scope_key == "messages": if date_from is not None: queryset = queryset.filter(ts__gte=int(date_from.timestamp() * 1000)) if date_to is not None: queryset = queryset.filter(ts__lte=int(date_to.timestamp() * 1000)) elif scope_key == "people": if date_from is not None: queryset = queryset.filter(last_interaction__gte=date_from) if date_to is not None: queryset = queryset.filter(last_interaction__lte=date_to) if plan.sentiment_min: try: queryset = queryset.filter(sentiment__gte=float(plan.sentiment_min)) except ValueError: pass if plan.sentiment_max: try: queryset = queryset.filter(sentiment__lte=float(plan.sentiment_max)) except ValueError: pass return queryset def _search_all_rows( self, request, plan: "OSINTSearch.SearchPlan", ) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] query = plan.query date_from, date_to = self._parse_date_boundaries(plan) per_scope_limit = max(40, min(plan.size * 3, 250)) allowed_scopes = set(self.all_scope_keys) tag_scopes = { tag.strip().lower() for tag in plan.tags if tag.strip().lower() in allowed_scopes } if tag_scopes: allowed_scopes = tag_scopes if "people" in allowed_scopes: people_qs = self._apply_common_filters( Person.objects.filter(user=request.user), "people", plan, ) if query: people_qs = people_qs.filter( Q(name__icontains=query) | Q(summary__icontains=query) | Q(profile__icontains=query) | Q(revealed__icontains=query) | Q(likes__icontains=query) | Q(dislikes__icontains=query) ) for item in people_qs.order_by("-last_interaction", "name")[:per_scope_limit]: secondary = self._snippet( f"{item.summary or ''} {item.profile or ''}".strip(), query if plan.annotate else "", ) rows.append( { "id": f"person:{item.id}", "scope": "Contact", "primary": item.name, "secondary": secondary or (item.timezone or ""), "service": "-", "when": item.last_interaction, "score": self._score_hit(query, item.name or "", secondary or ""), } ) if "identifiers" in allowed_scopes: identifiers_qs = self._apply_common_filters( PersonIdentifier.objects.filter(user=request.user).select_related("person"), "identifiers", plan, ) if query: identifiers_qs = identifiers_qs.filter( Q(identifier__icontains=query) | Q(person__name__icontains=query) | Q(service__icontains=query) ) for item in identifiers_qs.order_by("person__name", "identifier")[:per_scope_limit]: primary = item.person.name if item.person_id else item.identifier secondary = item.identifier if item.person_id else "" base_score = self._score_hit(query, primary or "", secondary or "") exact_boost = self._identifier_exact_boost(query, item.identifier) rows.append( { "id": f"identifier:{item.id}", "scope": "Identifier", "primary": primary, "secondary": secondary, "service": item.service or "", "when": None, "score": base_score + exact_boost, } ) if "messages" in allowed_scopes: messages_qs = self._apply_common_filters( Message.objects.filter(user=request.user), "messages", plan, ) if query: messages_qs = messages_qs.filter( Q(text__icontains=query) | Q(custom_author__icontains=query) | Q(sender_uuid__icontains=query) | Q(source_chat_id__icontains=query) | Q(source_message_id__icontains=query) ) for item in messages_qs.order_by("-ts")[:per_scope_limit]: when_dt = datetime.fromtimestamp(item.ts / 1000.0) if item.ts else None if date_from and when_dt and when_dt < date_from: continue if date_to and when_dt and when_dt > date_to: continue primary = item.custom_author or item.sender_uuid or (item.source_chat_id or "Message") secondary = self._snippet(item.text or "", query if plan.annotate else "") base_score = self._score_hit(query, primary or "", item.text or "") recency_boost = self._message_recency_boost(when_dt) rows.append( { "id": f"message:{item.id}", "scope": "Message", "primary": primary, "secondary": secondary, "service": item.source_service or "-", "when": when_dt, "score": base_score + recency_boost, } ) if plan.dedup: seen = set() deduped = [] for row in rows: key = ( row["scope"], str(row["primary"]).strip().lower(), str(row["secondary"]).strip().lower(), str(row["service"]).strip().lower(), ) if key in seen: continue seen.add(key) deduped.append(row) rows = deduped def row_time_key(row: dict[str, Any]) -> float: stamp = row.get("when") if stamp is None: return 0.0 if timezone.is_aware(stamp): return float(stamp.timestamp()) return float(stamp.replace(tzinfo=dt_timezone.utc).timestamp()) if plan.sort_mode == "oldest": rows.sort(key=row_time_key) elif plan.sort_mode == "recent": rows.sort(key=row_time_key, reverse=True) else: rows.sort( key=lambda row: ( row["score"], row_time_key(row), ), reverse=True, ) if plan.reverse: rows.reverse() return rows def _active_sort(self, scope: OsintScopeConfig) -> tuple[str, str]: direction = self.request.GET.get("dir", "asc").lower() if direction not in {"asc", "desc"}: direction = "asc" allowed = {col.sort_field for col in scope.columns if col.sort_field} sort_field = self.request.GET.get("sort") if sort_field not in allowed: sort_field = scope.default_sort return sort_field, direction def _build_column_context( self, scope: OsintScopeConfig, list_url: str, query_state: dict[str, Any], ) -> list[dict[str, Any]]: sort_field, direction = self._active_sort(scope) columns = [] for column in scope.columns: if not column.sort_field: columns.append( { "key": column.key, "field_name": _column_field_name(column), "label": column.label, "sortable": False, "kind": column.kind, } ) continue is_sorted = sort_field == column.sort_field next_direction = "desc" if is_sorted and direction == "asc" else "asc" sort_query = _merge_query( query_state, sort=column.sort_field, dir=next_direction, page=1, ) columns.append( { "key": column.key, "field_name": _column_field_name(column), "label": column.label, "sortable": True, "kind": column.kind, "is_sorted": is_sorted, "is_desc": is_sorted and direction == "desc", "sort_url": _url_with_query(list_url, sort_query), } ) return columns def _build_rows( self, scope: OsintScopeConfig, object_list: list[Any], request_type: str, ) -> list[dict[str, Any]]: rows = [] for item in object_list: row = {"id": str(item.pk), "cells": [], "actions": []} for column in scope.columns: row["cells"].append( { "kind": column.kind, "value": column.accessor(item), } ) rows.append(row) return rows def _build_pagination( self, page_obj: Any, list_url: str, query_state: dict[str, Any], ) -> dict[str, Any]: if page_obj is None: return {"enabled": False} pagination = { "enabled": page_obj.paginator.num_pages > 1, "count": page_obj.paginator.count, "current": page_obj.number, "total": page_obj.paginator.num_pages, "has_previous": page_obj.has_previous(), "has_next": page_obj.has_next(), "previous_url": None, "next_url": None, "pages": [], } if page_obj.has_previous(): previous_page = _safe_page_number(page_obj.previous_page_number()) pagination["previous_url"] = _url_with_query( list_url, {"page": previous_page}, ) if page_obj.has_next(): next_page = _safe_page_number(page_obj.next_page_number()) pagination["next_url"] = _url_with_query( list_url, {"page": next_page}, ) for entry in page_obj.paginator.get_elided_page_range(page_obj.number): if entry == "…": pagination["pages"].append({"ellipsis": True}) continue pagination["pages"].append( { "ellipsis": False, "number": entry, "current": entry == page_obj.number, "url": _url_with_query( list_url, {"page": _safe_page_number(entry)}, ), } ) return pagination def get(self, request, type): if type not in self.allowed_types: return HttpResponseBadRequest("Invalid type specified.") plan = self._prepare_siqtsrss_adr(request) scope_key = plan.index query = plan.query list_url = reverse("osint_search", kwargs={"type": type}) query_state = self._query_state(request) field_name = request.GET.get("field", "__all__") if scope_key == "all": rows_raw = self._search_all_rows(request, plan) paginator = Paginator(rows_raw, plan.size) page_obj = paginator.get_page(request.GET.get("page")) column_context = [ {"key": "scope", "field_name": "scope", "label": "Type", "sortable": False, "kind": "text"}, {"key": "primary", "field_name": "primary", "label": "Primary", "sortable": False, "kind": "text"}, {"key": "secondary", "field_name": "secondary", "label": "Details", "sortable": False, "kind": "text"}, {"key": "service", "field_name": "service", "label": "Service", "sortable": False, "kind": "text"}, {"key": "when", "field_name": "when", "label": "When", "sortable": False, "kind": "datetime"}, ] rows = [] for item in list(page_obj.object_list): rows.append( { "id": item["id"], "cells": [ {"kind": "text", "value": item.get("scope")}, {"kind": "text", "value": item.get("primary")}, {"kind": "text", "value": item.get("secondary")}, {"kind": "text", "value": item.get("service")}, {"kind": "datetime", "value": item.get("when")}, ], "actions": [], } ) pagination = self._build_pagination(page_obj, list_url, query_state) field_options: list[dict[str, str]] = [] selected_scope_key = "all" osint_title = "Search Everything" result_count = paginator.count else: scope = OSINT_SCOPES[scope_key] field_options = self._field_options(scope.model) if field_name != "__all__": allowed_fields = {option["value"] for option in field_options} if field_name not in allowed_fields: field_name = "__all__" queryset = scope.model.objects.filter(user=request.user) if scope.select_related: queryset = queryset.select_related(*scope.select_related) if scope.prefetch_related: queryset = queryset.prefetch_related(*scope.prefetch_related) queryset = self._apply_common_filters(queryset, scope.key, plan) queryset = self._search_queryset( queryset, scope.model, query, field_name, field_options, ) if plan.dedup: queryset = queryset.distinct() sort_field = request.GET.get("sort", scope.default_sort) direction = request.GET.get("dir", "asc").lower() allowed_sort_fields = { column.sort_field for column in scope.columns if column.sort_field } if sort_field not in allowed_sort_fields: sort_field = scope.default_sort if direction not in {"asc", "desc"}: direction = "asc" if sort_field: order_by = sort_field if direction == "asc" else f"-{sort_field}" queryset = queryset.order_by(order_by) if plan.reverse: queryset = queryset.reverse() paginator = Paginator(queryset, plan.size) page_obj = paginator.get_page(request.GET.get("page")) object_list = list(page_obj.object_list) column_context = self._build_column_context( scope, list_url, query_state, ) rows = self._build_rows( scope, object_list, type, ) pagination = self._build_pagination( page_obj, list_url, query_state, ) selected_scope_key = scope.key osint_title = f"Search {scope.title}" result_count = paginator.count context = { "osint_scope": selected_scope_key, "osint_title": osint_title, "osint_table_id": "osint-search-table", "osint_event_name": "", "osint_refresh_url": _url_with_query(list_url, query_state), "osint_columns": column_context, "osint_rows": rows, "osint_pagination": pagination, "osint_show_search": False, "osint_show_actions": False, "osint_shell_borderless": True, "osint_result_count": result_count, "osint_search_url": list_url, "scope_options": [ {"value": "all", "label": "All (Contacts + Messages)"}, *[ {"value": key, "label": conf.title} for key, conf in OSINT_SCOPES.items() ], ], "field_options": field_options, "selected_scope": selected_scope_key, "selected_field": field_name, "search_query": query, "selected_per_page": plan.size, "selected_source": plan.source, "selected_date_from": plan.date_from, "selected_date_to": plan.date_to, "selected_sort_mode": plan.sort_mode, "selected_sentiment_min": plan.sentiment_min, "selected_sentiment_max": plan.sentiment_max, "selected_annotate": plan.annotate, "selected_dedup": plan.dedup, "selected_reverse": plan.reverse, "search_page_url": reverse("osint_search", kwargs={"type": "page"}), "search_widget_url": reverse("osint_search", kwargs={"type": "widget"}), } hx_target = request.headers.get("HX-Target") if request.htmx and hx_target in {"osint-search-results", "osint-search-table"}: response = render(request, self.result_template, context) if type == "page": response["HX-Replace-Url"] = _url_with_query(list_url, query_state) return response if type == "widget": widget_context = { "title": "Search", "unique": "osint-search-widget", "window_content": self.panel_template, "widget_options": 'gs-w="8" gs-h="14" gs-x="0" gs-y="0" gs-min-w="5"', "widget_icon": _safe_icon_class( request.GET.get("widget_icon"), "fa-solid fa-magnifying-glass", ), **context, } return render(request, self.widget_template, widget_context) return render(request, self.page_template, context) class OSINTWorkspace(LoginRequiredMixin, View): template_name = "pages/osint-workspace.html" def get(self, request): context = { "tabs_widget_url": reverse("osint_workspace_tabs_widget"), } return render(request, self.template_name, context) class OSINTWorkspaceTabsWidget(LoginRequiredMixin, View): def get(self, request): tabs = [ { "key": "people", "label": "People", "icon": "fa-solid fa-user-group", "widget_url": _url_with_query( reverse("people", kwargs={"type": "widget"}), {"widget_icon": "fa-solid fa-user-group"}, ), }, { "key": "groups", "label": "Groups", "icon": "fa-solid fa-users", "widget_url": _url_with_query( reverse("groups", kwargs={"type": "widget"}), {"widget_icon": "fa-solid fa-users"}, ), }, { "key": "personas", "label": "Personas", "icon": "fa-solid fa-masks-theater", "widget_url": _url_with_query( reverse("personas", kwargs={"type": "widget"}), {"widget_icon": "fa-solid fa-masks-theater"}, ), }, { "key": "manipulations", "label": "Manipulations", "icon": "fa-solid fa-sliders", "widget_url": _url_with_query( reverse("manipulations", kwargs={"type": "widget"}), {"widget_icon": "fa-solid fa-sliders"}, ), }, ] context = { "title": "OSINT Workspace", "unique": "osint-workspace-tabs", "window_content": "partials/osint-workspace-tabs-widget.html", "widget_options": 'gs-w="12" gs-h="4" gs-x="0" gs-y="0" gs-min-w="6"', "tabs": tabs, } return render(request, "mixins/wm/widget.html", context)