462 lines
17 KiB
Python
462 lines
17 KiB
Python
import time
|
|
|
|
from django.http import JsonResponse
|
|
from django.shortcuts import render
|
|
from django.urls import reverse
|
|
from django.views import View
|
|
|
|
from core.models import (
|
|
AdapterHealthEvent,
|
|
AIRequest,
|
|
AIResult,
|
|
AIResultSignal,
|
|
Chat,
|
|
ChatSession,
|
|
ConversationEvent,
|
|
Group,
|
|
MemoryItem,
|
|
Message,
|
|
MessageEvent,
|
|
PatternArtifactExport,
|
|
PatternMitigationAutoSettings,
|
|
PatternMitigationCorrection,
|
|
PatternMitigationGame,
|
|
PatternMitigationMessage,
|
|
PatternMitigationPlan,
|
|
PatternMitigationRule,
|
|
Person,
|
|
Persona,
|
|
PersonIdentifier,
|
|
QueuedMessage,
|
|
WorkspaceConversation,
|
|
WorkspaceMetricSnapshot,
|
|
)
|
|
from core.events.projection import shadow_compare_session
|
|
from core.memory.search_backend import backend_status, get_memory_search_backend
|
|
from core.transports.capabilities import capability_snapshot
|
|
from core.views.manage.permissions import SuperUserRequiredMixin
|
|
|
|
|
|
class SystemSettings(SuperUserRequiredMixin, View):
|
|
template_name = "pages/system-settings.html"
|
|
|
|
def _counts(self, user):
|
|
return {
|
|
"chat_sessions": ChatSession.objects.filter(user=user).count(),
|
|
"messages": Message.objects.filter(user=user).count(),
|
|
"queued_messages": QueuedMessage.objects.filter(user=user).count(),
|
|
"message_events": MessageEvent.objects.filter(user=user).count(),
|
|
"conversation_events": ConversationEvent.objects.filter(user=user).count(),
|
|
"adapter_health_events": AdapterHealthEvent.objects.filter(user=user).count(),
|
|
"workspace_conversations": WorkspaceConversation.objects.filter(
|
|
user=user
|
|
).count(),
|
|
"workspace_snapshots": WorkspaceMetricSnapshot.objects.filter(
|
|
conversation__user=user
|
|
).count(),
|
|
"ai_requests": AIRequest.objects.filter(user=user).count(),
|
|
"ai_results": AIResult.objects.filter(user=user).count(),
|
|
"ai_result_signals": AIResultSignal.objects.filter(user=user).count(),
|
|
"memory_items": MemoryItem.objects.filter(user=user).count(),
|
|
"mitigation_plans": PatternMitigationPlan.objects.filter(user=user).count(),
|
|
"mitigation_rules": PatternMitigationRule.objects.filter(user=user).count(),
|
|
"mitigation_games": PatternMitigationGame.objects.filter(user=user).count(),
|
|
"mitigation_corrections": PatternMitigationCorrection.objects.filter(
|
|
user=user
|
|
).count(),
|
|
"mitigation_messages": PatternMitigationMessage.objects.filter(
|
|
user=user
|
|
).count(),
|
|
"mitigation_auto_settings": PatternMitigationAutoSettings.objects.filter(
|
|
user=user
|
|
).count(),
|
|
"mitigation_exports": PatternArtifactExport.objects.filter(
|
|
user=user
|
|
).count(),
|
|
"osint_people": Person.objects.filter(user=user).count(),
|
|
"osint_identifiers": PersonIdentifier.objects.filter(user=user).count(),
|
|
"osint_groups": Group.objects.filter(user=user).count(),
|
|
"osint_personas": Persona.objects.filter(user=user).count(),
|
|
}
|
|
|
|
def _purge_non_osint(self, user):
|
|
deleted = 0
|
|
deleted += PatternArtifactExport.objects.filter(user=user).delete()[0]
|
|
deleted += PatternMitigationMessage.objects.filter(user=user).delete()[0]
|
|
deleted += PatternMitigationCorrection.objects.filter(user=user).delete()[0]
|
|
deleted += PatternMitigationGame.objects.filter(user=user).delete()[0]
|
|
deleted += PatternMitigationRule.objects.filter(user=user).delete()[0]
|
|
deleted += PatternMitigationAutoSettings.objects.filter(user=user).delete()[0]
|
|
deleted += PatternMitigationPlan.objects.filter(user=user).delete()[0]
|
|
deleted += AIResultSignal.objects.filter(user=user).delete()[0]
|
|
deleted += AIResult.objects.filter(user=user).delete()[0]
|
|
deleted += AIRequest.objects.filter(user=user).delete()[0]
|
|
deleted += MemoryItem.objects.filter(user=user).delete()[0]
|
|
deleted += WorkspaceMetricSnapshot.objects.filter(
|
|
conversation__user=user
|
|
).delete()[0]
|
|
deleted += MessageEvent.objects.filter(user=user).delete()[0]
|
|
deleted += ConversationEvent.objects.filter(user=user).delete()[0]
|
|
deleted += AdapterHealthEvent.objects.filter(user=user).delete()[0]
|
|
deleted += Message.objects.filter(user=user).delete()[0]
|
|
deleted += QueuedMessage.objects.filter(user=user).delete()[0]
|
|
deleted += WorkspaceConversation.objects.filter(user=user).delete()[0]
|
|
deleted += ChatSession.objects.filter(user=user).delete()[0]
|
|
# Chat rows are legacy Signal cache rows and are not user-scoped.
|
|
deleted += Chat.objects.all().delete()[0]
|
|
return deleted
|
|
|
|
def _purge_osint_people(self, user):
|
|
return Person.objects.filter(user=user).delete()[0]
|
|
|
|
def _purge_osint_identifiers(self, user):
|
|
return PersonIdentifier.objects.filter(user=user).delete()[0]
|
|
|
|
def _purge_osint_groups(self, user):
|
|
return Group.objects.filter(user=user).delete()[0]
|
|
|
|
def _purge_osint_personas(self, user):
|
|
return Persona.objects.filter(user=user).delete()[0]
|
|
|
|
def _handle_action(self, request):
|
|
action = str(request.POST.get("action") or "").strip().lower()
|
|
if action == "purge_non_osint":
|
|
return (
|
|
"success",
|
|
f"Purged {self._purge_non_osint(request.user)} non-OSINT row(s).",
|
|
)
|
|
if action == "purge_osint_people":
|
|
return (
|
|
"warning",
|
|
f"Purged {self._purge_osint_people(request.user)} OSINT people row(s).",
|
|
)
|
|
if action == "purge_osint_identifiers":
|
|
return (
|
|
"warning",
|
|
f"Purged {self._purge_osint_identifiers(request.user)} OSINT identifier row(s).",
|
|
)
|
|
if action == "purge_osint_groups":
|
|
return (
|
|
"warning",
|
|
f"Purged {self._purge_osint_groups(request.user)} OSINT group row(s).",
|
|
)
|
|
if action == "purge_osint_personas":
|
|
return (
|
|
"warning",
|
|
f"Purged {self._purge_osint_personas(request.user)} OSINT persona row(s).",
|
|
)
|
|
return ("danger", "Unknown action.")
|
|
|
|
def _diagnostics_options(self, user):
|
|
session_rows = list(
|
|
ChatSession.objects.filter(user=user)
|
|
.select_related("identifier", "identifier__person")
|
|
.order_by("-last_interaction", "-id")[:120]
|
|
)
|
|
session_options = []
|
|
for row in session_rows:
|
|
identifier = getattr(row, "identifier", None)
|
|
person = getattr(identifier, "person", None) if identifier else None
|
|
session_options.append(
|
|
{
|
|
"id": str(row.id),
|
|
"label": " | ".join(
|
|
[
|
|
str(getattr(person, "name", "") or "-"),
|
|
str(row.id),
|
|
str(getattr(identifier, "service", "") or "-"),
|
|
str(getattr(identifier, "identifier", "") or "-"),
|
|
]
|
|
),
|
|
}
|
|
)
|
|
|
|
trace_options = []
|
|
seen_trace_ids = set()
|
|
for trace_id in (
|
|
ConversationEvent.objects.filter(user=user)
|
|
.exclude(trace_id="")
|
|
.order_by("-ts")
|
|
.values_list("trace_id", flat=True)[:400]
|
|
):
|
|
value = str(trace_id or "").strip()
|
|
if not value or value in seen_trace_ids:
|
|
continue
|
|
seen_trace_ids.add(value)
|
|
trace_options.append(value)
|
|
if len(trace_options) >= 120:
|
|
break
|
|
|
|
service_candidates = {"signal", "whatsapp", "xmpp", "instagram", "web"}
|
|
service_candidates.update(
|
|
str(item or "").strip().lower()
|
|
for item in ConversationEvent.objects.filter(user=user)
|
|
.exclude(origin_transport="")
|
|
.values_list("origin_transport", flat=True)
|
|
.distinct()[:50]
|
|
)
|
|
service_options = sorted(value for value in service_candidates if value)
|
|
|
|
event_type_candidates = {
|
|
"message_created",
|
|
"reaction_added",
|
|
"reaction_removed",
|
|
"read_receipt",
|
|
"message_updated",
|
|
"message_deleted",
|
|
}
|
|
event_type_candidates.update(
|
|
str(item or "").strip().lower()
|
|
for item in ConversationEvent.objects.filter(user=user)
|
|
.exclude(event_type="")
|
|
.values_list("event_type", flat=True)
|
|
.distinct()[:80]
|
|
)
|
|
event_type_options = sorted(value for value in event_type_candidates if value)
|
|
|
|
return {
|
|
"sessions": session_options,
|
|
"trace_ids": trace_options,
|
|
"services": service_options,
|
|
"event_types": event_type_options,
|
|
}
|
|
|
|
def _render_page(self, request, notice_level="", notice_message=""):
|
|
return render(
|
|
request,
|
|
self.template_name,
|
|
{
|
|
"counts": self._counts(request.user),
|
|
"notice_level": notice_level,
|
|
"notice_message": notice_message,
|
|
"diagnostics_options": self._diagnostics_options(request.user),
|
|
},
|
|
)
|
|
|
|
def get(self, request):
|
|
return self._render_page(request)
|
|
|
|
def post(self, request):
|
|
notice_level, notice_message = self._handle_action(request)
|
|
return self._render_page(
|
|
request,
|
|
notice_level=notice_level,
|
|
notice_message=notice_message,
|
|
)
|
|
|
|
|
|
class ServiceCapabilitySnapshotAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
service = str(request.GET.get("service") or "").strip().lower()
|
|
return JsonResponse(
|
|
{
|
|
"ok": True,
|
|
"data": capability_snapshot(service),
|
|
}
|
|
)
|
|
|
|
|
|
class AdapterHealthSummaryAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
latest_by_service = {}
|
|
rows = AdapterHealthEvent.objects.order_by("service", "-ts")[:200]
|
|
for row in rows:
|
|
key = str(row.service or "").strip().lower()
|
|
if key in latest_by_service:
|
|
continue
|
|
latest_by_service[key] = {
|
|
"status": str(row.status or ""),
|
|
"reason": str(row.reason or ""),
|
|
"ts": int(row.ts or 0),
|
|
"created_at": row.created_at.isoformat(),
|
|
}
|
|
return JsonResponse({"ok": True, "services": latest_by_service})
|
|
|
|
|
|
class TraceDiagnosticsAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
trace_id = str(request.GET.get("trace_id") or "").strip()
|
|
if not trace_id:
|
|
return JsonResponse(
|
|
{"ok": False, "error": "trace_id_required"},
|
|
status=400,
|
|
)
|
|
rows = list(
|
|
ConversationEvent.objects.filter(
|
|
user=request.user,
|
|
trace_id=trace_id,
|
|
)
|
|
.select_related("session")
|
|
.order_by("ts", "created_at")[:500]
|
|
)
|
|
related_session_ids = []
|
|
seen_sessions = set()
|
|
for row in rows:
|
|
session_id = str(row.session_id or "").strip()
|
|
if not session_id or session_id in seen_sessions:
|
|
continue
|
|
seen_sessions.add(session_id)
|
|
related_session_ids.append(session_id)
|
|
|
|
return JsonResponse(
|
|
{
|
|
"ok": True,
|
|
"trace_id": trace_id,
|
|
"count": len(rows),
|
|
"related_session_ids": related_session_ids,
|
|
"projection_shadow_urls": [
|
|
f"{reverse('system_projection_shadow')}?session_id={session_id}"
|
|
for session_id in related_session_ids
|
|
],
|
|
"events": [
|
|
{
|
|
"id": str(row.id),
|
|
"ts": int(row.ts or 0),
|
|
"event_type": str(row.event_type or ""),
|
|
"direction": str(row.direction or ""),
|
|
"session_id": str(row.session_id or ""),
|
|
"projection_shadow_url": (
|
|
f"{reverse('system_projection_shadow')}?session_id={str(row.session_id or '').strip()}"
|
|
if str(row.session_id or "").strip()
|
|
else ""
|
|
),
|
|
"origin_transport": str(row.origin_transport or ""),
|
|
"origin_message_id": str(row.origin_message_id or ""),
|
|
"payload": dict(row.payload or {}),
|
|
}
|
|
for row in rows
|
|
],
|
|
}
|
|
)
|
|
|
|
|
|
class EventProjectionShadowAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
session_id = str(request.GET.get("session_id") or "").strip()
|
|
if not session_id:
|
|
return JsonResponse(
|
|
{"ok": False, "error": "session_id_required"},
|
|
status=400,
|
|
)
|
|
detail_limit = int(request.GET.get("detail_limit") or 25)
|
|
session = ChatSession.objects.filter(
|
|
id=session_id,
|
|
user=request.user,
|
|
).first()
|
|
if session is None:
|
|
return JsonResponse(
|
|
{"ok": False, "error": "session_not_found"},
|
|
status=404,
|
|
)
|
|
compared = shadow_compare_session(session, detail_limit=max(0, detail_limit))
|
|
return JsonResponse(
|
|
{
|
|
"ok": True,
|
|
"result": compared,
|
|
"cause_summary": dict(compared.get("cause_counts") or {}),
|
|
"cause_samples": dict(compared.get("cause_samples") or {}),
|
|
}
|
|
)
|
|
|
|
|
|
class EventLedgerSmokeAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
minutes = max(1, int(request.GET.get("minutes") or 120))
|
|
service = str(request.GET.get("service") or "").strip().lower()
|
|
user_id = str(request.GET.get("user_id") or "").strip() or str(request.user.id)
|
|
limit = max(1, min(500, int(request.GET.get("limit") or 200)))
|
|
require_types_raw = str(request.GET.get("require_types") or "").strip()
|
|
required_types = [
|
|
item.strip().lower()
|
|
for item in require_types_raw.split(",")
|
|
if item.strip()
|
|
]
|
|
|
|
cutoff_ts = int(time.time() * 1000) - (minutes * 60 * 1000)
|
|
queryset = ConversationEvent.objects.filter(ts__gte=cutoff_ts).order_by("-ts")
|
|
if service:
|
|
queryset = queryset.filter(origin_transport=service)
|
|
if user_id:
|
|
queryset = queryset.filter(user_id=user_id)
|
|
|
|
rows = list(
|
|
queryset.values(
|
|
"id",
|
|
"user_id",
|
|
"session_id",
|
|
"ts",
|
|
"event_type",
|
|
"direction",
|
|
"origin_transport",
|
|
"trace_id",
|
|
)[:limit]
|
|
)
|
|
event_type_counts = {}
|
|
for row in rows:
|
|
key = str(row.get("event_type") or "")
|
|
event_type_counts[key] = int(event_type_counts.get(key) or 0) + 1
|
|
missing_required_types = [
|
|
event_type
|
|
for event_type in required_types
|
|
if int(event_type_counts.get(event_type) or 0) <= 0
|
|
]
|
|
return JsonResponse(
|
|
{
|
|
"ok": True,
|
|
"minutes": minutes,
|
|
"service": service,
|
|
"user_id": user_id,
|
|
"count": len(rows),
|
|
"event_type_counts": event_type_counts,
|
|
"required_types": required_types,
|
|
"missing_required_types": missing_required_types,
|
|
"sample": rows[:25],
|
|
}
|
|
)
|
|
|
|
|
|
class MemorySearchStatusAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
return JsonResponse({"ok": True, "status": backend_status()})
|
|
|
|
|
|
class MemorySearchQueryAPI(SuperUserRequiredMixin, View):
|
|
def get(self, request):
|
|
query = str(request.GET.get("q") or "").strip()
|
|
user_id = int(request.GET.get("user_id") or request.user.id)
|
|
conversation_id = str(request.GET.get("conversation_id") or "").strip()
|
|
limit = max(1, min(50, int(request.GET.get("limit") or 20)))
|
|
statuses = tuple(
|
|
item.strip().lower()
|
|
for item in str(request.GET.get("statuses") or "active").split(",")
|
|
if item.strip()
|
|
)
|
|
if not query:
|
|
return JsonResponse({"ok": False, "error": "query_required"}, status=400)
|
|
|
|
backend = get_memory_search_backend()
|
|
hits = backend.search(
|
|
user_id=user_id,
|
|
query=query,
|
|
conversation_id=conversation_id,
|
|
limit=limit,
|
|
include_statuses=statuses,
|
|
)
|
|
return JsonResponse(
|
|
{
|
|
"ok": True,
|
|
"backend": getattr(backend, "name", "unknown"),
|
|
"query": query,
|
|
"count": len(hits),
|
|
"hits": [
|
|
{
|
|
"memory_id": item.memory_id,
|
|
"score": item.score,
|
|
"summary": item.summary,
|
|
"payload": item.payload,
|
|
}
|
|
for item in hits
|
|
],
|
|
}
|
|
)
|