XF/GIA
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Manticore fully and re-theme

Browse Source
This commit is contained in:
Mark Veidemanis
2026-03-11 02:19:08 +00:00
parent da044be68c
commit cbedcd67f6
46 changed files with 3444 additions and 944 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
core/views/availability.py
View File

@@ -1,14 +1,12 @@
from __future__ import annotations
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Count, Max, Q
from django.shortcuts import render
from django.views import View
from core.models import (
ContactAvailabilityEvent,
ContactAvailabilitySettings,
)
from core.events.manticore import get_behavioral_availability_stats
from core.events.shadow import get_shadow_behavioral_availability_stats
from core.models import ContactAvailabilitySettings, Person
def _to_int(value, default=0):
@@ -64,42 +62,105 @@ class AvailabilitySettingsPage(LoginRequiredMixin, View):
return self.get(request)
def get(self, request):
settings_row = self._settings(request)
contact_stats = list(
ContactAvailabilityEvent.objects.filter(
user=request.user, person__isnull=False
)
.values("person_id", "person__name", "service")
.annotate(
total_events=Count("id"),
available_events=Count("id", filter=Q(availability_state="available")),
fading_events=Count("id", filter=Q(availability_state="fading")),
unavailable_events=Count(
"id", filter=Q(availability_state="unavailable")
),
unknown_events=Count("id", filter=Q(availability_state="unknown")),
native_presence_events=Count(
"id", filter=Q(source_kind="native_presence")
),
read_receipt_events=Count("id", filter=Q(source_kind="read_receipt")),
typing_events=Count(
"id",
filter=Q(source_kind="typing_start") | Q(source_kind="typing_stop"),
),
message_activity_events=Count(
"id",
filter=Q(source_kind="message_in") | Q(source_kind="message_out"),
),
inferred_timeout_events=Count(
"id", filter=Q(source_kind="inferred_timeout")
),
last_event_ts=Max("ts"),
)
.order_by("-total_events", "person__name", "service")
)
behavioral_stats, stats_source = self._behavioral_stats(request.user)
transport_stats = self._transport_stats(behavioral_stats)
totals = self._totals(behavioral_stats)
context = {
"settings_row": settings_row,
"contact_stats": contact_stats,
"settings_row": self._settings(request),
"behavioral_stats": behavioral_stats,
"behavioral_stats_source": stats_source,
"transport_stats": transport_stats,
"behavioral_totals": totals,
}
return render(request, self.template_name, context)
def _behavioral_stats(self, user):
try:
person_map = {
str(row["id"]): str(row["name"] or "")
for row in Person.objects.filter(user=user).values("id", "name")
}
rows = []
for row in list(
get_behavioral_availability_stats(user_id=int(user.id)) or []
):
person_id = str(row.get("person_id") or "").strip()
rows.append(
{
"person_id": person_id,
"person_name": person_map.get(person_id, person_id or "-"),
"service": str(row.get("transport") or "").strip().lower(),
"total_events": _to_int(row.get("total_events"), 0),
"presence_events": _to_int(row.get("presence_events"), 0),
"read_events": _to_int(row.get("read_events"), 0),
"typing_events": _to_int(row.get("typing_events"), 0),
"message_events": _to_int(row.get("message_events"), 0),
"abandoned_events": _to_int(row.get("abandoned_events"), 0),
"last_event_ts": _to_int(row.get("last_event_ts"), 0),
}
)
if rows:
return rows, "manticore"
except Exception:
pass
return list(get_shadow_behavioral_availability_stats(user=user)), "conversation_event_shadow"
def _transport_stats(self, behavioral_stats: list[dict]) -> list[dict]:
by_transport = {}
for row in list(behavioral_stats or []):
service = str(row.get("service") or "").strip().lower() or "-"
state = by_transport.setdefault(
service,
{
"service": service,
"contacts": 0,
"total_events": 0,
"presence_events": 0,
"read_events": 0,
"typing_events": 0,
"message_events": 0,
"abandoned_events": 0,
"last_event_ts": 0,
},
)
state["contacts"] += 1
for key in (
"total_events",
"presence_events",
"read_events",
"typing_events",
"message_events",
"abandoned_events",
):
state[key] += _to_int(row.get(key), 0)
state["last_event_ts"] = max(
int(state.get("last_event_ts") or 0),
_to_int(row.get("last_event_ts"), 0),
)
return sorted(
by_transport.values(),
key=lambda row: (-int(row.get("total_events") or 0), str(row.get("service") or "")),
)
def _totals(self, behavioral_stats: list[dict]) -> dict:
totals = {
"contacts": 0,
"total_events": 0,
"presence_events": 0,
"read_events": 0,
"typing_events": 0,
"message_events": 0,
"abandoned_events": 0,
}
for row in list(behavioral_stats or []):
totals["contacts"] += 1
for key in (
"total_events",
"presence_events",
"read_events",
"typing_events",
"message_events",
"abandoned_events",
):
totals[key] += _to_int(row.get(key), 0)
return totals

62
core/views/compose.py
View File

@@ -179,18 +179,23 @@ def _format_ts_label(ts_value: int) -> str:
def _serialize_availability_spans(spans):
def _value(row, key, default=None):
if isinstance(row, dict):
return row.get(key, default)
return getattr(row, key, default)
rows = []
for row in list(spans or []):
rows.append(
{
"id": int(getattr(row, "id", 0) or 0),
"service": str(getattr(row, "service", "") or ""),
"state": str(getattr(row, "state", "unknown") or "unknown"),
"start_ts": int(getattr(row, "start_ts", 0) or 0),
"end_ts": int(getattr(row, "end_ts", 0) or 0),
"confidence_start": float(getattr(row, "confidence_start", 0.0) or 0.0),
"confidence_end": float(getattr(row, "confidence_end", 0.0) or 0.0),
"payload": dict(getattr(row, "payload", {}) or {}),
"id": int(_value(row, "id", 0) or 0),
"service": str(_value(row, "service", "") or ""),
"state": str(_value(row, "state", "unknown") or "unknown"),
"start_ts": int(_value(row, "start_ts", 0) or 0),
"end_ts": int(_value(row, "end_ts", 0) or 0),
"confidence_start": float(_value(row, "confidence_start", 0.0) or 0.0),
"confidence_end": float(_value(row, "confidence_end", 0.0) or 0.0),
"payload": dict(_value(row, "payload", {}) or {}),
}
)
return rows
@@ -1765,8 +1770,31 @@ def _engage_source_from_ref(plan, source_ref):
def _context_base(user, service, identifier, person):
service = _default_service(service)
identifier_variants = _identifier_variants(service, identifier)
person_identifier = None
if person is None and identifier:
for candidate in identifier_variants or [identifier]:
bare_id = str(candidate or "").strip().split("@", 1)[0].strip()
if not bare_id:
continue
group_link = PlatformChatLink.objects.filter(
user=user,
service=service,
chat_identifier=bare_id,
is_group=True,
).first()
if group_link:
return {
"person_identifier": None,
"service": service,
"identifier": _group_channel_identifier(
service, group_link, bare_id
),
"person": None,
"is_group": True,
"group_name": group_link.chat_name or bare_id,
}
if person is not None:
if identifier_variants:
person_identifier = PersonIdentifier.objects.filter(
@@ -1811,24 +1839,6 @@ def _context_base(user, service, identifier, person):
if group_link is not None:
identifier = str(group_link.chat_jid or f"{bare_id}@g.us")
if person_identifier is None and identifier:
bare_id = identifier.split("@", 1)[0].strip()
group_link = PlatformChatLink.objects.filter(
user=user,
service=service,
chat_identifier=bare_id,
is_group=True,
).first()
if group_link:
return {
"person_identifier": None,
"service": service,
"identifier": _group_channel_identifier(service, group_link, bare_id),
"person": None,
"is_group": True,
"group_name": group_link.chat_name or bare_id,
}
return {
"person_identifier": person_identifier,
"service": service,

176
core/views/system.py
View File

@@ -7,6 +7,12 @@ from django.urls import reverse
from django.views import View
from core.clients import transport
from core.events.manticore import (
count_behavioral_events,
get_recent_event_rows,
get_trace_event_rows,
get_trace_ids,
)
from core.events.projection import shadow_compare_session
from core.memory.search_backend import backend_status, get_memory_search_backend
from core.models import (
@@ -56,12 +62,20 @@ class SystemSettings(SuperUserRequiredMixin, View):
template_name = "pages/system-settings.html"
def _counts(self, user):
behavioral_event_rows = 0
try:
behavioral_event_rows = int(count_behavioral_events(user_id=int(user.id)) or 0)
except Exception:
behavioral_event_rows = 0
return {
"chat_sessions": ChatSession.objects.filter(user=user).count(),
"messages": Message.objects.filter(user=user).count(),
"queued_messages": QueuedMessage.objects.filter(user=user).count(),
"message_events": MessageEvent.objects.filter(user=user).count(),
"conversation_events": ConversationEvent.objects.filter(user=user).count(),
"conversation_event_shadow_rows": ConversationEvent.objects.filter(
user=user
).count(),
"behavioral_event_rows": behavioral_event_rows,
"adapter_health_events": AdapterHealthEvent.objects.filter(
user=user
).count(),
@@ -203,6 +217,21 @@ class SystemSettings(SuperUserRequiredMixin, View):
trace_options.append(value)
if len(trace_options) >= 120:
break
if len(trace_options) < 120:
try:
for trace_id in get_trace_ids(
user_id=int(user.id),
limit=max(1, 120 - len(trace_options)),
):
value = str(trace_id or "").strip()
if not value or value in seen_trace_ids:
continue
seen_trace_ids.add(value)
trace_options.append(value)
if len(trace_options) >= 120:
break
except Exception:
pass
service_candidates = {"signal", "whatsapp", "xmpp", "instagram", "web"}
service_candidates.update(
@@ -212,6 +241,18 @@ class SystemSettings(SuperUserRequiredMixin, View):
.values_list("origin_transport", flat=True)
.distinct()[:50]
)
try:
service_candidates.update(
str(item.get("origin_transport") or "").strip().lower()
for item in get_recent_event_rows(
minutes=60 * 24 * 30,
user_id=str(user.id),
limit=500,
)
if str(item.get("origin_transport") or "").strip()
)
except Exception:
pass
service_options = sorted(value for value in service_candidates if value)
event_type_candidates = {
@@ -229,6 +270,18 @@ class SystemSettings(SuperUserRequiredMixin, View):
.values_list("event_type", flat=True)
.distinct()[:80]
)
try:
event_type_candidates.update(
str(item.get("event_type") or "").strip().lower()
for item in get_recent_event_rows(
minutes=60 * 24 * 30,
user_id=str(user.id),
limit=500,
)
if str(item.get("event_type") or "").strip()
)
except Exception:
pass
event_type_options = sorted(value for value in event_type_candidates if value)
return {
@@ -306,10 +359,26 @@ class TraceDiagnosticsAPI(SuperUserRequiredMixin, View):
.select_related("session")
.order_by("ts", "created_at")[:500]
)
data_source = "django"
if not rows:
try:
rows = get_trace_event_rows(
user_id=int(request.user.id),
trace_id=trace_id,
limit=500,
)
except Exception:
rows = []
if rows:
data_source = "manticore"
related_session_ids = []
seen_sessions = set()
for row in rows:
session_id = str(row.session_id or "").strip()
session_id = (
str(row.session_id or "").strip()
if not isinstance(row, dict)
else str(row.get("session_id") or "").strip()
)
if not session_id or session_id in seen_sessions:
continue
seen_sessions.add(session_id)
@@ -319,6 +388,7 @@ class TraceDiagnosticsAPI(SuperUserRequiredMixin, View):
{
"ok": True,
"trace_id": trace_id,
"data_source": data_source,
"count": len(rows),
"related_session_ids": related_session_ids,
"projection_shadow_urls": [
@@ -327,19 +397,56 @@ class TraceDiagnosticsAPI(SuperUserRequiredMixin, View):
],
"events": [
{
"id": str(row.id),
"ts": int(row.ts or 0),
"event_type": str(row.event_type or ""),
"direction": str(row.direction or ""),
"session_id": str(row.session_id or ""),
"id": (
str(row.id)
if not isinstance(row, dict)
else str(row.get("id") or "")
),
"ts": (
int(row.ts or 0)
if not isinstance(row, dict)
else int(row.get("ts") or 0)
),
"event_type": (
str(row.event_type or "")
if not isinstance(row, dict)
else str(row.get("event_type") or "")
),
"direction": (
str(row.direction or "")
if not isinstance(row, dict)
else str(row.get("direction") or "")
),
"session_id": (
str(row.session_id or "")
if not isinstance(row, dict)
else str(row.get("session_id") or "")
),
"projection_shadow_url": (
f"{reverse('system_projection_shadow')}?session_id={str(row.session_id or '').strip()}"
if str(row.session_id or "").strip()
f"{reverse('system_projection_shadow')}?session_id="
f"{(str(row.session_id or '').strip() if not isinstance(row, dict) else str(row.get('session_id') or '').strip())}"
if (
str(row.session_id or "").strip()
if not isinstance(row, dict)
else str(row.get("session_id") or "").strip()
)
else ""
),
"origin_transport": str(row.origin_transport or ""),
"origin_message_id": str(row.origin_message_id or ""),
"payload": dict(row.payload or {}),
"origin_transport": (
str(row.origin_transport or "")
if not isinstance(row, dict)
else str(row.get("origin_transport") or "")
),
"origin_message_id": (
str(row.origin_message_id or "")
if not isinstance(row, dict)
else str(row.get("origin_message_id") or "")
),
"payload": (
dict(row.payload or {})
if not isinstance(row, dict)
else dict(row.get("payload") or {})
),
}
for row in rows
],
@@ -377,18 +484,7 @@ class EventProjectionShadowAPI(SuperUserRequiredMixin, View):
class EventLedgerSmokeAPI(SuperUserRequiredMixin, View):
def get(self, request):
minutes = max(1, int(request.GET.get("minutes") or 120))
service = str(request.GET.get("service") or "").strip().lower()
user_id = str(request.GET.get("user_id") or "").strip() or str(request.user.id)
limit = max(1, min(500, int(request.GET.get("limit") or 200)))
require_types_raw = str(request.GET.get("require_types") or "").strip()
required_types = [
item.strip().lower()
for item in require_types_raw.split(",")
if item.strip()
]
def _recent_rows(self, *, minutes: int, service: str, user_id: str, limit: int):
cutoff_ts = int(time.time() * 1000) - (minutes * 60 * 1000)
queryset = ConversationEvent.objects.filter(ts__gte=cutoff_ts).order_by("-ts")
if service:
@@ -408,6 +504,37 @@ class EventLedgerSmokeAPI(SuperUserRequiredMixin, View):
"trace_id",
)[:limit]
)
if rows:
return rows, "django"
try:
manticore_rows = get_recent_event_rows(
minutes=minutes,
service=service,
user_id=user_id,
limit=limit,
)
except Exception:
manticore_rows = []
return manticore_rows, "manticore" if manticore_rows else "django"
def get(self, request):
minutes = max(1, int(request.GET.get("minutes") or 120))
service = str(request.GET.get("service") or "").strip().lower()
user_id = str(request.GET.get("user_id") or "").strip() or str(request.user.id)
limit = max(1, min(500, int(request.GET.get("limit") or 200)))
require_types_raw = str(request.GET.get("require_types") or "").strip()
required_types = [
item.strip().lower()
for item in require_types_raw.split(",")
if item.strip()
]
rows, data_source = self._recent_rows(
minutes=minutes,
service=service,
user_id=user_id,
limit=limit,
)
event_type_counts = {}
for row in rows:
key = str(row.get("event_type") or "")
@@ -423,6 +550,7 @@ class EventLedgerSmokeAPI(SuperUserRequiredMixin, View):
"minutes": minutes,
"service": service,
"user_id": user_id,
"data_source": data_source,
"count": len(rows),
"event_type_counts": event_type_counts,
"required_types": required_types,