XF/GIA
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Manticore fully and re-theme

Browse Source
This commit is contained in:
Mark Veidemanis
2026-03-11 02:19:08 +00:00
parent da044be68c
commit cbedcd67f6
46 changed files with 3444 additions and 944 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
core/management/commands/backfill_contact_availability.py
View File

@@ -4,13 +4,16 @@ from typing import Iterable
from django.core.management.base import BaseCommand
from core.events.ledger import append_event_sync
from core.models import Message
from core.presence import AvailabilitySignal, record_inferred_signal
from core.presence.inference import now_ms
class Command(BaseCommand):
help = "Backfill inferred contact availability events from historical message/read-receipt activity."
help = (
"Backfill behavioral event ledger rows from historical message and "
"read-receipt activity."
)
def add_arguments(self, parser):
parser.add_argument("--days", type=int, default=30)
@@ -39,17 +42,18 @@ class Command(BaseCommand):
user_filter = str(options.get("user_id") or "").strip()
dry_run = bool(options.get("dry_run"))
created = 0
indexed = 0
scanned = 0
for msg in self._iter_messages(
days=days, limit=limit, service=service_filter, user_id=user_filter
):
scanned += 1
identifier = getattr(getattr(msg, "session", None), "identifier", None)
session = getattr(msg, "session", None)
identifier = getattr(session, "identifier", None)
person = getattr(identifier, "person", None)
user = getattr(msg, "user", None)
if not identifier or not person or not user:
if not session or not identifier or not person or not user:
continue
service = (
@@ -60,76 +64,65 @@ class Command(BaseCommand):
if not service:
continue
base_ts = int(getattr(msg, "ts", 0) or 0)
message_author = (
str(getattr(msg, "custom_author", "") or "").strip().upper()
)
outgoing = message_author in {"USER", "BOT"}
author = str(getattr(msg, "custom_author", "") or "").strip().upper()
outgoing = author in {"USER", "BOT"}
message_id = str(
getattr(msg, "source_message_id", "") or f"django-message-{msg.id}"
).strip()
candidates = []
if base_ts > 0:
candidates.append(
{
"source_kind": "message_out" if outgoing else "message_in",
"availability_state": "available",
"confidence": 0.65 if outgoing else 0.75,
"ts": base_ts,
"payload": {
"origin": "backfill_contact_availability",
"message_id": str(msg.id),
"inferred_from": "message_activity",
},
}
if not dry_run:
append_event_sync(
user=user,
session=session,
ts=int(getattr(msg, "ts", 0) or 0),
event_type="message_created",
direction="out" if outgoing else "in",
actor_identifier=str(
getattr(msg, "sender_uuid", "") or identifier.identifier or ""
),
origin_transport=service,
origin_message_id=message_id,
origin_chat_id=str(getattr(msg, "source_chat_id", "") or ""),
payload={
"origin": "backfill_contact_availability",
"message_id": str(msg.id),
"text": str(getattr(msg, "text", "") or ""),
"outgoing": outgoing,
},
)
indexed += 1
read_ts = int(getattr(msg, "read_ts", 0) or 0)
if read_ts > 0:
candidates.append(
{
"source_kind": "read_receipt",
"availability_state": "available",
"confidence": 0.95,
"ts": read_ts,
"payload": {
"origin": "backfill_contact_availability",
"message_id": str(msg.id),
"inferred_from": "read_receipt",
"read_by": str(
getattr(msg, "read_by_identifier", "") or ""
),
},
}
)
for row in candidates:
exists = user.contact_availability_events.filter(
person=person,
person_identifier=identifier,
service=service,
source_kind=row["source_kind"],
ts=int(row["ts"]),
).exists()
if exists:
continue
created += 1
if dry_run:
continue
record_inferred_signal(
AvailabilitySignal(
user=user,
person=person,
person_identifier=identifier,
service=service,
source_kind=row["source_kind"],
availability_state=row["availability_state"],
confidence=float(row["confidence"]),
ts=int(row["ts"]),
payload=dict(row["payload"]),
)
if read_ts <= 0:
continue
if not dry_run:
append_event_sync(
user=user,
session=session,
ts=read_ts,
event_type="read_receipt",
direction="system",
actor_identifier=str(
getattr(msg, "read_by_identifier", "") or identifier.identifier
),
origin_transport=service,
origin_message_id=message_id,
origin_chat_id=str(getattr(msg, "source_chat_id", "") or ""),
payload={
"origin": "backfill_contact_availability",
"message_id": str(msg.id),
"message_ts": int(getattr(msg, "ts", 0) or 0),
"read_by": str(
getattr(msg, "read_by_identifier", "") or ""
).strip(),
},
)
indexed += 1
self.stdout.write(
self.style.SUCCESS(
f"backfill_contact_availability complete scanned={scanned} created={created} dry_run={dry_run} days={days} limit={limit}"
"backfill_contact_availability complete "
f"scanned={scanned} indexed={indexed} dry_run={dry_run} "
f"days={days} limit={limit}"
)
)

62
core/management/commands/event_ledger_smoke.py
View File

@@ -5,12 +5,46 @@ import time
from django.core.management.base import BaseCommand, CommandError
from core.events.manticore import get_recent_event_rows
from core.models import ConversationEvent
class Command(BaseCommand):
help = "Quick non-mutating sanity check for recent canonical event writes."
def _recent_rows(self, *, minutes: int, service: str, user_id: str, limit: int):
cutoff_ts = int(time.time() * 1000) - (minutes * 60 * 1000)
queryset = ConversationEvent.objects.filter(ts__gte=cutoff_ts).order_by("-ts")
if service:
queryset = queryset.filter(origin_transport=service)
if user_id:
queryset = queryset.filter(user_id=user_id)
rows = list(
queryset.values(
"id",
"user_id",
"session_id",
"ts",
"event_type",
"direction",
"origin_transport",
"trace_id",
)[:limit]
)
if rows:
return rows, "django"
try:
manticore_rows = get_recent_event_rows(
minutes=minutes,
service=service,
user_id=user_id,
limit=limit,
)
except Exception:
manticore_rows = []
return manticore_rows, "manticore" if manticore_rows else "django"
def add_arguments(self, parser):
parser.add_argument("--minutes", type=int, default=120)
parser.add_argument("--service", default="")
@@ -34,24 +68,11 @@ class Command(BaseCommand):
if item.strip()
]
cutoff_ts = int(time.time() * 1000) - (minutes * 60 * 1000)
queryset = ConversationEvent.objects.filter(ts__gte=cutoff_ts).order_by("-ts")
if service:
queryset = queryset.filter(origin_transport=service)
if user_id:
queryset = queryset.filter(user_id=user_id)
rows = list(
queryset.values(
"id",
"user_id",
"session_id",
"ts",
"event_type",
"direction",
"origin_transport",
"trace_id",
)[:limit]
rows, data_source = self._recent_rows(
minutes=minutes,
service=service,
user_id=user_id,
limit=limit,
)
event_type_counts = {}
for row in rows:
@@ -67,6 +88,7 @@ class Command(BaseCommand):
"minutes": minutes,
"service": service,
"user_id": user_id,
"data_source": data_source,
"count": len(rows),
"event_type_counts": event_type_counts,
"required_types": required_types,
@@ -79,7 +101,7 @@ class Command(BaseCommand):
return
self.stdout.write(
f"event-ledger-smoke minutes={minutes} service={service or '-'} user={user_id or '-'} count={len(rows)}"
f"event-ledger-smoke minutes={minutes} service={service or '-'} user={user_id or '-'} source={data_source} count={len(rows)}"
)
self.stdout.write(f"event_type_counts={event_type_counts}")
if required_types:
@@ -88,7 +110,7 @@ class Command(BaseCommand):
)
if fail_if_empty and len(rows) == 0:
raise CommandError("No recent ConversationEvent rows found.")
raise CommandError("No recent canonical event rows found.")
if missing_required_types:
raise CommandError(
"Missing required event types: " + ", ".join(missing_required_types)

96
core/management/commands/gia_analysis.py Normal file
View File

@@ -0,0 +1,96 @@
from __future__ import annotations
import time
from django.core.management.base import BaseCommand
from core.events.behavior import summarize_metrics
from core.events.manticore import get_event_ledger_backend
from core.util import logs
log = logs.get_logger("gia_analysis")
class Command(BaseCommand):
help = "Compute behavioral metrics from Manticore event rows into gia_metrics."
def add_arguments(self, parser):
parser.add_argument("--once", action="store_true", default=False)
parser.add_argument("--user-id", type=int)
parser.add_argument("--person-id")
parser.add_argument("--sleep-seconds", type=float, default=60.0)
parser.add_argument("--window-days", nargs="*", type=int, default=[1, 7, 30, 90])
def _run_cycle(
self,
*,
user_id: int | None = None,
person_id: str = "",
window_days: list[int] | None = None,
) -> int:
backend = get_event_ledger_backend()
now_ms = int(time.time() * 1000)
baseline_since = now_ms - (90 * 86400000)
windows = sorted({max(1, int(value)) for value in list(window_days or [1, 7, 30, 90])})
targets = backend.list_event_targets(user_id=user_id)
if person_id:
targets = [
row
for row in targets
if str(row.get("person_id") or "").strip() == str(person_id).strip()
]
written = 0
for target in targets:
target_user_id = int(target.get("user_id") or 0)
target_person_id = str(target.get("person_id") or "").strip()
if target_user_id <= 0 or not target_person_id:
continue
baseline_rows = backend.fetch_events(
user_id=target_user_id,
person_id=target_person_id,
since_ts=baseline_since,
)
if not baseline_rows:
continue
for window in windows:
since_ts = now_ms - (int(window) * 86400000)
window_rows = [
row
for row in baseline_rows
if int(row.get("ts") or 0) >= since_ts
]
metrics = summarize_metrics(window_rows, baseline_rows)
for metric, values in metrics.items():
backend.upsert_metric(
user_id=target_user_id,
person_id=target_person_id,
window_days=int(window),
metric=metric,
value_ms=int(values.get("value_ms") or 0),
baseline_ms=int(values.get("baseline_ms") or 0),
z_score=float(values.get("z_score") or 0.0),
sample_n=int(values.get("sample_n") or 0),
computed_at=now_ms,
)
written += 1
return written
def handle(self, *args, **options):
once = bool(options.get("once"))
sleep_seconds = max(1.0, float(options.get("sleep_seconds") or 60.0))
user_id = options.get("user_id")
person_id = str(options.get("person_id") or "").strip()
window_days = list(options.get("window_days") or [1, 7, 30, 90])
while True:
written = self._run_cycle(
user_id=user_id,
person_id=person_id,
window_days=window_days,
)
self.stdout.write(f"gia-analysis wrote={written}")
if once:
return
time.sleep(sleep_seconds)

46
core/management/commands/manticore_backfill.py Normal file
View File

@@ -0,0 +1,46 @@
from __future__ import annotations
from django.core.management.base import BaseCommand, CommandError
from core.events.manticore import upsert_conversation_event
from core.models import ConversationEvent
class Command(BaseCommand):
help = "Backfill behavioral events into Manticore from ConversationEvent rows."
def add_arguments(self, parser):
parser.add_argument(
"--from-conversation-events",
action="store_true",
help="Replay ConversationEvent rows into the Manticore event table.",
)
parser.add_argument("--user-id", type=int, default=None)
parser.add_argument("--limit", type=int, default=5000)
def handle(self, *args, **options):
if not bool(options.get("from_conversation_events")):
raise CommandError("Pass --from-conversation-events to run this backfill.")
queryset = (
ConversationEvent.objects.select_related("session__identifier")
.order_by("ts", "created_at")
)
user_id = options.get("user_id")
if user_id is not None:
queryset = queryset.filter(user_id=int(user_id))
scanned = 0
indexed = 0
limit = max(1, int(options.get("limit") or 5000))
for event in queryset[:limit]:
scanned += 1
upsert_conversation_event(event)
indexed += 1
self.stdout.write(
self.style.SUCCESS(
"manticore-backfill scanned=%s indexed=%s user=%s"
% (scanned, indexed, user_id if user_id is not None else "-")
)
)

62
core/management/commands/prune_behavioral_orm_data.py Normal file
View File

@@ -0,0 +1,62 @@
from __future__ import annotations
import time
from django.conf import settings
from django.core.management.base import BaseCommand
from core.models import ConversationEvent
class Command(BaseCommand):
help = (
"Prune high-growth behavioral ORM shadow tables after data has been "
"persisted to Manticore."
)
def add_arguments(self, parser):
parser.add_argument("--user-id", default="")
parser.add_argument("--dry-run", action="store_true", default=False)
parser.add_argument("--conversation-days", type=int)
parser.add_argument(
"--tables",
default="conversation_events",
help="Comma separated subset of: conversation_events",
)
def _cutoff_ms(self, days: int) -> int:
return int(time.time() * 1000) - (max(1, int(days)) * 24 * 60 * 60 * 1000)
def handle(self, *args, **options):
user_id = str(options.get("user_id") or "").strip()
dry_run = bool(options.get("dry_run"))
conversation_days = int(
options.get("conversation_days")
or getattr(settings, "CONVERSATION_EVENT_RETENTION_DAYS", 90)
or 90
)
selected_tables = {
str(item or "").strip().lower()
for item in str(options.get("tables") or "").split(",")
if str(item or "").strip()
}
deleted = {
"conversation_events": 0,
}
if "conversation_events" in selected_tables:
qs = ConversationEvent.objects.filter(
ts__lt=self._cutoff_ms(conversation_days)
)
if user_id:
qs = qs.filter(user_id=user_id)
deleted["conversation_events"] = int(qs.count() if dry_run else qs.delete()[0])
self.stdout.write(
"prune-behavioral-orm-data "
f"dry_run={dry_run} "
f"user_id={user_id or '-'} "
f"conversation_days={conversation_days} "
f"deleted={deleted}"
)

266
core/management/commands/recalculate_contact_availability.py
View File

@@ -2,22 +2,15 @@ from __future__ import annotations
from django.core.management.base import BaseCommand
from core.models import ContactAvailabilityEvent, ContactAvailabilitySpan, Message
from core.presence import AvailabilitySignal, record_native_signal
from core.events.ledger import append_event_sync
from core.models import Message
from core.presence.inference import now_ms
_SOURCE_ORDER = {
"message_in": 10,
"message_out": 20,
"read_receipt": 30,
"native_presence": 40,
}
class Command(BaseCommand):
help = (
"Recalculate contact availability events/spans from persisted message, "
"read-receipt, and reaction history (deterministic rebuild)."
"Replay behavioral event ledger rows from persisted message, receipt, "
"and reaction history."
)
def add_arguments(self, parser):
@@ -39,70 +32,93 @@ class Command(BaseCommand):
qs = qs.filter(user_id=str(user_id).strip())
return qs.order_by("ts")[: max(1, int(limit))]
def _build_event_rows(self, messages):
rows = []
def handle(self, *args, **options):
days = max(1, int(options.get("days") or 90))
limit = max(1, int(options.get("limit") or 20000))
service_filter = str(options.get("service") or "").strip().lower()
user_filter = str(options.get("user_id") or "").strip()
dry_run = bool(options.get("dry_run"))
messages = list(
self._iter_messages(
days=days,
limit=limit,
service=service_filter,
user_id=user_filter,
)
)
indexed = 0
for msg in messages:
identifier = getattr(getattr(msg, "session", None), "identifier", None)
session = getattr(msg, "session", None)
identifier = getattr(session, "identifier", None)
person = getattr(identifier, "person", None)
user = getattr(msg, "user", None)
if not identifier or not person or not user:
if not session or not identifier or not person or not user:
continue
service = (
str(
getattr(msg, "source_service", "")
or getattr(identifier, "service", "")
)
str(getattr(msg, "source_service", "") or identifier.service or "")
.strip()
.lower()
)
if not service:
continue
ts = int(getattr(msg, "ts", 0) or 0)
if ts > 0:
author = str(getattr(msg, "custom_author", "") or "").strip().upper()
outgoing = author in {"USER", "BOT"}
rows.append(
{
"user": user,
"person": person,
"person_identifier": identifier,
"service": service,
"source_kind": "message_out" if outgoing else "message_in",
"availability_state": "available",
"confidence": 0.65 if outgoing else 0.75,
"ts": ts,
"payload": {
"origin": "recalculate_contact_availability",
"message_id": str(msg.id),
"inferred_from": "message_activity",
},
}
author = str(getattr(msg, "custom_author", "") or "").strip().upper()
outgoing = author in {"USER", "BOT"}
message_id = str(
getattr(msg, "source_message_id", "") or f"django-message-{msg.id}"
).strip()
if not dry_run:
append_event_sync(
user=user,
session=session,
ts=int(getattr(msg, "ts", 0) or 0),
event_type="message_created",
direction="out" if outgoing else "in",
actor_identifier=str(
getattr(msg, "sender_uuid", "") or identifier.identifier or ""
),
origin_transport=service,
origin_message_id=message_id,
origin_chat_id=str(getattr(msg, "source_chat_id", "") or ""),
payload={
"origin": "recalculate_contact_availability",
"message_id": str(msg.id),
"text": str(getattr(msg, "text", "") or ""),
"outgoing": outgoing,
},
)
indexed += 1
read_ts = int(getattr(msg, "read_ts", 0) or 0)
if read_ts > 0:
rows.append(
{
"user": user,
"person": person,
"person_identifier": identifier,
"service": service,
"source_kind": "read_receipt",
"availability_state": "available",
"confidence": 0.95,
"ts": read_ts,
"payload": {
if not dry_run:
append_event_sync(
user=user,
session=session,
ts=read_ts,
event_type="read_receipt",
direction="system",
actor_identifier=str(
getattr(msg, "read_by_identifier", "")
or identifier.identifier
),
origin_transport=service,
origin_message_id=message_id,
origin_chat_id=str(getattr(msg, "source_chat_id", "") or ""),
payload={
"origin": "recalculate_contact_availability",
"message_id": str(msg.id),
"inferred_from": "read_receipt",
"message_ts": int(getattr(msg, "ts", 0) or 0),
"read_by": str(
getattr(msg, "read_by_identifier", "") or ""
),
).strip(),
},
}
)
)
indexed += 1
reactions = list(
(getattr(msg, "receipt_payload", {}) or {}).get("reactions") or []
@@ -114,138 +130,32 @@ class Command(BaseCommand):
reaction_ts = int(item.get("updated_at") or 0)
if reaction_ts <= 0:
continue
rows.append(
{
"user": user,
"person": person,
"person_identifier": identifier,
"service": service,
"source_kind": "native_presence",
"availability_state": "available",
"confidence": 0.9,
"ts": reaction_ts,
"payload": {
if not dry_run:
append_event_sync(
user=user,
session=session,
ts=reaction_ts,
event_type="presence_available",
direction="system",
actor_identifier=str(item.get("actor") or ""),
origin_transport=service,
origin_message_id=message_id,
origin_chat_id=str(getattr(msg, "source_chat_id", "") or ""),
payload={
"origin": "recalculate_contact_availability",
"message_id": str(msg.id),
"inferred_from": "reaction",
"emoji": str(item.get("emoji") or ""),
"actor": str(item.get("actor") or ""),
"source_service": str(
item.get("source_service") or service
),
"source_service": str(item.get("source_service") or service),
},
}
)
rows.sort(
key=lambda row: (
str(getattr(row["user"], "id", "")),
str(getattr(row["person"], "id", "")),
str(row.get("service") or ""),
int(row.get("ts") or 0),
_SOURCE_ORDER.get(str(row.get("source_kind") or ""), 999),
str((row.get("payload") or {}).get("message_id") or ""),
)
)
return rows
def handle(self, *args, **options):
days = max(1, int(options.get("days") or 90))
limit = max(1, int(options.get("limit") or 20000))
service_filter = str(options.get("service") or "").strip().lower()
user_filter = str(options.get("user_id") or "").strip()
dry_run = bool(options.get("dry_run"))
reset = not bool(options.get("no_reset"))
cutoff_ts = now_ms() - (days * 24 * 60 * 60 * 1000)
messages = list(
self._iter_messages(
days=days,
limit=limit,
service=service_filter,
user_id=user_filter,
)
)
rows = self._build_event_rows(messages)
keys_to_reset = set()
for row in rows:
keys_to_reset.add(
(
str(getattr(row["user"], "id", "")),
str(getattr(row["person"], "id", "")),
str(row.get("service") or ""),
)
)
deleted_events = 0
deleted_spans = 0
if reset and keys_to_reset and not dry_run:
for user_id, person_id, service in keys_to_reset:
deleted_events += ContactAvailabilityEvent.objects.filter(
user_id=user_id,
person_id=person_id,
service=service,
ts__gte=cutoff_ts,
).delete()[0]
deleted_spans += ContactAvailabilitySpan.objects.filter(
user_id=user_id,
person_id=person_id,
service=service,
end_ts__gte=cutoff_ts,
).delete()[0]
created = 0
dedup_seen = set()
for row in rows:
dedup_key = (
str(getattr(row["user"], "id", "")),
str(getattr(row["person"], "id", "")),
str(getattr(row["person_identifier"], "id", "")),
str(row.get("service") or ""),
str(row.get("source_kind") or ""),
int(row.get("ts") or 0),
str((row.get("payload") or {}).get("message_id") or ""),
str((row.get("payload") or {}).get("inferred_from") or ""),
)
if dedup_key in dedup_seen:
continue
dedup_seen.add(dedup_key)
if not reset:
exists = ContactAvailabilityEvent.objects.filter(
user=row["user"],
person=row["person"],
person_identifier=row["person_identifier"],
service=row["service"],
source_kind=row["source_kind"],
ts=row["ts"],
).exists()
if exists:
continue
created += 1
if dry_run:
continue
record_native_signal(
AvailabilitySignal(
user=row["user"],
person=row["person"],
person_identifier=row["person_identifier"],
service=row["service"],
source_kind=row["source_kind"],
availability_state=row["availability_state"],
confidence=float(row["confidence"]),
ts=int(row["ts"]),
payload=dict(row["payload"]),
)
)
)
indexed += 1
self.stdout.write(
self.style.SUCCESS(
"recalculate_contact_availability complete "
f"messages_scanned={len(messages)} candidates={len(rows)} "
f"created={created} deleted_events={deleted_events} deleted_spans={deleted_spans} "
f"reset={reset} dry_run={dry_run} days={days} limit={limit}"
f"messages_scanned={len(messages)} indexed={indexed} "
f"dry_run={dry_run} no_reset={bool(options.get('no_reset'))} "
f"days={days} limit={limit}"
)
)