Improve search

This commit is contained in:
2026-03-02 02:26:25 +00:00
parent a9f5f3f75d
commit b94219fc5b
20 changed files with 1626 additions and 314 deletions

View File

@@ -66,6 +66,16 @@ urlpatterns = [
automation.CommandRoutingSettings.as_view(), automation.CommandRoutingSettings.as_view(),
name="command_routing", name="command_routing",
), ),
path(
"settings/ai-execution/",
automation.AIExecutionLogSettings.as_view(),
name="ai_execution_log",
),
path(
"settings/translation/",
automation.TranslationSettings.as_view(),
name="translation_settings",
),
path( path(
"settings/business-plan/<str:doc_id>/", "settings/business-plan/<str:doc_id>/",
automation.BusinessPlanEditor.as_view(), automation.BusinessPlanEditor.as_view(),

View File

@@ -588,7 +588,11 @@ class HandleMessage(Command):
None, None,
) )
log.info("Running Signal mutate prompt") log.info("Running Signal mutate prompt")
result = await ai.run_prompt(prompt, manip.ai) result = await ai.run_prompt(
prompt,
manip.ai,
operation="signal_mutate",
)
log.info( log.info(
f"Sending {len(xmpp_attachments)} attachments from Signal to XMPP." f"Sending {len(xmpp_attachments)} attachments from Signal to XMPP."
) )
@@ -733,7 +737,11 @@ class HandleMessage(Command):
) )
log.info("Running context prompt") log.info("Running context prompt")
result = await ai.run_prompt(prompt, manip.ai) result = await ai.run_prompt(
prompt,
manip.ai,
operation="signal_reply",
)
if manip.mode == "active": if manip.mode == "active":
await history.store_own_message( await history.store_own_message(
session=chat_session, session=chat_session,

View File

@@ -1313,7 +1313,11 @@ class XMPPComponent(ComponentXMPP):
chat_history, chat_history,
) )
self.log.debug("Running XMPP context prompt") self.log.debug("Running XMPP context prompt")
result = await ai.run_prompt(prompt, manip.ai) result = await ai.run_prompt(
prompt,
manip.ai,
operation="xmpp_mutate",
)
self.log.debug("Generated mutated response for XMPP message") self.log.debug("Generated mutated response for XMPP message")
await history.store_own_message( await history.store_own_message(
session=session, session=session,

View File

@@ -299,7 +299,12 @@ class BPCommandHandler(CommandHandler):
}, },
] ]
try: try:
summary = str(await ai_runner.run_prompt(prompt, ai_obj) or "").strip() summary = str(
await ai_runner.run_prompt(
prompt, ai_obj, operation="command_bp_extract"
)
or ""
).strip()
if not summary: if not summary:
raise RuntimeError("empty_ai_response") raise RuntimeError("empty_ai_response")
except Exception as exc: except Exception as exc:

View File

@@ -1,20 +1,68 @@
import time
from asgiref.sync import sync_to_async
from django.utils import timezone
from openai import AsyncOpenAI from openai import AsyncOpenAI
from core.models import AI from core.models import AI, AIRunLog
def _prompt_metrics(prompt: list[dict]) -> tuple[int, int]:
rows = list(prompt or [])
message_count = len(rows)
prompt_chars = 0
for row in rows:
content = ""
if isinstance(row, dict):
content = str(row.get("content") or "")
else:
content = str(row or "")
prompt_chars += len(content)
return message_count, prompt_chars
async def run_prompt( async def run_prompt(
prompt: list[str], prompt: list[dict],
ai: AI, ai: AI,
operation: str = "",
): ):
started_monotonic = time.perf_counter()
message_count, prompt_chars = _prompt_metrics(prompt)
run_log = await sync_to_async(AIRunLog.objects.create)(
user=ai.user,
ai=ai,
operation=str(operation or "").strip(),
model=str(ai.model or ""),
base_url=str(ai.base_url or ""),
status="running",
message_count=message_count,
prompt_chars=prompt_chars,
)
cast = {"api_key": ai.api_key} cast = {"api_key": ai.api_key}
if ai.base_url is not None: if ai.base_url is not None:
cast["base_url"] = ai.base_url cast["base_url"] = ai.base_url
client = AsyncOpenAI(**cast) client = AsyncOpenAI(**cast)
try:
response = await client.chat.completions.create( response = await client.chat.completions.create(
model=ai.model, model=ai.model,
messages=prompt, messages=prompt,
) )
content = response.choices[0].message.content content = response.choices[0].message.content
duration_ms = int((time.perf_counter() - started_monotonic) * 1000)
await sync_to_async(AIRunLog.objects.filter(id=run_log.id).update)(
status="ok",
response_chars=len(str(content or "")),
finished_at=timezone.now(),
duration_ms=duration_ms,
error="",
)
return content return content
except Exception as exc:
duration_ms = int((time.perf_counter() - started_monotonic) * 1000)
await sync_to_async(AIRunLog.objects.filter(id=run_log.id).update)(
status="failed",
finished_at=timezone.now(),
duration_ms=duration_ms,
error=str(exc),
)
raise

View File

@@ -1,6 +1,6 @@
from django.utils import timezone from django.utils import timezone
from openai import AsyncOpenAI
from core.messaging import ai as ai_runner
from core.models import AI, Manipulation, Person from core.models import AI, Manipulation, Person
@@ -50,17 +50,11 @@ def generate_prompt(msg: dict, person: Person, manip: Manipulation, chat_history
async def run_context_prompt( async def run_context_prompt(
prompt: list[str], prompt: list[dict],
ai: AI, ai: AI,
): ):
cast = {"api_key": ai.api_key} return await ai_runner.run_prompt(
if ai.base_url is not None: prompt,
cast["base_url"] = ai.base_url ai,
client = AsyncOpenAI(**cast) operation="analysis_context",
response = await client.chat.completions.create(
model=ai.model,
messages=prompt,
) )
content = response.choices[0].message.content
return content

View File

@@ -0,0 +1,86 @@
# Generated by Django 5.2.7 on 2026-03-02 00:00
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("core", "0027_businessplandocument_businessplanrevision_and_more"),
]
operations = [
migrations.CreateModel(
name="AIRunLog",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("operation", models.CharField(blank=True, default="", max_length=64)),
("model", models.CharField(blank=True, default="", max_length=255)),
("base_url", models.CharField(blank=True, default="", max_length=255)),
(
"status",
models.CharField(
choices=[("running", "Running"), ("ok", "OK"), ("failed", "Failed")],
default="running",
max_length=16,
),
),
("message_count", models.PositiveIntegerField(default=0)),
("prompt_chars", models.PositiveIntegerField(default=0)),
("response_chars", models.PositiveIntegerField(default=0)),
("error", models.TextField(blank=True, default="")),
("started_at", models.DateTimeField(auto_now_add=True)),
("finished_at", models.DateTimeField(blank=True, null=True)),
("duration_ms", models.PositiveIntegerField(blank=True, null=True)),
(
"ai",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="run_logs",
to="core.ai",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="ai_run_logs",
to=settings.AUTH_USER_MODEL,
),
),
],
),
migrations.AddIndex(
model_name="airunlog",
index=models.Index(fields=["user", "started_at"], name="core_airunl_user_id_6f4700_idx"),
),
migrations.AddIndex(
model_name="airunlog",
index=models.Index(
fields=["user", "status", "started_at"],
name="core_airunl_user_id_b4486e_idx",
),
),
migrations.AddIndex(
model_name="airunlog",
index=models.Index(
fields=["user", "operation", "started_at"],
name="core_airunl_user_id_4f0f5e_idx",
),
),
migrations.AddIndex(
model_name="airunlog",
index=models.Index(fields=["user", "model", "started_at"], name="core_airunl_user_id_953bff_idx"),
),
]

View File

@@ -128,6 +128,42 @@ class AI(models.Model):
return f"{self.id} - {self.model}" return f"{self.id} - {self.model}"
class AIRunLog(models.Model):
STATUS_CHOICES = (
("running", "Running"),
("ok", "OK"),
("failed", "Failed"),
)
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="ai_run_logs")
ai = models.ForeignKey(
AI,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="run_logs",
)
operation = models.CharField(max_length=64, blank=True, default="")
model = models.CharField(max_length=255, blank=True, default="")
base_url = models.CharField(max_length=255, blank=True, default="")
status = models.CharField(max_length=16, choices=STATUS_CHOICES, default="running")
message_count = models.PositiveIntegerField(default=0)
prompt_chars = models.PositiveIntegerField(default=0)
response_chars = models.PositiveIntegerField(default=0)
error = models.TextField(blank=True, default="")
started_at = models.DateTimeField(auto_now_add=True)
finished_at = models.DateTimeField(null=True, blank=True)
duration_ms = models.PositiveIntegerField(null=True, blank=True)
class Meta:
indexes = [
models.Index(fields=["user", "started_at"]),
models.Index(fields=["user", "status", "started_at"]),
models.Index(fields=["user", "operation", "started_at"]),
models.Index(fields=["user", "model", "started_at"]),
]
class Person(models.Model): class Person(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, on_delete=models.CASCADE) user = models.ForeignKey(User, on_delete=models.CASCADE)

View File

@@ -390,11 +390,17 @@
Notifications Notifications
</a> </a>
<a class="navbar-item" href="{% url 'ais' type='page' %}"> <a class="navbar-item" href="{% url 'ais' type='page' %}">
AI Models
</a> </a>
<a class="navbar-item" href="{% url 'command_routing' %}"> <a class="navbar-item" href="{% url 'command_routing' %}">
Command Routing Command Routing
</a> </a>
<a class="navbar-item" href="{% url 'translation_settings' %}">
Translation
</a>
<a class="navbar-item" href="{% url 'ai_execution_log' %}">
AI Execution Log
</a>
{% if user.is_superuser %} {% if user.is_superuser %}
<a class="navbar-item" href="{% url 'system_settings' %}"> <a class="navbar-item" href="{% url 'system_settings' %}">
System System

View File

@@ -0,0 +1,119 @@
{% extends "base.html" %}
{% block content %}
<style>
.ai-stat-box {
height: 100%;
min-height: 92px;
margin: 0;
}
</style>
<section class="section">
<div class="container">
<h1 class="title is-4">AI Execution Log</h1>
<p class="subtitle is-6">Tracked model calls and usage metrics for this account.</p>
<article class="box">
<div class="columns is-multiline">
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Total Runs</p><p class="title is-6">{{ stats.total_runs }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">OK</p><p class="title is-6 has-text-success">{{ stats.total_ok }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Failed</p><p class="title is-6 has-text-danger">{{ stats.total_failed }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Success Rate</p><p class="title is-6 has-text-info">{{ stats.success_rate }}%</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">24h Runs</p><p class="title is-6">{{ stats.last_24h_runs }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">24h Failed</p><p class="title is-6 has-text-warning">{{ stats.last_24h_failed }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">7d Runs</p><p class="title is-6">{{ stats.last_7d_runs }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Avg Duration</p><p class="title is-6">{{ stats.avg_duration_ms }}ms</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Prompt Chars</p><p class="title is-6">{{ stats.total_prompt_chars }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Response Chars</p><p class="title is-6">{{ stats.total_response_chars }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Avg Prompt</p><p class="title is-6">{{ stats.avg_prompt_chars }}</p></div></div>
<div class="column is-6-mobile is-4-tablet is-3-desktop"><div class="box ai-stat-box"><p class="heading">Avg Response</p><p class="title is-6">{{ stats.avg_response_chars }}</p></div></div>
</div>
</article>
<div class="columns">
<div class="column is-6">
<article class="box">
<h2 class="title is-6">By Operation</h2>
<table class="table is-fullwidth is-size-7 is-striped">
<thead>
<tr><th>Operation</th><th>Total</th><th>OK</th><th>Failed</th></tr>
</thead>
<tbody>
{% for row in operation_breakdown %}
<tr>
<td>{{ row.operation|default:"(none)" }}</td>
<td>{{ row.total }}</td>
<td>{{ row.ok }}</td>
<td>{{ row.failed }}</td>
</tr>
{% empty %}
<tr><td colspan="4">No runs yet.</td></tr>
{% endfor %}
</tbody>
</table>
</article>
</div>
<div class="column is-6">
<article class="box">
<h2 class="title is-6">By Model</h2>
<table class="table is-fullwidth is-size-7 is-striped">
<thead>
<tr><th>Model</th><th>Total</th><th>OK</th><th>Failed</th></tr>
</thead>
<tbody>
{% for row in model_breakdown %}
<tr>
<td>{{ row.model|default:"(none)" }}</td>
<td>{{ row.total }}</td>
<td>{{ row.ok }}</td>
<td>{{ row.failed }}</td>
</tr>
{% empty %}
<tr><td colspan="4">No runs yet.</td></tr>
{% endfor %}
</tbody>
</table>
</article>
</div>
</div>
<article class="box">
<h2 class="title is-6">Recent Runs</h2>
<div class="table-container">
<table class="table is-fullwidth is-size-7 is-striped">
<thead>
<tr>
<th>Started</th>
<th>Status</th>
<th>Operation</th>
<th>Model</th>
<th>Messages</th>
<th>Prompt</th>
<th>Response</th>
<th>Duration</th>
<th>Error</th>
</tr>
</thead>
<tbody>
{% for run in runs %}
<tr>
<td>{{ run.started_at }}</td>
<td>{{ run.status }}</td>
<td>{{ run.operation|default:"-" }}</td>
<td>{{ run.model|default:"-" }}</td>
<td>{{ run.message_count }}</td>
<td>{{ run.prompt_chars }}</td>
<td>{{ run.response_chars }}</td>
<td>{% if run.duration_ms %}{{ run.duration_ms }}ms{% else %}-{% endif %}</td>
<td style="max-width: 26rem; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;" title="{{ run.error }}">{{ run.error|default:"-" }}</td>
</tr>
{% empty %}
<tr><td colspan="9">No runs yet.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</article>
</div>
</section>
{% endblock %}

View File

@@ -8,59 +8,84 @@
<article class="box"> <article class="box">
<h2 class="title is-6">Create Command Profile</h2> <h2 class="title is-6">Create Command Profile</h2>
<form method="post"> <p class="help">Create reusable command behavior. Example: <code>#bp#</code> reply command for business-plan extraction.</p>
<form method="post" aria-label="Create command profile">
{% csrf_token %} {% csrf_token %}
<input type="hidden" name="action" value="profile_create"> <input type="hidden" name="action" value="profile_create">
<div class="columns"> <div class="columns">
<div class="column"> <div class="column">
<input class="input is-small" name="slug" placeholder="slug (bp)" value="bp"> <label class="label is-size-7" for="create_slug">Slug</label>
<input id="create_slug" class="input is-small" name="slug" placeholder="slug (bp)" value="bp" aria-describedby="create_slug_help">
<p id="create_slug_help" class="help">Stable command id, e.g. <code>bp</code>.</p>
</div> </div>
<div class="column"> <div class="column">
<input class="input is-small" name="name" placeholder="name" value="Business Plan"> <label class="label is-size-7" for="create_name">Name</label>
<input id="create_name" class="input is-small" name="name" placeholder="name" value="Business Plan">
</div> </div>
<div class="column"> <div class="column">
<input class="input is-small" name="trigger_token" placeholder="trigger token" value="#bp#"> <label class="label is-size-7" for="create_trigger_token">Trigger Token</label>
<input id="create_trigger_token" class="input is-small" name="trigger_token" placeholder="trigger token" value="#bp#">
</div> </div>
</div> </div>
<textarea class="textarea is-small" name="template_text" rows="4" placeholder="Business plan template"></textarea> <label class="label is-size-7" for="create_template_text">Template Text</label>
<button class="button is-link is-small" type="submit">Create Profile</button> <textarea id="create_template_text" class="textarea is-small" name="template_text" rows="4" placeholder="Business plan template"></textarea>
<button class="button is-link is-small" style="margin-top: 0.75rem;" type="submit">Create Profile</button>
</form> </form>
</article> </article>
{% for profile in profiles %} {% for profile in profiles %}
<article class="box"> <article class="box">
<h2 class="title is-6">{{ profile.name }} ({{ profile.slug }})</h2> <h2 class="title is-6">{{ profile.name }} ({{ profile.slug }})</h2>
<form method="post" style="margin-bottom: 0.75rem;"> <div class="content is-size-7" style="margin-bottom: 0.6rem;">
<p><strong>Flag Definitions</strong></p>
<ul>
<li><strong>enabled</strong>: master on/off switch for this command profile.</li>
<li><strong>reply required</strong>: command only runs when the trigger message is sent as a reply to another message.</li>
<li><strong>exact match</strong>: message text must be exactly the trigger token (for example <code>#bp#</code>) with no extra text.</li>
<li><strong>visibility = status_in_source</strong>: post command status updates back into the source channel.</li>
<li><strong>visibility = silent</strong>: do not post status updates in the source channel.</li>
<li><strong>binding direction ingress</strong>: channels where trigger messages are accepted.</li>
<li><strong>binding direction egress</strong>: channels where command outputs are posted.</li>
<li><strong>binding direction scratchpad_mirror</strong>: scratchpad/mirror channel used for relay-only behavior.</li>
<li><strong>action extract_bp</strong>: run AI extraction to produce business plan content.</li>
<li><strong>action save_document</strong>: save/editable document and revision history.</li>
<li><strong>action post_result</strong>: fan out generated result to enabled egress bindings.</li>
<li><strong>position</strong>: execution order (lower runs first).</li>
</ul>
</div>
<form method="post" style="margin-bottom: 0.75rem;" aria-label="Update command profile {{ profile.name }}">
{% csrf_token %} {% csrf_token %}
<input type="hidden" name="action" value="profile_update"> <input type="hidden" name="action" value="profile_update">
<input type="hidden" name="profile_id" value="{{ profile.id }}"> <input type="hidden" name="profile_id" value="{{ profile.id }}">
<div class="columns is-multiline"> <div class="columns is-multiline">
<div class="column is-3"> <div class="column is-3">
<label class="label is-size-7">Name</label> <label class="label is-size-7" for="profile_name_{{ profile.id }}">Name</label>
<input class="input is-small" name="name" value="{{ profile.name }}"> <input id="profile_name_{{ profile.id }}" class="input is-small" name="name" value="{{ profile.name }}">
</div> </div>
<div class="column is-2"> <div class="column is-2">
<label class="label is-size-7">Trigger</label> <label class="label is-size-7" for="trigger_token_{{ profile.id }}">Trigger</label>
<input class="input is-small" name="trigger_token" value="{{ profile.trigger_token }}"> <input id="trigger_token_{{ profile.id }}" class="input is-small" name="trigger_token" value="{{ profile.trigger_token }}">
</div> </div>
<div class="column is-2"> <div class="column is-2">
<label class="label is-size-7">Visibility</label> <label class="label is-size-7" for="visibility_mode_{{ profile.id }}">Visibility</label>
<div class="select is-small is-fullwidth"> <div class="select is-small is-fullwidth">
<select name="visibility_mode"> <select id="visibility_mode_{{ profile.id }}" name="visibility_mode">
<option value="status_in_source" {% if profile.visibility_mode == 'status_in_source' %}selected{% endif %}>status_in_source</option> <option value="status_in_source" {% if profile.visibility_mode == 'status_in_source' %}selected{% endif %}>Show Status In Source Chat</option>
<option value="silent" {% if profile.visibility_mode == 'silent' %}selected{% endif %}>silent</option> <option value="silent" {% if profile.visibility_mode == 'silent' %}selected{% endif %}>Silent (No Status Message)</option>
</select> </select>
</div> </div>
</div> </div>
<div class="column is-5"> <div class="column is-5">
<label class="label is-size-7">Flags</label> <fieldset>
<legend class="label is-size-7">Flags</legend>
<label class="checkbox is-size-7"><input type="checkbox" name="enabled" value="1" {% if profile.enabled %}checked{% endif %}> enabled</label> <label class="checkbox is-size-7"><input type="checkbox" name="enabled" value="1" {% if profile.enabled %}checked{% endif %}> enabled</label>
<label class="checkbox is-size-7" style="margin-left: 0.6rem;"><input type="checkbox" name="reply_required" value="1" {% if profile.reply_required %}checked{% endif %}> reply required</label> <label class="checkbox is-size-7" style="margin-left: 0.6rem;"><input type="checkbox" name="reply_required" value="1" {% if profile.reply_required %}checked{% endif %}> reply required</label>
<label class="checkbox is-size-7" style="margin-left: 0.6rem;"><input type="checkbox" name="exact_match_only" value="1" {% if profile.exact_match_only %}checked{% endif %}> exact match</label> <label class="checkbox is-size-7" style="margin-left: 0.6rem;"><input type="checkbox" name="exact_match_only" value="1" {% if profile.exact_match_only %}checked{% endif %}> exact match</label>
</fieldset>
</div> </div>
</div> </div>
<label class="label is-size-7">BP Template</label> <label class="label is-size-7" for="template_text_{{ profile.id }}">BP Template</label>
<textarea class="textarea is-small" name="template_text" rows="5">{{ profile.template_text }}</textarea> <textarea id="template_text_{{ profile.id }}" class="textarea is-small" name="template_text" rows="5">{{ profile.template_text }}</textarea>
<div class="buttons" style="margin-top: 0.6rem;"> <div class="buttons" style="margin-top: 0.6rem;">
<button class="button is-link is-small" type="submit">Save Profile</button> <button class="button is-link is-small" type="submit">Save Profile</button>
</div> </div>
@@ -69,18 +94,24 @@
<div class="columns"> <div class="columns">
<div class="column"> <div class="column">
<h3 class="title is-7">Channel Bindings</h3> <h3 class="title is-7">Channel Bindings</h3>
<p class="help">A command runs only when the source channel is in <code>ingress</code>. Output is sent to all enabled <code>egress</code> bindings.</p>
<table class="table is-fullwidth is-striped is-size-7"> <table class="table is-fullwidth is-striped is-size-7">
<thead> <thead>
<tr><th>Direction</th><th>Service</th><th>Channel</th><th></th></tr> <tr><th scope="col">Direction</th><th scope="col">Service</th><th scope="col">Channel</th><th scope="col">Actions</th></tr>
</thead> </thead>
<tbody> <tbody>
{% for binding in profile.channel_bindings.all %} {% for binding in profile.channel_bindings.all %}
<tr> <tr>
<td>{{ binding.direction }}</td> <td>
{% if binding.direction == "ingress" %}Ingress (Accept Triggers)
{% elif binding.direction == "egress" %}Egress (Post Results)
{% else %}Scratchpad Mirror
{% endif %}
</td>
<td>{{ binding.service }}</td> <td>{{ binding.service }}</td>
<td>{{ binding.channel_identifier }}</td> <td>{{ binding.channel_identifier }}</td>
<td> <td>
<form method="post"> <form method="post" aria-label="Delete binding {{ binding.direction }} {{ binding.service }} {{ binding.channel_identifier }}">
{% csrf_token %} {% csrf_token %}
<input type="hidden" name="action" value="binding_delete"> <input type="hidden" name="action" value="binding_delete">
<input type="hidden" name="binding_id" value="{{ binding.id }}"> <input type="hidden" name="binding_id" value="{{ binding.id }}">
@@ -93,23 +124,30 @@
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
<form method="post"> <form method="post" aria-label="Add channel binding for {{ profile.name }}">
{% csrf_token %} {% csrf_token %}
<input type="hidden" name="action" value="binding_create"> <input type="hidden" name="action" value="binding_create">
<input type="hidden" name="profile_id" value="{{ profile.id }}"> <input type="hidden" name="profile_id" value="{{ profile.id }}">
<div class="columns"> <div class="columns">
<div class="column"> <div class="column">
<label class="label is-size-7" for="binding_direction_{{ profile.id }}">Direction</label>
<div class="select is-small is-fullwidth"> <div class="select is-small is-fullwidth">
<select name="direction"> <select id="binding_direction_{{ profile.id }}" name="direction">
{% for value in directions %} {% for value in directions %}
<option value="{{ value }}">{{ value }}</option> <option value="{{ value }}">
{% if value == "ingress" %}Ingress (Accept Triggers)
{% elif value == "egress" %}Egress (Post Results)
{% else %}Scratchpad Mirror
{% endif %}
</option>
{% endfor %} {% endfor %}
</select> </select>
</div> </div>
</div> </div>
<div class="column"> <div class="column">
<label class="label is-size-7" for="binding_service_{{ profile.id }}">Service</label>
<div class="select is-small is-fullwidth"> <div class="select is-small is-fullwidth">
<select name="service"> <select id="binding_service_{{ profile.id }}" name="service">
{% for value in channel_services %} {% for value in channel_services %}
<option value="{{ value }}">{{ value }}</option> <option value="{{ value }}">{{ value }}</option>
{% endfor %} {% endfor %}
@@ -117,7 +155,8 @@
</div> </div>
</div> </div>
<div class="column"> <div class="column">
<input class="input is-small" name="channel_identifier" placeholder="channel identifier"> <label class="label is-size-7" for="binding_channel_identifier_{{ profile.id }}">Channel Identifier</label>
<input id="binding_channel_identifier_{{ profile.id }}" class="input is-small" name="channel_identifier" placeholder="channel identifier">
</div> </div>
<div class="column is-narrow"> <div class="column is-narrow">
<button class="button is-link is-small" type="submit">Add</button> <button class="button is-link is-small" type="submit">Add</button>
@@ -128,23 +167,45 @@
<div class="column"> <div class="column">
<h3 class="title is-7">Actions</h3> <h3 class="title is-7">Actions</h3>
<p class="help">Enable/disable each step and set execution order with <code>position</code>.</p>
<table class="table is-fullwidth is-striped is-size-7"> <table class="table is-fullwidth is-striped is-size-7">
<thead> <thead>
<tr><th>Type</th><th>Enabled</th><th>Position</th><th></th></tr> <tr><th scope="col">Type</th><th scope="col">Enabled</th><th scope="col">Order</th><th scope="col">Actions</th></tr>
</thead> </thead>
<tbody> <tbody>
{% for action_row in profile.actions.all %} {% for action_row in profile.actions.all %}
<tr> <tr>
<td>{{ action_row.action_type }}</td>
<td>{{ action_row.enabled }}</td>
<td>{{ action_row.position }}</td>
<td> <td>
<form method="post"> {% if action_row.action_type == "extract_bp" %}Extract Business Plan
{% elif action_row.action_type == "save_document" %}Save Document
{% elif action_row.action_type == "post_result" %}Post Result
{% else %}{{ action_row.action_type }}
{% endif %}
</td>
<td>{{ action_row.enabled }}</td>
<td>{{ forloop.counter }}</td>
<td>
<div class="buttons are-small" style="margin-bottom: 0.35rem;">
<form method="post" style="display:inline;" aria-label="Move action {{ action_row.action_type }} up for {{ profile.name }}">
{% csrf_token %}
<input type="hidden" name="action" value="action_move">
<input type="hidden" name="command_action_id" value="{{ action_row.id }}">
<input type="hidden" name="direction" value="up">
<button class="button is-light" type="submit" {% if forloop.first %}disabled{% endif %} aria-label="Move up">Up</button>
</form>
<form method="post" style="display:inline;" aria-label="Move action {{ action_row.action_type }} down for {{ profile.name }}">
{% csrf_token %}
<input type="hidden" name="action" value="action_move">
<input type="hidden" name="command_action_id" value="{{ action_row.id }}">
<input type="hidden" name="direction" value="down">
<button class="button is-light" type="submit" {% if forloop.last %}disabled{% endif %} aria-label="Move down">Down</button>
</form>
</div>
<form method="post" aria-label="Update action {{ action_row.action_type }} for {{ profile.name }}">
{% csrf_token %} {% csrf_token %}
<input type="hidden" name="action" value="action_update"> <input type="hidden" name="action" value="action_update">
<input type="hidden" name="command_action_id" value="{{ action_row.id }}"> <input type="hidden" name="command_action_id" value="{{ action_row.id }}">
<label class="checkbox is-size-7"><input type="checkbox" name="enabled" value="1" {% if action_row.enabled %}checked{% endif %}> enabled</label> <label class="checkbox is-size-7"><input type="checkbox" name="enabled" value="1" {% if action_row.enabled %}checked{% endif %}> enabled</label>
<input class="input is-small" style="width: 5rem;" name="position" value="{{ action_row.position }}">
<button class="button is-link is-light is-small" type="submit">Save</button> <button class="button is-link is-light is-small" type="submit">Save</button>
</form> </form>
</td> </td>
@@ -157,11 +218,11 @@
</div> </div>
</div> </div>
<form method="post" style="margin-top: 0.75rem;"> <form method="post" style="margin-top: 0.75rem;" aria-label="Delete profile {{ profile.name }}">
{% csrf_token %} {% csrf_token %}
<input type="hidden" name="action" value="profile_delete"> <input type="hidden" name="action" value="profile_delete">
<input type="hidden" name="profile_id" value="{{ profile.id }}"> <input type="hidden" name="profile_id" value="{{ profile.id }}">
<button class="button is-danger is-light is-small" type="submit">Delete Profile</button> <button class="button is-danger is-light is-small" type="submit" aria-label="Delete profile {{ profile.name }}">Delete Profile</button>
</form> </form>
</article> </article>
{% empty %} {% empty %}
@@ -172,7 +233,7 @@
<h2 class="title is-6">Business Plan Documents</h2> <h2 class="title is-6">Business Plan Documents</h2>
<table class="table is-fullwidth is-striped is-size-7"> <table class="table is-fullwidth is-striped is-size-7">
<thead> <thead>
<tr><th>Title</th><th>Status</th><th>Source</th><th>Updated</th><th></th></tr> <tr><th scope="col">Title</th><th scope="col">Status</th><th scope="col">Source</th><th scope="col">Updated</th><th scope="col">Actions</th></tr>
</thead> </thead>
<tbody> <tbody>
{% for doc in documents %} {% for doc in documents %}
@@ -181,7 +242,7 @@
<td>{{ doc.status }}</td> <td>{{ doc.status }}</td>
<td>{{ doc.source_service }} · {{ doc.source_channel_identifier }}</td> <td>{{ doc.source_service }} · {{ doc.source_channel_identifier }}</td>
<td>{{ doc.updated_at }}</td> <td>{{ doc.updated_at }}</td>
<td><a class="button is-small is-link is-light" href="{% url 'business_plan_editor' doc_id=doc.id %}">Open</a></td> <td><a class="button is-small is-link is-light" href="{% url 'business_plan_editor' doc_id=doc.id %}" aria-label="Open business plan document {{ doc.title }}">Open</a></td>
</tr> </tr>
{% empty %} {% empty %}
<tr><td colspan="5">No business plan documents yet.</td></tr> <tr><td colspan="5">No business plan documents yet.</td></tr>
@@ -190,78 +251,6 @@
</table> </table>
</article> </article>
<article class="box">
<h2 class="title is-6">Translation Bridges</h2>
<form method="post" style="margin-bottom: 0.75rem;">
{% csrf_token %}
<input type="hidden" name="action" value="bridge_create">
<div class="columns is-multiline">
<div class="column is-2"><input class="input is-small" name="name" placeholder="name"></div>
<div class="column is-2"><input class="input is-small" name="quick_mode_title" placeholder="quick mode: en|es"></div>
<div class="column is-2">
<div class="select is-small is-fullwidth"><select name="a_service">{% for value in channel_services %}<option value="{{ value }}">{{ value }}</option>{% endfor %}</select></div>
</div>
<div class="column is-2"><input class="input is-small" name="a_channel_identifier" placeholder="A channel"></div>
<div class="column is-1"><input class="input is-small" name="a_language" value="en"></div>
<div class="column is-2">
<div class="select is-small is-fullwidth"><select name="b_service">{% for value in channel_services %}<option value="{{ value }}">{{ value }}</option>{% endfor %}</select></div>
</div>
<div class="column is-2"><input class="input is-small" name="b_channel_identifier" placeholder="B channel"></div>
<div class="column is-1"><input class="input is-small" name="b_language" value="es"></div>
<div class="column is-2">
<div class="select is-small is-fullwidth"><select name="direction">{% for value in bridge_directions %}<option value="{{ value }}">{{ value }}</option>{% endfor %}</select></div>
</div>
<div class="column is-1"><button class="button is-link is-small" type="submit">Add</button></div>
</div>
</form>
<table class="table is-fullwidth is-striped is-size-7">
<thead>
<tr><th>Name</th><th>A</th><th>B</th><th>Direction</th><th></th></tr>
</thead>
<tbody>
{% for bridge in bridges %}
<tr>
<td>{{ bridge.name }}</td>
<td>{{ bridge.a_service }} · {{ bridge.a_channel_identifier }} · {{ bridge.a_language }}</td>
<td>{{ bridge.b_service }} · {{ bridge.b_channel_identifier }} · {{ bridge.b_language }}</td>
<td>{{ bridge.direction }}</td>
<td>
<form method="post">
{% csrf_token %}
<input type="hidden" name="action" value="bridge_delete">
<input type="hidden" name="bridge_id" value="{{ bridge.id }}">
<button class="button is-danger is-light is-small" type="submit">Delete</button>
</form>
</td>
</tr>
{% empty %}
<tr><td colspan="5">No translation bridges configured.</td></tr>
{% endfor %}
</tbody>
</table>
</article>
<article class="box">
<h2 class="title is-6">Translation Event Log</h2>
<table class="table is-fullwidth is-striped is-size-7">
<thead>
<tr><th>Bridge</th><th>Status</th><th>Target</th><th>Error</th><th>At</th></tr>
</thead>
<tbody>
{% for event in events %}
<tr>
<td>{{ event.bridge.name }}</td>
<td>{{ event.status }}</td>
<td>{{ event.target_service }} · {{ event.target_channel }}</td>
<td>{{ event.error|default:"-" }}</td>
<td>{{ event.created_at }}</td>
</tr>
{% empty %}
<tr><td colspan="5">No events yet.</td></tr>
{% endfor %}
</tbody>
</table>
</article>
</div> </div>
</section> </section>
{% endblock %} {% endblock %}

View File

@@ -1,17 +1,20 @@
{% extends "base.html" %} {% extends "base.html" %}
{% block content %} {% block content %}
<div class="columns is-centered"> <section class="section">
<div class="column is-11-tablet is-10-desktop is-9-widescreen"> <div class="container is-max-desktop">
<div class="is-flex is-justify-content-space-between is-align-items-center"> <div class="mb-4">
<div> <h1 class="title is-4 mb-2">Search</h1>
<h1 class="title is-4">Search</h1> <p class="subtitle is-6 mb-3">
<p class="subtitle is-6"> Unified lookup across contacts, identifiers, and messages, with advanced filters for source, date range, sentiment, sort, dedup, and reverse.
Search across OSINT objects with sortable, paginated results.
</p> </p>
<div class="tags">
<span class="tag is-light">Default Scope: All</span>
<span class="tag is-light">Contacts + Messages</span>
<span class="tag is-light">SIQTSRSS/ADR Controls</span>
</div> </div>
</div> </div>
{% include "partials/osint/search-panel.html" %} {% include "partials/osint/search-panel.html" %}
</div> </div>
</div> </section>
{% endblock %} {% endblock %}

View File

@@ -0,0 +1,93 @@
{% extends "base.html" %}
{% block content %}
<section class="section">
<div class="container">
<h1 class="title is-4">Translation Settings</h1>
<p class="subtitle is-6">Configure translation bridges, routing direction, and inspect sync events.</p>
<article class="box">
<h2 class="title is-6">Translation Bridges</h2>
<form method="post" style="margin-bottom: 0.75rem;" aria-label="Create translation bridge">
{% csrf_token %}
<input type="hidden" name="action" value="bridge_create">
<div class="columns is-multiline">
<div class="column is-2"><label class="label is-size-7" for="bridge_name">Name</label><input id="bridge_name" class="input is-small" name="name" placeholder="name"></div>
<div class="column is-2"><label class="label is-size-7" for="bridge_quick_title">Quick Mode Hint</label><input id="bridge_quick_title" class="input is-small" name="quick_mode_title" placeholder="quick mode: en|es"></div>
<div class="column is-2">
<label class="label is-size-7" for="bridge_a_service">A Service</label>
<div class="select is-small is-fullwidth"><select id="bridge_a_service" name="a_service">{% for value in channel_services %}<option value="{{ value }}">{{ value }}</option>{% endfor %}</select></div>
</div>
<div class="column is-2"><label class="label is-size-7" for="bridge_a_channel">A Channel</label><input id="bridge_a_channel" class="input is-small" name="a_channel_identifier" placeholder="A channel"></div>
<div class="column is-1"><label class="label is-size-7" for="bridge_a_language">A Lang</label><input id="bridge_a_language" class="input is-small" name="a_language" value="en"></div>
<div class="column is-2">
<label class="label is-size-7" for="bridge_b_service">B Service</label>
<div class="select is-small is-fullwidth"><select id="bridge_b_service" name="b_service">{% for value in channel_services %}<option value="{{ value }}">{{ value }}</option>{% endfor %}</select></div>
</div>
<div class="column is-2"><label class="label is-size-7" for="bridge_b_channel">B Channel</label><input id="bridge_b_channel" class="input is-small" name="b_channel_identifier" placeholder="B channel"></div>
<div class="column is-1"><label class="label is-size-7" for="bridge_b_language">B Lang</label><input id="bridge_b_language" class="input is-small" name="b_language" value="es"></div>
<div class="column is-2">
<label class="label is-size-7" for="bridge_direction">Direction</label>
<div class="select is-small is-fullwidth"><select id="bridge_direction" name="direction">{% for value in bridge_directions %}<option value="{{ value }}">{{ value }}</option>{% endfor %}</select></div>
</div>
<div class="column is-1"><button class="button is-link is-small" type="submit">Add</button></div>
</div>
</form>
<table class="table is-fullwidth is-striped is-size-7">
<caption>Configured translation bridges</caption>
<thead>
<tr><th scope="col">Name</th><th scope="col">A</th><th scope="col">B</th><th scope="col">Direction</th><th scope="col">Actions</th></tr>
</thead>
<tbody>
{% for bridge in bridges %}
<tr>
<td>{{ bridge.name }}</td>
<td>{{ bridge.a_service }} · {{ bridge.a_channel_identifier }} · {{ bridge.a_language }}</td>
<td>{{ bridge.b_service }} · {{ bridge.b_channel_identifier }} · {{ bridge.b_language }}</td>
<td>
{% if bridge.direction == "a_to_b" %}A to B
{% elif bridge.direction == "b_to_a" %}B to A
{% else %}Bidirectional
{% endif %}
</td>
<td>
<form method="post" aria-label="Delete translation bridge {{ bridge.name }}">
{% csrf_token %}
<input type="hidden" name="action" value="bridge_delete">
<input type="hidden" name="bridge_id" value="{{ bridge.id }}">
<button class="button is-danger is-light is-small" type="submit">Delete</button>
</form>
</td>
</tr>
{% empty %}
<tr><td colspan="5">No translation bridges configured.</td></tr>
{% endfor %}
</tbody>
</table>
</article>
<article class="box">
<h2 class="title is-6">Translation Event Log</h2>
<table class="table is-fullwidth is-striped is-size-7">
<caption>Recent translation sync events</caption>
<thead>
<tr><th scope="col">Bridge</th><th scope="col">Status</th><th scope="col">Target</th><th scope="col">Error</th><th scope="col">At</th></tr>
</thead>
<tbody>
{% for event in events %}
<tr>
<td>{{ event.bridge.name }}</td>
<td>{{ event.status }}</td>
<td>{{ event.target_service }} · {{ event.target_channel }}</td>
<td>{{ event.error|default:"-" }}</td>
<td>{{ event.created_at }}</td>
</tr>
{% empty %}
<tr><td colspan="5">No events yet.</td></tr>
{% endfor %}
</tbody>
</table>
</article>
</div>
</section>
{% endblock %}

View File

@@ -1,11 +1,13 @@
{% include 'mixins/partials/notify.html' %} {% include 'mixins/partials/notify.html' %}
<div <div
id="{{ osint_table_id }}" id="{{ osint_table_id }}"
class="osint-table-shell" class="osint-table-shell {% if osint_shell_borderless %}osint-table-shell--borderless{% endif %}"
{% if osint_event_name %}
hx-get="{{ osint_refresh_url }}" hx-get="{{ osint_refresh_url }}"
hx-target="#{{ osint_table_id }}" hx-target="#{{ osint_table_id }}"
hx-swap="outerHTML" hx-swap="outerHTML"
{% if osint_event_name %}hx-trigger="{{ osint_event_name }} from:body"{% endif %}> hx-trigger="{{ osint_event_name }} from:body"
{% endif %}>
{% if osint_show_search %} {% if osint_show_search %}
<form <form
@@ -53,9 +55,15 @@
<div class="osint-results-meta"> <div class="osint-results-meta">
<div class="osint-results-meta-left"> <div class="osint-results-meta-left">
<div class="dropdown is-hoverable" id="{{ osint_table_id }}-columns-dropdown"> <div class="dropdown" id="{{ osint_table_id }}-columns-dropdown" hx-disable>
<div class="dropdown-trigger"> <div class="dropdown-trigger">
<button class="button is-small is-light" aria-haspopup="true" aria-controls="{{ osint_table_id }}-columns-menu"> <button
class="button is-small is-light"
type="button"
data-role="columns-toggle-trigger"
aria-haspopup="true"
aria-expanded="false"
aria-controls="{{ osint_table_id }}-columns-menu">
<span>Show/Hide Fields</span> <span>Show/Hide Fields</span>
<span class="icon is-small"><i class="fa-solid fa-angle-down"></i></span> <span class="icon is-small"><i class="fa-solid fa-angle-down"></i></span>
</button> </button>
@@ -63,15 +71,15 @@
<div class="dropdown-menu" id="{{ osint_table_id }}-columns-menu" role="menu"> <div class="dropdown-menu" id="{{ osint_table_id }}-columns-menu" role="menu">
<div class="dropdown-content"> <div class="dropdown-content">
{% for column in osint_columns %} {% for column in osint_columns %}
<a <button
type="button"
class="dropdown-item osint-col-toggle" class="dropdown-item osint-col-toggle"
data-col-index="{{ forloop.counter0 }}" data-col-index="{{ forloop.counter0 }}"
href="#" aria-pressed="false">
onclick="return false;">
<span class="icon is-small"><i class="fa-solid fa-check"></i></span> <span class="icon is-small"><i class="fa-solid fa-check"></i></span>
<span>{{ column.label }}</span> <span>{{ column.label }}</span>
<span class="is-size-7 has-text-grey ml-2">({{ column.field_name }})</span> <span class="is-size-7 has-text-grey ml-2">({{ column.field_name }})</span>
</a> </button>
{% endfor %} {% endfor %}
</div> </div>
</div> </div>
@@ -126,11 +134,33 @@
{% if cell.kind == "id_copy" %} {% if cell.kind == "id_copy" %}
<a <a
class="button is-small has-text-grey" class="button is-small has-text-grey"
title="Copy value"
onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ cell.value }}');"> onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ cell.value }}');">
<span class="icon"> <span class="icon">
<i class="fa-solid fa-copy"></i> <i class="fa-solid fa-copy"></i>
</span> </span>
<span>{{ cell.value }}</span> </a>
{% elif cell.kind == "chat_ref" %}
{% if cell.value.copy %}
<a
class="button is-small has-text-grey"
title="Copy chat id"
onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ cell.value.copy }}');">
<span class="icon">
<i class="fa-solid fa-copy"></i>
</span>
</a>
{% else %}
<span class="has-text-grey">-</span>
{% endif %}
{% elif cell.value.copy %}
<a
class="button is-small has-text-grey"
title="Copy value"
onclick="window.prompt('Copy to clipboard: Ctrl+C, Enter', '{{ cell.value.copy }}');">
<span class="icon">
<i class="fa-solid fa-copy"></i>
</span>
</a> </a>
{% elif cell.kind == "bool" %} {% elif cell.kind == "bool" %}
{% if cell.value %} {% if cell.value %}
@@ -281,6 +311,22 @@
#{{ osint_table_id }} .osint-col-toggle.is-hidden-col .icon { #{{ osint_table_id }} .osint-col-toggle.is-hidden-col .icon {
color: #b5b5b5; color: #b5b5b5;
} }
#{{ osint_table_id }} .osint-col-toggle {
width: 100%;
text-align: left;
border: 0;
background: transparent;
cursor: pointer;
display: flex;
align-items: center;
gap: 0.35rem;
}
#{{ osint_table_id }}.osint-table-shell--borderless {
border: 0;
border-radius: 0;
padding: 0;
background: transparent;
}
</style> </style>
<script> <script>
@@ -292,6 +338,13 @@
if (!shell) { if (!shell) {
return; return;
} }
const dropdown = document.getElementById(tableId + "-columns-dropdown");
const trigger = dropdown
? dropdown.querySelector("[data-role='columns-toggle-trigger']")
: null;
const dropdownMenu = dropdown
? dropdown.querySelector(".dropdown-menu")
: null;
const storageKey = [ const storageKey = [
"gia_osint_hidden_cols_v2", "gia_osint_hidden_cols_v2",
tableId, tableId,
@@ -319,6 +372,7 @@
const idx = String(toggle.getAttribute("data-col-index") || ""); const idx = String(toggle.getAttribute("data-col-index") || "");
const isHidden = hiddenSet.has(idx); const isHidden = hiddenSet.has(idx);
toggle.classList.toggle("is-hidden-col", isHidden); toggle.classList.toggle("is-hidden-col", isHidden);
toggle.setAttribute("aria-pressed", isHidden ? "true" : "false");
const icon = toggle.querySelector("i"); const icon = toggle.querySelector("i");
if (icon) { if (icon) {
icon.className = isHidden ? "fa-solid fa-xmark" : "fa-solid fa-check"; icon.className = isHidden ? "fa-solid fa-xmark" : "fa-solid fa-check";
@@ -335,7 +389,9 @@
}; };
shell.querySelectorAll(".osint-col-toggle").forEach(function (toggle) { shell.querySelectorAll(".osint-col-toggle").forEach(function (toggle) {
toggle.addEventListener("click", function () { toggle.addEventListener("click", function (event) {
event.preventDefault();
event.stopPropagation();
const idx = String(toggle.getAttribute("data-col-index") || ""); const idx = String(toggle.getAttribute("data-col-index") || "");
if (!idx) { if (!idx) {
return; return;
@@ -347,9 +403,51 @@
} }
persist(); persist();
applyVisibility(); applyVisibility();
if (dropdown) {
dropdown.classList.remove("is-active");
}
if (trigger) {
trigger.setAttribute("aria-expanded", "false");
}
}); });
}); });
if (trigger && dropdown) {
trigger.addEventListener("click", function (event) {
event.preventDefault();
event.stopPropagation();
const nextActive = !dropdown.classList.contains("is-active");
dropdown.classList.toggle("is-active", nextActive);
trigger.setAttribute("aria-expanded", nextActive ? "true" : "false");
});
}
if (dropdownMenu) {
dropdownMenu.addEventListener("click", function (event) {
event.stopPropagation();
});
}
document.addEventListener("click", function (event) {
if (!dropdown) {
return;
}
if (dropdown.contains(event.target)) {
return;
}
dropdown.classList.remove("is-active");
if (trigger) {
trigger.setAttribute("aria-expanded", "false");
}
});
document.addEventListener("keydown", function (event) {
if (event.key !== "Escape" || !dropdown) {
return;
}
dropdown.classList.remove("is-active");
if (trigger) {
trigger.setAttribute("aria-expanded", "false");
}
});
applyVisibility(); applyVisibility();
})(); })();
</script> </script>

View File

@@ -1,15 +1,15 @@
<div class="box"> <form
<form class="osint-search-form box"
class="osint-search-form"
method="get" method="get"
action="{{ osint_search_url }}" action="{{ osint_search_url }}"
hx-get="{{ osint_search_url }}" hx-get="{{ osint_search_url }}"
hx-trigger="change delay:120ms"
hx-target="#osint-search-results" hx-target="#osint-search-results"
hx-swap="innerHTML"> hx-swap="innerHTML">
<div class="columns is-multiline"> <div class="columns is-multiline">
<div class="column is-4"> <div class="column is-4">
<label class="label">Scope</label> <label class="label">Scope</label>
<div class="select is-fullwidth"> <div class="select">
<select name="scope"> <select name="scope">
{% for option in scope_options %} {% for option in scope_options %}
<option <option
@@ -23,7 +23,7 @@
</div> </div>
<div class="column is-4"> <div class="column is-4">
<label class="label">Field</label> <label class="label">Field</label>
<div class="select is-fullwidth"> <div class="select">
<select name="field"> <select name="field">
<option value="__all__" {% if selected_field == "__all__" %}selected{% endif %}> <option value="__all__" {% if selected_field == "__all__" %}selected{% endif %}>
All Fields All Fields
@@ -40,7 +40,7 @@
</div> </div>
<div class="column is-4"> <div class="column is-4">
<label class="label">Rows Per Page</label> <label class="label">Rows Per Page</label>
<div class="select is-fullwidth"> <div class="select">
<select name="per_page"> <select name="per_page">
<option value="10" {% if selected_per_page == 10 %}selected{% endif %}>10</option> <option value="10" {% if selected_per_page == 10 %}selected{% endif %}>10</option>
<option value="20" {% if selected_per_page == 20 %}selected{% endif %}>20</option> <option value="20" {% if selected_per_page == 20 %}selected{% endif %}>20</option>
@@ -55,8 +55,13 @@
class="input" class="input"
type="text" type="text"
name="q" name="q"
hx-get="{{ osint_search_url }}"
hx-trigger="keyup changed delay:220ms"
hx-include="closest form"
hx-target="#osint-search-results"
hx-swap="innerHTML"
value="{{ search_query }}" value="{{ search_query }}"
placeholder="Search text, values, or relation names..."> placeholder="Search contacts, identifiers, and messages...">
</div> </div>
<div class="column is-3"> <div class="column is-3">
<label class="label">&nbsp;</label> <label class="label">&nbsp;</label>
@@ -64,15 +69,108 @@
<button class="button is-link is-light is-fullwidth" type="submit"> <button class="button is-link is-light is-fullwidth" type="submit">
Search Search
</button> </button>
<a class="button is-light is-fullwidth" href="{{ osint_search_url }}"> <a class="button is-light is-fullwidth" href="{{ osint_search_url }}" style="margin-bottom: 0.55rem;">
Reset Reset
</a> </a>
</div> </div>
</div> </div>
</div> </div>
</form> <details style="margin-top: 0.45rem;">
<summary class="has-text-weight-semibold">Advanced Filters (SIQTSRSS/ADR)</summary>
<div id="osint-search-results"> <p class="help" style="margin: 0.1rem 0 0.2rem 0;">Tip: use inline tags like <code>tag:messages</code> or <code>tag:people</code> in the query to narrow All-scope results quickly.</p>
{% include "partials/results_table.html" %} <p class="help" style="margin: 0 0 0.35rem 0;"><strong>Source Service</strong>, <strong>From Date</strong>, <strong>To Date</strong>, and <strong>Sort Mode</strong> shape which results you see and in what order.</p>
<div style="margin-top: 0.2rem;">
<div class="columns is-multiline" style="margin-top: 0.3rem;">
<div class="column is-12-mobile is-6-tablet is-4-desktop">
<label class="label">Source Service</label>
<div class="select">
<select name="source">
<option value="all" {% if selected_source == "all" %}selected{% endif %}>All</option>
<option value="web" {% if selected_source == "web" %}selected{% endif %}>Web</option>
<option value="xmpp" {% if selected_source == "xmpp" %}selected{% endif %}>XMPP</option>
<option value="signal" {% if selected_source == "signal" %}selected{% endif %}>Signal</option>
<option value="whatsapp" {% if selected_source == "whatsapp" %}selected{% endif %}>WhatsApp</option>
</select>
</div> </div>
</div>
<div class="column is-12"></div>
<div class="column is-12-mobile is-6-tablet is-3-desktop">
<label class="label">From Date</label>
<input
class="input{% if selected_date_from %} date-has-value{% endif %}"
type="date"
name="date_from"
value="{{ selected_date_from }}"
aria-label="From date">
</div>
<div class="column is-12-mobile is-6-tablet is-3-desktop">
<label class="label">To Date</label>
<input
class="input{% if selected_date_to %} date-has-value{% endif %}"
type="date"
name="date_to"
value="{{ selected_date_to }}"
aria-label="To date">
</div>
<div class="column is-12-mobile is-6-tablet is-3-desktop">
<label class="label">Sort Mode</label>
<div class="select">
<select name="sort_mode">
<option value="relevance" {% if selected_sort_mode == "relevance" %}selected{% endif %}>Relevance</option>
<option value="recent" {% if selected_sort_mode == "recent" %}selected{% endif %}>Recent First</option>
<option value="oldest" {% if selected_sort_mode == "oldest" %}selected{% endif %}>Oldest First</option>
</select>
</div>
</div>
<div class="column is-12"></div>
<div class="column is-12-mobile is-6-tablet is-2-desktop">
<label class="label">Min Sent.</label>
<input class="input" type="number" step="0.01" min="-1" max="1" name="sentiment_min" value="{{ selected_sentiment_min }}">
</div>
<div class="column is-12-mobile is-6-tablet is-2-desktop">
<label class="label">Max Sent.</label>
<input class="input" type="number" step="0.01" min="-1" max="1" name="sentiment_max" value="{{ selected_sentiment_max }}">
</div>
<div class="column is-12"></div>
<div class="column is-12">
<label class="checkbox" style="margin-right: 0.9rem;">
<input type="checkbox" name="annotate" value="1" {% if selected_annotate %}checked{% endif %}>
Annotate snippets
</label>
<label class="checkbox" style="margin-right: 0.9rem;">
<input type="checkbox" name="dedup" value="1" {% if selected_dedup %}checked{% endif %}>
Deduplicate
</label>
<label class="checkbox">
<input type="checkbox" name="reverse" value="1" {% if selected_reverse %}checked{% endif %}>
Reverse output
</label>
</div>
</div>
<div class="content is-size-7" style="margin-top: 0.2rem;">
<ul>
<li><strong>Min/Max Sent.</strong>: sentiment bounds for people/contact results (-1 to 1).</li>
<li><strong>Annotate snippets</strong>: shows contextual snippets around query hits.</li>
<li><strong>Deduplicate</strong>: removes near-identical repeated rows.</li>
<li><strong>Reverse output</strong>: reverses final result order after sorting.</li>
</ul>
</div>
</div>
</details>
</form>
<div id="osint-search-results">
{% include "partials/results_table.html" %}
</div> </div>
<style>
.date-has-value::-webkit-calendar-picker-indicator {
filter: brightness(0) saturate(100%) invert(38%) sepia(85%) saturate(1820%) hue-rotate(201deg) brightness(96%) contrast(92%);
}
</style>

View File

@@ -0,0 +1,92 @@
from __future__ import annotations
from asgiref.sync import async_to_sync
from django.test import TestCase
from unittest.mock import patch
from core.messaging.ai import run_prompt
from core.models import AI, AIRunLog, User
class _FakeResponseMessage:
def __init__(self, content):
self.content = content
class _FakeChoice:
def __init__(self, content):
self.message = _FakeResponseMessage(content)
class _FakeResponse:
def __init__(self, content):
self.choices = [_FakeChoice(content)]
class _FakeCompletionsOK:
async def create(self, **kwargs):
return _FakeResponse("ok-output")
class _FakeChatOK:
def __init__(self):
self.completions = _FakeCompletionsOK()
class _FakeClientOK:
def __init__(self, **kwargs):
self.chat = _FakeChatOK()
class _FakeCompletionsFail:
async def create(self, **kwargs):
raise RuntimeError("provider boom")
class _FakeChatFail:
def __init__(self):
self.completions = _FakeCompletionsFail()
class _FakeClientFail:
def __init__(self, **kwargs):
self.chat = _FakeChatFail()
class AIRunLogTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(
username="ai-log-user",
email="ai-log@example.com",
password="x",
)
self.ai = AI.objects.create(
user=self.user,
base_url="https://example.invalid",
api_key="test-key",
model="gpt-4o-mini",
)
self.prompt = [{"role": "user", "content": "Hello world"}]
def test_run_prompt_logs_success(self):
with patch("core.messaging.ai.AsyncOpenAI", _FakeClientOK):
out = async_to_sync(run_prompt)(self.prompt, self.ai, operation="test_ok")
self.assertEqual("ok-output", out)
row = AIRunLog.objects.order_by("-id").first()
self.assertIsNotNone(row)
self.assertEqual("ok", row.status)
self.assertEqual("test_ok", row.operation)
self.assertEqual(1, row.message_count)
self.assertEqual(len("Hello world"), row.prompt_chars)
self.assertEqual(len("ok-output"), row.response_chars)
self.assertTrue((row.duration_ms or 0) >= 0)
def test_run_prompt_logs_failure(self):
with patch("core.messaging.ai.AsyncOpenAI", _FakeClientFail):
with self.assertRaises(RuntimeError):
async_to_sync(run_prompt)(self.prompt, self.ai, operation="test_fail")
row = AIRunLog.objects.order_by("-id").first()
self.assertIsNotNone(row)
self.assertEqual("failed", row.status)
self.assertEqual("test_fail", row.operation)
self.assertIn("provider boom", row.error)

View File

@@ -55,7 +55,10 @@ async def _translate_text(user, text: str, source_lang: str, target_lang: str) -
), ),
}, },
] ]
return str(await ai_runner.run_prompt(prompt, ai_obj) or "").strip() return str(
await ai_runner.run_prompt(prompt, ai_obj, operation="translation")
or ""
).strip()
async def process_inbound_translation(message: Message): async def process_inbound_translation(message: Message):

View File

@@ -1,11 +1,17 @@
from __future__ import annotations from __future__ import annotations
from datetime import timedelta
from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.auth.mixins import LoginRequiredMixin
from django.db import transaction
from django.db.models import Avg, Count, Q, Sum
from django.http import JsonResponse from django.http import JsonResponse
from django.shortcuts import get_object_or_404, redirect, render from django.shortcuts import get_object_or_404, redirect, render
from django.utils import timezone
from django.views import View from django.views import View
from core.models import ( from core.models import (
AIRunLog,
BusinessPlanDocument, BusinessPlanDocument,
BusinessPlanRevision, BusinessPlanRevision,
CommandAction, CommandAction,
@@ -20,6 +26,15 @@ from core.translation.engine import parse_quick_mode_title
class CommandRoutingSettings(LoginRequiredMixin, View): class CommandRoutingSettings(LoginRequiredMixin, View):
template_name = "pages/command-routing.html" template_name = "pages/command-routing.html"
@staticmethod
def _normalize_action_positions(profile):
rows = list(profile.actions.order_by("position", "id"))
for idx, row in enumerate(rows):
if row.position != idx:
row.position = idx
row.save(update_fields=["position", "updated_at"])
return rows
def _context(self, request): def _context(self, request):
profiles = ( profiles = (
CommandProfile.objects.filter(user=request.user) CommandProfile.objects.filter(user=request.user)
@@ -29,21 +44,12 @@ class CommandRoutingSettings(LoginRequiredMixin, View):
documents = BusinessPlanDocument.objects.filter(user=request.user).order_by( documents = BusinessPlanDocument.objects.filter(user=request.user).order_by(
"-updated_at" "-updated_at"
)[:30] )[:30]
bridges = TranslationBridge.objects.filter(user=request.user).order_by("-id")
events = (
TranslationEventLog.objects.filter(bridge__user=request.user)
.select_related("bridge")
.order_by("-created_at")[:50]
)
return { return {
"profiles": profiles, "profiles": profiles,
"documents": documents, "documents": documents,
"bridges": bridges,
"events": events,
"channel_services": ("web", "xmpp", "signal", "whatsapp"), "channel_services": ("web", "xmpp", "signal", "whatsapp"),
"directions": ("ingress", "egress", "scratchpad_mirror"), "directions": ("ingress", "egress", "scratchpad_mirror"),
"action_types": ("extract_bp", "post_result", "save_document"), "action_types": ("extract_bp", "post_result", "save_document"),
"bridge_directions": ("a_to_b", "b_to_a", "bidirectional"),
} }
def get(self, request): def get(self, request):
@@ -149,10 +155,66 @@ class CommandRoutingSettings(LoginRequiredMixin, View):
profile__user=request.user, profile__user=request.user,
) )
row.enabled = bool(request.POST.get("enabled")) row.enabled = bool(request.POST.get("enabled"))
if request.POST.get("position") not in (None, ""):
row.position = int(request.POST.get("position") or 0) row.position = int(request.POST.get("position") or 0)
row.save() row.save(update_fields=["enabled", "position", "updated_at"])
else:
row.save(update_fields=["enabled", "updated_at"])
return redirect("command_routing") return redirect("command_routing")
if action == "action_move":
row = get_object_or_404(
CommandAction,
id=request.POST.get("command_action_id"),
profile__user=request.user,
)
direction = str(request.POST.get("direction") or "").strip().lower()
if direction not in {"up", "down"}:
return redirect("command_routing")
with transaction.atomic():
ordered = self._normalize_action_positions(row.profile)
action_ids = [entry.id for entry in ordered]
try:
idx = action_ids.index(row.id)
except ValueError:
return redirect("command_routing")
target_idx = idx - 1 if direction == "up" else idx + 1
if target_idx < 0 or target_idx >= len(ordered):
return redirect("command_routing")
other = ordered[target_idx]
current_pos = ordered[idx].position
ordered[idx].position = other.position
other.position = current_pos
ordered[idx].save(update_fields=["position", "updated_at"])
other.save(update_fields=["position", "updated_at"])
return redirect("command_routing")
return redirect("command_routing")
class TranslationSettings(LoginRequiredMixin, View):
template_name = "pages/translation-settings.html"
def _context(self, request):
bridges = TranslationBridge.objects.filter(user=request.user).order_by("-id")
events = (
TranslationEventLog.objects.filter(bridge__user=request.user)
.select_related("bridge")
.order_by("-created_at")[:50]
)
return {
"bridges": bridges,
"events": events,
"channel_services": ("web", "xmpp", "signal", "whatsapp"),
"bridge_directions": ("a_to_b", "b_to_a", "bidirectional"),
}
def get(self, request):
return render(request, self.template_name, self._context(request))
def post(self, request):
action = str(request.POST.get("action") or "").strip()
if action == "bridge_create": if action == "bridge_create":
quick_title = str(request.POST.get("quick_mode_title") or "").strip() quick_title = str(request.POST.get("quick_mode_title") or "").strip()
inferred = parse_quick_mode_title(quick_title) inferred = parse_quick_mode_title(quick_title)
@@ -183,16 +245,84 @@ class CommandRoutingSettings(LoginRequiredMixin, View):
quick_mode_title=quick_title, quick_mode_title=quick_title,
settings={}, settings={},
) )
return redirect("command_routing") return redirect("translation_settings")
if action == "bridge_delete": if action == "bridge_delete":
bridge = get_object_or_404( bridge = get_object_or_404(
TranslationBridge, id=request.POST.get("bridge_id"), user=request.user TranslationBridge, id=request.POST.get("bridge_id"), user=request.user
) )
bridge.delete() bridge.delete()
return redirect("command_routing") return redirect("translation_settings")
return redirect("command_routing") return redirect("translation_settings")
class AIExecutionLogSettings(LoginRequiredMixin, View):
template_name = "pages/ai-execution-log.html"
def _context(self, request):
now = timezone.now()
runs_qs = AIRunLog.objects.filter(user=request.user)
runs = runs_qs.order_by("-started_at")[:300]
last_24h = runs_qs.filter(started_at__gte=now - timedelta(hours=24))
last_7d = runs_qs.filter(started_at__gte=now - timedelta(days=7))
total_runs = runs_qs.count()
total_ok = runs_qs.filter(status="ok").count()
total_failed = runs_qs.filter(status="failed").count()
avg_ms = runs_qs.aggregate(v=Avg("duration_ms")).get("v") or 0
success_rate = (float(total_ok) / float(total_runs) * 100.0) if total_runs else 0.0
usage_totals = runs_qs.aggregate(
prompt_chars_total=Sum("prompt_chars"),
response_chars_total=Sum("response_chars"),
avg_prompt_chars=Avg("prompt_chars"),
avg_response_chars=Avg("response_chars"),
)
stats = {
"total_runs": total_runs,
"total_ok": total_ok,
"total_failed": total_failed,
"last_24h_runs": last_24h.count(),
"last_24h_failed": last_24h.filter(status="failed").count(),
"last_7d_runs": last_7d.count(),
"avg_duration_ms": int(avg_ms),
"success_rate": round(success_rate, 1),
"total_prompt_chars": int(usage_totals.get("prompt_chars_total") or 0),
"total_response_chars": int(usage_totals.get("response_chars_total") or 0),
"avg_prompt_chars": int(usage_totals.get("avg_prompt_chars") or 0),
"avg_response_chars": int(usage_totals.get("avg_response_chars") or 0),
}
operation_breakdown = (
runs_qs.values("operation")
.annotate(
total=Count("id"),
failed=Count("id", filter=Q(status="failed")),
ok=Count("id", filter=Q(status="ok")),
)
.order_by("-total", "operation")[:20]
)
model_breakdown = (
runs_qs.values("model")
.annotate(
total=Count("id"),
failed=Count("id", filter=Q(status="failed")),
ok=Count("id", filter=Q(status="ok")),
)
.order_by("-total", "model")[:20]
)
return {
"stats": stats,
"runs": runs,
"operation_breakdown": operation_breakdown,
"model_breakdown": model_breakdown,
}
def get(self, request):
return render(request, self.template_name, self._context(request))
class BusinessPlanEditor(LoginRequiredMixin, View): class BusinessPlanEditor(LoginRequiredMixin, View):

View File

@@ -3406,6 +3406,7 @@ class ComposeDrafts(LoginRequiredMixin, View):
transcript=transcript, transcript=transcript,
), ),
ai_obj, ai_obj,
operation="compose_drafts",
) )
parsed = _parse_draft_options(result) parsed = _parse_draft_options(result)
if parsed: if parsed:
@@ -3478,6 +3479,7 @@ class ComposeSummary(LoginRequiredMixin, View):
transcript=transcript, transcript=transcript,
), ),
ai_obj, ai_obj,
operation="compose_summary",
) )
except Exception as exc: except Exception as exc:
return JsonResponse({"ok": False, "error": str(exc)}) return JsonResponse({"ok": False, "error": str(exc)})
@@ -3692,6 +3694,7 @@ class ComposeEngagePreview(LoginRequiredMixin, View):
generated = async_to_sync(ai_runner.run_prompt)( generated = async_to_sync(ai_runner.run_prompt)(
_build_engage_prompt(owner_name, recipient_name, transcript), _build_engage_prompt(owner_name, recipient_name, transcript),
ai_obj, ai_obj,
operation="compose_engage",
) )
outbound = _plain_text(generated) outbound = _plain_text(generated)
except Exception: except Exception:

View File

@@ -1,7 +1,7 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import date, datetime from datetime import date, datetime, timezone as dt_timezone
from decimal import Decimal, InvalidOperation from decimal import Decimal, InvalidOperation
from typing import Any, Callable from typing import Any, Callable
from urllib.parse import urlencode from urllib.parse import urlencode
@@ -14,10 +14,11 @@ from django.db.models import Q
from django.http import HttpResponseBadRequest from django.http import HttpResponseBadRequest
from django.shortcuts import render from django.shortcuts import render
from django.urls import reverse from django.urls import reverse
from django.utils import timezone
from django.views import View from django.views import View
from mixins.views import ObjectList from mixins.views import ObjectList
from core.models import Group, Manipulation, Person, Persona from core.models import Group, Manipulation, Message, Person, PersonIdentifier, Persona
def _context_type(request_type: str) -> str: def _context_type(request_type: str) -> str:
@@ -389,13 +390,130 @@ OSINT_SCOPES: dict[str, OsintScopeConfig] = {
"persona__alias__icontains", "persona__alias__icontains",
), ),
), ),
"identifiers": OsintScopeConfig(
key="identifiers",
title="Identifiers",
model=PersonIdentifier,
list_url_name="identifiers",
update_url_name="identifier_update",
delete_url_name="identifier_delete",
default_sort="identifier",
columns=(
OsintColumn(
key="id",
label="ID",
accessor=lambda item: item.id,
sort_field="id",
search_lookup="id__icontains",
kind="id_copy",
),
OsintColumn(
key="person",
label="Person",
accessor=lambda item: item.person.name if item.person_id else "",
sort_field="person__name",
search_lookup="person__name__icontains",
),
OsintColumn(
key="service",
label="Service",
accessor=lambda item: item.service,
sort_field="service",
search_lookup="service__icontains",
),
OsintColumn(
key="identifier",
label="Identifier",
accessor=lambda item: item.identifier,
sort_field="identifier",
search_lookup="identifier__icontains",
),
),
select_related=("person",),
delete_label=lambda item: item.identifier,
search_lookups=(
"id__icontains",
"person__name__icontains",
"service__icontains",
"identifier__icontains",
),
),
"messages": OsintScopeConfig(
key="messages",
title="Messages",
model=Message,
list_url_name="sessions",
update_url_name="session_update",
delete_url_name="session_delete",
default_sort="ts",
columns=(
OsintColumn(
key="id",
label="ID",
accessor=lambda item: item.id,
sort_field="id",
search_lookup="id__icontains",
kind="id_copy",
),
OsintColumn(
key="service",
label="Service",
accessor=lambda item: item.source_service or "",
sort_field="source_service",
search_lookup="source_service__icontains",
),
OsintColumn(
key="chat",
label="Chat",
accessor=lambda item: {
"display": "",
"copy": item.source_chat_id or "",
},
sort_field="source_chat_id",
search_lookup="source_chat_id__icontains",
kind="chat_ref",
),
OsintColumn(
key="sender",
label="Sender",
accessor=lambda item: item.custom_author or item.sender_uuid or "",
sort_field="sender_uuid",
search_lookup="sender_uuid__icontains",
),
OsintColumn(
key="text",
label="Text",
accessor=lambda item: item.text or "",
search_lookup="text__icontains",
),
OsintColumn(
key="ts",
label="Timestamp",
accessor=lambda item: datetime.fromtimestamp(item.ts / 1000.0),
sort_field="ts",
kind="datetime",
),
),
search_lookups=(
"id__icontains",
"text__icontains",
"source_service__icontains",
"source_chat_id__icontains",
"sender_uuid__icontains",
"custom_author__icontains",
"source_message_id__icontains",
),
),
} }
OSINT_SCOPE_ICONS: dict[str, str] = { OSINT_SCOPE_ICONS: dict[str, str] = {
"all": "fa-solid fa-globe",
"people": "fa-solid fa-user-group", "people": "fa-solid fa-user-group",
"groups": "fa-solid fa-users", "groups": "fa-solid fa-users",
"personas": "fa-solid fa-masks-theater", "personas": "fa-solid fa-masks-theater",
"manipulations": "fa-solid fa-sliders", "manipulations": "fa-solid fa-sliders",
"identifiers": "fa-solid fa-id-card",
"messages": "fa-solid fa-message",
} }
@@ -692,6 +810,124 @@ class OSINTSearch(LoginRequiredMixin, View):
result_template = "partials/results_table.html" result_template = "partials/results_table.html"
per_page_default = 20 per_page_default = 20
per_page_max = 100 per_page_max = 100
all_scope_keys = ("people", "identifiers", "messages")
@dataclass(frozen=True)
class SearchPlan:
size: int
index: str
query: str
tags: tuple[str, ...]
source: str
date_from: str
date_to: str
sort_mode: str
sentiment_min: str
sentiment_max: str
annotate: bool
dedup: bool
reverse: bool
def _prepare_siqtsrss_adr(self, request) -> "OSINTSearch.SearchPlan":
"""
Parse search controls following the Neptune-style SIQTSRSS/ADR flow.
S - Size, I - Index, Q - Query, T - Tags, S - Source, R - Ranges,
S - Sort, S - Sentiment, A - Annotate, D - Dedup, R - Reverse.
"""
query = str(request.GET.get("q") or "").strip()
tags = tuple(
token[4:].strip()
for token in query.split()
if token.lower().startswith("tag:")
)
return self.SearchPlan(
size=self._per_page(request.GET.get("per_page")),
index=self._scope_key(request.GET.get("scope")),
query=query,
tags=tags,
source=str(request.GET.get("source") or "all").strip().lower() or "all",
date_from=str(request.GET.get("date_from") or "").strip(),
date_to=str(request.GET.get("date_to") or "").strip(),
sort_mode=str(request.GET.get("sort_mode") or "relevance").strip().lower(),
sentiment_min=str(request.GET.get("sentiment_min") or "").strip(),
sentiment_max=str(request.GET.get("sentiment_max") or "").strip(),
annotate=str(request.GET.get("annotate") or "1").strip() not in {"0", "false", "off"},
dedup=str(request.GET.get("dedup") or "").strip() in {"1", "true", "on"},
reverse=str(request.GET.get("reverse") or "").strip() in {"1", "true", "on"},
)
def _parse_date_boundaries(self, plan: "OSINTSearch.SearchPlan") -> tuple[datetime | None, datetime | None]:
parsed_from = None
parsed_to = None
if plan.date_from:
try:
parsed_from = datetime.fromisoformat(plan.date_from)
except ValueError:
parsed_from = None
if plan.date_to:
try:
parsed_to = datetime.fromisoformat(plan.date_to)
except ValueError:
parsed_to = None
if parsed_to is not None:
parsed_to = parsed_to.replace(hour=23, minute=59, second=59, microsecond=999999)
return parsed_from, parsed_to
def _score_hit(self, query: str, primary: str, secondary: str) -> int:
if not query:
return 0
needle = query.lower()
return primary.lower().count(needle) * 3 + secondary.lower().count(needle)
def _identifier_exact_boost(self, query: str, identifier: str) -> int:
needle = str(query or "").strip().lower()
hay = str(identifier or "").strip().lower()
if not needle or not hay:
return 0
if hay == needle:
return 120
if hay.startswith(needle):
return 45
if needle in hay:
return 15
return 0
def _message_recency_boost(self, stamp: datetime | None) -> int:
if stamp is None:
return 0
now_ts = timezone.now()
if timezone.is_naive(stamp):
stamp = timezone.make_aware(stamp, dt_timezone.utc)
age = now_ts - stamp
if age.days < 1:
return 40
if age.days < 7:
return 25
if age.days < 30:
return 12
if age.days < 90:
return 6
return 0
def _snippet(self, text: str, query: str, max_len: int = 180) -> str:
value = str(text or "").strip()
if not value:
return ""
if not query:
return value[:max_len]
lower = value.lower()
needle = query.lower()
idx = lower.find(needle)
if idx < 0:
return value[:max_len]
start = max(idx - 40, 0)
end = min(idx + len(needle) + 90, len(value))
snippet = value[start:end]
if start > 0:
snippet = "" + snippet
if end < len(value):
snippet = snippet + ""
return snippet
def _field_options(self, model_cls: type[models.Model]) -> list[dict[str, str]]: def _field_options(self, model_cls: type[models.Model]) -> list[dict[str, str]]:
options = [] options = []
@@ -826,13 +1062,209 @@ class OSINTSearch(LoginRequiredMixin, View):
return min(value, self.per_page_max) return min(value, self.per_page_max)
def _scope_key(self, raw_scope: str | None) -> str: def _scope_key(self, raw_scope: str | None) -> str:
if raw_scope == "all":
return "all"
if raw_scope in OSINT_SCOPES: if raw_scope in OSINT_SCOPES:
return raw_scope return raw_scope
return "people" return "all"
def _query_state(self, request) -> dict[str, Any]: def _query_state(self, request) -> dict[str, Any]:
return {k: v for k, v in request.GET.items() if v not in {None, ""}} return {k: v for k, v in request.GET.items() if v not in {None, ""}}
def _apply_common_filters(
self,
queryset: models.QuerySet,
scope_key: str,
plan: "OSINTSearch.SearchPlan",
) -> models.QuerySet:
date_from, date_to = self._parse_date_boundaries(plan)
if plan.source and plan.source != "all":
if scope_key == "messages":
queryset = queryset.filter(source_service=plan.source)
elif scope_key == "identifiers":
queryset = queryset.filter(service=plan.source)
elif scope_key == "people":
queryset = queryset.filter(personidentifier__service=plan.source).distinct()
if scope_key == "messages":
if date_from is not None:
queryset = queryset.filter(ts__gte=int(date_from.timestamp() * 1000))
if date_to is not None:
queryset = queryset.filter(ts__lte=int(date_to.timestamp() * 1000))
elif scope_key == "people":
if date_from is not None:
queryset = queryset.filter(last_interaction__gte=date_from)
if date_to is not None:
queryset = queryset.filter(last_interaction__lte=date_to)
if plan.sentiment_min:
try:
queryset = queryset.filter(sentiment__gte=float(plan.sentiment_min))
except ValueError:
pass
if plan.sentiment_max:
try:
queryset = queryset.filter(sentiment__lte=float(plan.sentiment_max))
except ValueError:
pass
return queryset
def _search_all_rows(
self,
request,
plan: "OSINTSearch.SearchPlan",
) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
query = plan.query
date_from, date_to = self._parse_date_boundaries(plan)
per_scope_limit = max(40, min(plan.size * 3, 250))
allowed_scopes = set(self.all_scope_keys)
tag_scopes = {
tag.strip().lower()
for tag in plan.tags
if tag.strip().lower() in allowed_scopes
}
if tag_scopes:
allowed_scopes = tag_scopes
if "people" in allowed_scopes:
people_qs = self._apply_common_filters(
Person.objects.filter(user=request.user),
"people",
plan,
)
if query:
people_qs = people_qs.filter(
Q(name__icontains=query)
| Q(summary__icontains=query)
| Q(profile__icontains=query)
| Q(revealed__icontains=query)
| Q(likes__icontains=query)
| Q(dislikes__icontains=query)
)
for item in people_qs.order_by("-last_interaction", "name")[:per_scope_limit]:
secondary = self._snippet(
f"{item.summary or ''} {item.profile or ''}".strip(),
query if plan.annotate else "",
)
rows.append(
{
"id": f"person:{item.id}",
"scope": "Contact",
"primary": item.name,
"secondary": secondary or (item.timezone or ""),
"service": "-",
"when": item.last_interaction,
"score": self._score_hit(query, item.name or "", secondary or ""),
}
)
if "identifiers" in allowed_scopes:
identifiers_qs = self._apply_common_filters(
PersonIdentifier.objects.filter(user=request.user).select_related("person"),
"identifiers",
plan,
)
if query:
identifiers_qs = identifiers_qs.filter(
Q(identifier__icontains=query)
| Q(person__name__icontains=query)
| Q(service__icontains=query)
)
for item in identifiers_qs.order_by("person__name", "identifier")[:per_scope_limit]:
primary = item.person.name if item.person_id else item.identifier
secondary = item.identifier if item.person_id else ""
base_score = self._score_hit(query, primary or "", secondary or "")
exact_boost = self._identifier_exact_boost(query, item.identifier)
rows.append(
{
"id": f"identifier:{item.id}",
"scope": "Identifier",
"primary": primary,
"secondary": secondary,
"service": item.service or "",
"when": None,
"score": base_score + exact_boost,
}
)
if "messages" in allowed_scopes:
messages_qs = self._apply_common_filters(
Message.objects.filter(user=request.user),
"messages",
plan,
)
if query:
messages_qs = messages_qs.filter(
Q(text__icontains=query)
| Q(custom_author__icontains=query)
| Q(sender_uuid__icontains=query)
| Q(source_chat_id__icontains=query)
| Q(source_message_id__icontains=query)
)
for item in messages_qs.order_by("-ts")[:per_scope_limit]:
when_dt = datetime.fromtimestamp(item.ts / 1000.0) if item.ts else None
if date_from and when_dt and when_dt < date_from:
continue
if date_to and when_dt and when_dt > date_to:
continue
primary = item.custom_author or item.sender_uuid or (item.source_chat_id or "Message")
secondary = self._snippet(item.text or "", query if plan.annotate else "")
base_score = self._score_hit(query, primary or "", item.text or "")
recency_boost = self._message_recency_boost(when_dt)
rows.append(
{
"id": f"message:{item.id}",
"scope": "Message",
"primary": primary,
"secondary": secondary,
"service": item.source_service or "-",
"when": when_dt,
"score": base_score + recency_boost,
}
)
if plan.dedup:
seen = set()
deduped = []
for row in rows:
key = (
row["scope"],
str(row["primary"]).strip().lower(),
str(row["secondary"]).strip().lower(),
str(row["service"]).strip().lower(),
)
if key in seen:
continue
seen.add(key)
deduped.append(row)
rows = deduped
def row_time_key(row: dict[str, Any]) -> float:
stamp = row.get("when")
if stamp is None:
return 0.0
if timezone.is_aware(stamp):
return float(stamp.timestamp())
return float(stamp.replace(tzinfo=dt_timezone.utc).timestamp())
if plan.sort_mode == "oldest":
rows.sort(key=row_time_key)
elif plan.sort_mode == "recent":
rows.sort(key=row_time_key, reverse=True)
else:
rows.sort(
key=lambda row: (
row["score"],
row_time_key(row),
),
reverse=True,
)
if plan.reverse:
rows.reverse()
return rows
def _active_sort(self, scope: OsintScopeConfig) -> tuple[str, str]: def _active_sort(self, scope: OsintScopeConfig) -> tuple[str, str]:
direction = self.request.GET.get("dir", "asc").lower() direction = self.request.GET.get("dir", "asc").lower()
if direction not in {"asc", "desc"}: if direction not in {"asc", "desc"}:
@@ -958,11 +1390,48 @@ class OSINTSearch(LoginRequiredMixin, View):
if type not in self.allowed_types: if type not in self.allowed_types:
return HttpResponseBadRequest("Invalid type specified.") return HttpResponseBadRequest("Invalid type specified.")
scope_key = self._scope_key(request.GET.get("scope")) plan = self._prepare_siqtsrss_adr(request)
scope_key = plan.index
query = plan.query
list_url = reverse("osint_search", kwargs={"type": type})
query_state = self._query_state(request)
field_name = request.GET.get("field", "__all__")
if scope_key == "all":
rows_raw = self._search_all_rows(request, plan)
paginator = Paginator(rows_raw, plan.size)
page_obj = paginator.get_page(request.GET.get("page"))
column_context = [
{"key": "scope", "field_name": "scope", "label": "Type", "sortable": False, "kind": "text"},
{"key": "primary", "field_name": "primary", "label": "Primary", "sortable": False, "kind": "text"},
{"key": "secondary", "field_name": "secondary", "label": "Details", "sortable": False, "kind": "text"},
{"key": "service", "field_name": "service", "label": "Service", "sortable": False, "kind": "text"},
{"key": "when", "field_name": "when", "label": "When", "sortable": False, "kind": "datetime"},
]
rows = []
for item in list(page_obj.object_list):
rows.append(
{
"id": item["id"],
"cells": [
{"kind": "text", "value": item.get("scope")},
{"kind": "text", "value": item.get("primary")},
{"kind": "text", "value": item.get("secondary")},
{"kind": "text", "value": item.get("service")},
{"kind": "datetime", "value": item.get("when")},
],
"actions": [],
}
)
pagination = self._build_pagination(page_obj, list_url, query_state)
field_options: list[dict[str, str]] = []
selected_scope_key = "all"
osint_title = "Search Everything"
result_count = paginator.count
else:
scope = OSINT_SCOPES[scope_key] scope = OSINT_SCOPES[scope_key]
field_options = self._field_options(scope.model) field_options = self._field_options(scope.model)
query = request.GET.get("q", "").strip()
field_name = request.GET.get("field", "__all__")
if field_name != "__all__": if field_name != "__all__":
allowed_fields = {option["value"] for option in field_options} allowed_fields = {option["value"] for option in field_options}
if field_name not in allowed_fields: if field_name not in allowed_fields:
@@ -974,6 +1443,7 @@ class OSINTSearch(LoginRequiredMixin, View):
if scope.prefetch_related: if scope.prefetch_related:
queryset = queryset.prefetch_related(*scope.prefetch_related) queryset = queryset.prefetch_related(*scope.prefetch_related)
queryset = self._apply_common_filters(queryset, scope.key, plan)
queryset = self._search_queryset( queryset = self._search_queryset(
queryset, queryset,
scope.model, scope.model,
@@ -981,6 +1451,8 @@ class OSINTSearch(LoginRequiredMixin, View):
field_name, field_name,
field_options, field_options,
) )
if plan.dedup:
queryset = queryset.distinct()
sort_field = request.GET.get("sort", scope.default_sort) sort_field = request.GET.get("sort", scope.default_sort)
direction = request.GET.get("dir", "asc").lower() direction = request.GET.get("dir", "asc").lower()
@@ -994,14 +1466,13 @@ class OSINTSearch(LoginRequiredMixin, View):
if sort_field: if sort_field:
order_by = sort_field if direction == "asc" else f"-{sort_field}" order_by = sort_field if direction == "asc" else f"-{sort_field}"
queryset = queryset.order_by(order_by) queryset = queryset.order_by(order_by)
if plan.reverse:
queryset = queryset.reverse()
per_page = self._per_page(request.GET.get("per_page")) paginator = Paginator(queryset, plan.size)
paginator = Paginator(queryset, per_page)
page_obj = paginator.get_page(request.GET.get("page")) page_obj = paginator.get_page(request.GET.get("page"))
object_list = list(page_obj.object_list) object_list = list(page_obj.object_list)
list_url = reverse("osint_search", kwargs={"type": type})
query_state = self._query_state(request)
column_context = self._build_column_context( column_context = self._build_column_context(
scope, scope,
list_url, list_url,
@@ -1017,10 +1488,13 @@ class OSINTSearch(LoginRequiredMixin, View):
list_url, list_url,
query_state, query_state,
) )
selected_scope_key = scope.key
osint_title = f"Search {scope.title}"
result_count = paginator.count
context = { context = {
"osint_scope": scope.key, "osint_scope": selected_scope_key,
"osint_title": f"Search {scope.title}", "osint_title": osint_title,
"osint_table_id": "osint-search-table", "osint_table_id": "osint-search-table",
"osint_event_name": "", "osint_event_name": "",
"osint_refresh_url": _url_with_query(list_url, query_state), "osint_refresh_url": _url_with_query(list_url, query_state),
@@ -1029,17 +1503,30 @@ class OSINTSearch(LoginRequiredMixin, View):
"osint_pagination": pagination, "osint_pagination": pagination,
"osint_show_search": False, "osint_show_search": False,
"osint_show_actions": False, "osint_show_actions": False,
"osint_result_count": paginator.count, "osint_shell_borderless": True,
"osint_result_count": result_count,
"osint_search_url": list_url, "osint_search_url": list_url,
"scope_options": [ "scope_options": [
{"value": "all", "label": "All (Contacts + Messages)"},
*[
{"value": key, "label": conf.title} {"value": key, "label": conf.title}
for key, conf in OSINT_SCOPES.items() for key, conf in OSINT_SCOPES.items()
], ],
],
"field_options": field_options, "field_options": field_options,
"selected_scope": scope.key, "selected_scope": selected_scope_key,
"selected_field": field_name, "selected_field": field_name,
"search_query": query, "search_query": query,
"selected_per_page": per_page, "selected_per_page": plan.size,
"selected_source": plan.source,
"selected_date_from": plan.date_from,
"selected_date_to": plan.date_to,
"selected_sort_mode": plan.sort_mode,
"selected_sentiment_min": plan.sentiment_min,
"selected_sentiment_max": plan.sentiment_max,
"selected_annotate": plan.annotate,
"selected_dedup": plan.dedup,
"selected_reverse": plan.reverse,
"search_page_url": reverse("osint_search", kwargs={"type": "page"}), "search_page_url": reverse("osint_search", kwargs={"type": "page"}),
"search_widget_url": reverse("osint_search", kwargs={"type": "widget"}), "search_widget_url": reverse("osint_search", kwargs={"type": "widget"}),
} }