Implement executing tasks

2026-03-03 16:41:28 +00:00
parent d6bd56dace
commit 9c14e51b43
42 changed files with 3410 additions and 121 deletions
--- a/scripts/quadlet/manage.sh
+++ b/scripts/quadlet/manage.sh
@@ -3,21 +3,39 @@ set -euo pipefail

 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 STACK_ENV="${STACK_ENV:-$ROOT_DIR/stack.env}"
-POD_NAME="gia"

-REDIS_CONTAINER="redis_gia"
-SIGNAL_CONTAINER="signal"
-MIGRATION_CONTAINER="migration_gia"
-COLLECTSTATIC_CONTAINER="collectstatic_gia"
-APP_CONTAINER="gia"
-ASGI_CONTAINER="asgi_gia"
-UR_CONTAINER="ur_gia"
-SCHED_CONTAINER="scheduling_gia"
+STACK_ID="${GIA_STACK_ID:-${STACK_ID:-}}"
+STACK_ID="$(echo "$STACK_ID" | tr -cs 'a-zA-Z0-9._-' '-' | sed 's/^-*//; s/-*$//')"
+
+name_with_stack() {
+  local base="$1"
+  if [[ -n "$STACK_ID" ]]; then
+    echo "${base}_${STACK_ID}"
+  else
+    echo "$base"
+  fi
+}
+
+POD_NAME="$(name_with_stack "gia")"
+
+REDIS_CONTAINER="$(name_with_stack "redis_gia")"
+SIGNAL_CONTAINER="$(name_with_stack "signal")"
+MIGRATION_CONTAINER="$(name_with_stack "migration_gia")"
+COLLECTSTATIC_CONTAINER="$(name_with_stack "collectstatic_gia")"
+APP_CONTAINER="$(name_with_stack "gia")"
+ASGI_CONTAINER="$(name_with_stack "asgi_gia")"
+UR_CONTAINER="$(name_with_stack "ur_gia")"
+SCHED_CONTAINER="$(name_with_stack "scheduling_gia")"
+CODEX_WORKER_CONTAINER="$(name_with_stack "codex_worker_gia")"

 REDIS_DATA_DIR="${QUADLET_REDIS_DATA_DIR:-$ROOT_DIR/.podman/gia_redis_data}"
 WHATSAPP_DATA_DIR="${QUADLET_WHATSAPP_DATA_DIR:-$ROOT_DIR/.podman/gia_whatsapp_data}"
 SQLITE_DATA_DIR="${QUADLET_SQLITE_DATA_DIR:-$ROOT_DIR/.podman/gia_sqlite_data}"
-VRUN_DIR="/code/vrun"
+if [[ -n "${STACK_ID}" ]]; then
+  VRUN_DIR="/code/vrun/${STACK_ID}"
+else
+  VRUN_DIR="/code/vrun"
+fi

 load_env() {
  set -a
@@ -134,6 +152,7 @@ down_stack() {
  rm_if_exists "$ASGI_CONTAINER"
  rm_if_exists "$UR_CONTAINER"
  rm_if_exists "$SCHED_CONTAINER"
+  rm_if_exists "$CODEX_WORKER_CONTAINER"
 }

 start_stack() {
@@ -153,7 +172,15 @@ start_stack() {
  chmod 0666 "$HOST_DATABASE_FILE" 2>/dev/null || true
  down_stack

-  podman pod create --name "$POD_NAME" -p "${APP_PORT:-5006}:8000" -p "8080:8080" >/dev/null
+  local port_offset="${GIA_STACK_PORT_OFFSET:-}"
+  if [[ -z "$port_offset" && -n "$STACK_ID" ]]; then
+    port_offset="$(( $(printf '%s' "$STACK_ID" | cksum | awk '{print $1}') % 500 + 1 ))"
+  fi
+  port_offset="${port_offset:-0}"
+  local app_port="${APP_PORT:-$((5006 + port_offset))}"
+  local signal_port="${SIGNAL_PUBLIC_PORT:-$((8080 + port_offset))}"
+
+  podman pod create --name "$POD_NAME" -p "${app_port}:8000" -p "${signal_port}:8080" >/dev/null

  podman run -d \
    --replace \
@@ -182,6 +209,7 @@ start_stack() {
  run_worker_container "$ASGI_CONTAINER" "rm -f /var/run/asgi-gia.sock && . /venv/bin/activate && python -m pip install --disable-pip-version-check -q uvicorn && python -m uvicorn app.asgi:application --uds /var/run/asgi-gia.sock --workers 1" 0 1
  run_worker_container "$UR_CONTAINER" ". /venv/bin/activate && python manage.py ur" 1 1
  run_worker_container "$SCHED_CONTAINER" ". /venv/bin/activate && python manage.py scheduling" 1 0
+  run_worker_container "$CODEX_WORKER_CONTAINER" ". /venv/bin/activate && python manage.py codex_worker" 1 0
 }

 render_units() {
@@ -211,18 +239,23 @@ case "${1:-}" in
  status)
    require_podman
    podman pod ps --format "table {{.Name}}\t{{.Status}}" | grep -E "^$POD_NAME\b" || true
-    podman ps --format "table {{.Names}}\t{{.Status}}" | grep -E "^($APP_CONTAINER|$ASGI_CONTAINER|$UR_CONTAINER|$SCHED_CONTAINER|$REDIS_CONTAINER|$SIGNAL_CONTAINER)\b" || true
+    podman ps --format "table {{.Names}}\t{{.Status}}" | grep -E "^($APP_CONTAINER|$ASGI_CONTAINER|$UR_CONTAINER|$SCHED_CONTAINER|$CODEX_WORKER_CONTAINER|$REDIS_CONTAINER|$SIGNAL_CONTAINER)\b" || true
    ;;
  logs)
    require_podman
    if is_remote; then
      podman logs -f "$APP_CONTAINER"
    else
-      podman logs -f "$APP_CONTAINER" "$ASGI_CONTAINER" "$UR_CONTAINER" "$SCHED_CONTAINER" "$REDIS_CONTAINER" "$SIGNAL_CONTAINER"
+      podman logs -f "$APP_CONTAINER" "$ASGI_CONTAINER" "$UR_CONTAINER" "$SCHED_CONTAINER" "$CODEX_WORKER_CONTAINER" "$REDIS_CONTAINER" "$SIGNAL_CONTAINER"
    fi
    ;;
+  watch)
+    require_podman
+    load_env
+    exec "$ROOT_DIR/scripts/quadlet/watchdog.sh"
+    ;;
  *)
-    echo "Usage: $0 {install|up|down|restart|status|logs}" >&2
+    echo "Usage: $0 {install|up|down|restart|status|logs|watch}" >&2
    exit 2
    ;;
 esac
--- a/scripts/quadlet/render_units.py
+++ b/scripts/quadlet/render_units.py
@@ -45,6 +45,27 @@ def main() -> int:
    repo_root = Path(__file__).resolve().parents[2]
    stack_env_path = abs_from(repo_root, args.stack_env, "stack.env")
    env = parse_env(stack_env_path)
+    stack_id = str(env.get("GIA_STACK_ID") or env.get("STACK_ID") or "").strip()
+    stack_id = "".join(ch if (ch.isalnum() or ch in "._-") else "-" for ch in stack_id).strip("-")
+
+    def with_stack(base: str) -> str:
+        return f"{base}_{stack_id}" if stack_id else base
+
+    unit_prefix = f"gia-{stack_id}" if stack_id else "gia"
+    pod_ref = f"{unit_prefix}.pod"
+    target_ref = f"{unit_prefix}.target"
+    stack_offset_raw = str(env.get("GIA_STACK_PORT_OFFSET") or "").strip()
+    if stack_offset_raw:
+        try:
+            stack_port_offset = max(0, int(stack_offset_raw))
+        except Exception:
+            stack_port_offset = 0
+    elif stack_id:
+        stack_port_offset = (sum(ord(ch) for ch in stack_id) % 500) + 1
+    else:
+        stack_port_offset = 0
+    app_port = int(env.get("APP_PORT") or (5006 + stack_port_offset))
+    signal_public_port = int(env.get("SIGNAL_PUBLIC_PORT") or (8080 + stack_port_offset))

    repo_dir = abs_from(repo_root, env.get("REPO_DIR", "."), ".")
    host_uid = int(os.getuid())
@@ -62,7 +83,7 @@ def main() -> int:
    redis_data_dir = abs_from(repo_root, env.get("QUADLET_REDIS_DATA_DIR", "./.podman/gia_redis_data"), "./.podman/gia_redis_data")
    whatsapp_data_dir = abs_from(repo_root, env.get("QUADLET_WHATSAPP_DATA_DIR", "./.podman/gia_whatsapp_data"), "./.podman/gia_whatsapp_data")

-    vrun_dir = Path("/code/vrun")
+    vrun_dir = Path("/code/vrun") / stack_id if stack_id else Path("/code/vrun")
    signal_cli_dir = (repo_dir / "signal-cli-config").resolve()
    uwsgi_ini = (repo_dir / "docker" / "uwsgi.ini").resolve()
    redis_conf = (repo_dir / "docker" / "redis.conf").resolve()
@@ -80,22 +101,24 @@ def main() -> int:

    env_file = stack_env_path

-    pod_unit = """
+    pod_unit = f"""
 [Unit]
 Description=GIA Pod

 [Pod]
-PodName=gia
+PodName={with_stack('gia')}
+PublishPort={app_port}:8000
+PublishPort={signal_public_port}:8080

 [Install]
 WantedBy=default.target
 """

-    target_unit = """
+    target_unit = f"""
 [Unit]
 Description=GIA Stack Target
-Wants=gia-redis.service gia-signal.service gia-migration.service gia-collectstatic.service gia-app.service gia-asgi.service gia-ur.service gia-scheduling.service
-After=gia-redis.service gia-signal.service gia-migration.service gia-collectstatic.service
+Wants={unit_prefix}-redis.service {unit_prefix}-signal.service {unit_prefix}-migration.service {unit_prefix}-collectstatic.service {unit_prefix}-app.service {unit_prefix}-asgi.service {unit_prefix}-ur.service {unit_prefix}-scheduling.service {unit_prefix}-codex-worker.service
+After={unit_prefix}-redis.service {unit_prefix}-signal.service {unit_prefix}-migration.service {unit_prefix}-collectstatic.service

 [Install]
 WantedBy=default.target
@@ -104,14 +127,14 @@ WantedBy=default.target
    redis_unit = f"""
 [Unit]
 Description=GIA Redis
-PartOf=gia.target
+PartOf={target_ref}
 After=network-online.target
 Wants=network-online.target

 [Container]
 Image=docker.io/library/redis:latest
-ContainerName=redis_gia
-Pod=gia.pod
+ContainerName={with_stack('redis_gia')}
+Pod={pod_ref}
 Volume={redis_conf}:/etc/redis.conf:ro
 Volume={redis_data_dir}:/data
 Volume={vrun_dir}:/var/run
@@ -122,20 +145,20 @@ Restart=always
 RestartSec=2

 [Install]
-WantedBy=gia.target
+WantedBy={target_ref}
 """

    signal_unit = f"""
 [Unit]
 Description=GIA Signal API
-PartOf=gia.target
+PartOf={target_ref}
 After=network-online.target
 Wants=network-online.target

 [Container]
 Image=docker.io/bbernhard/signal-cli-rest-api:latest
-ContainerName=signal
-Pod=gia.pod
+ContainerName={with_stack('signal')}
+Pod={pod_ref}
 Volume={signal_cli_dir}:/home/.local/share/signal-cli
 Environment=MODE=json-rpc

@@ -144,21 +167,21 @@ Restart=always
 RestartSec=2

 [Install]
-WantedBy=gia.target
+WantedBy={target_ref}
 """

    def gia_container(name: str, container_name: str, command: str, include_uwsgi: bool, include_whatsapp: bool, after: str, requires: str, one_shot: bool = False) -> str:
        lines = [
            "[Unit]",
            f"Description={name}",
-            "PartOf=gia.target",
+            f"PartOf={target_ref}",
            f"After={after}",
            f"Requires={requires}",
            "",
            "[Container]",
            "Image=localhost/xf/gia:prod",
            f"ContainerName={container_name}",
-            "Pod=gia.pod",
+            f"Pod={pod_ref}",
            f"User={host_uid}:{host_gid}",
            f"EnvironmentFile={env_file}",
            "Environment=SIGNAL_HTTP_URL=http://127.0.0.1:8080",
@@ -178,7 +201,7 @@ WantedBy=gia.target
                "Type=oneshot",
                "RemainAfterExit=yes",
                "TimeoutStartSec=0",
-                "ExecStartPre=/bin/sh -c 'for i in $(seq 1 60); do [ -S /code/vrun/gia-redis.sock ] && exit 0; sleep 1; done; exit 1'",
+                f"ExecStartPre=/bin/sh -c 'for i in $(seq 1 60); do [ -S {vrun_dir}/gia-redis.sock ] && exit 0; sleep 1; done; exit 1'",
            ])
        else:
            lines.extend([
@@ -186,80 +209,92 @@ WantedBy=gia.target
                "RestartSec=2",
            ])
        lines.extend(["", "[Install]", "WantedBy=gia.target"])
+        lines[-1] = f"WantedBy={target_ref}"
        return "\n".join(lines)

    migration_unit = gia_container(
        "GIA Migration",
-        "migration_gia",
+        with_stack("migration_gia"),
        "sh -c '. /venv/bin/activate && python manage.py migrate --noinput'",
        include_uwsgi=False,
        include_whatsapp=False,
-        after="gia-redis.service gia-signal.service",
-        requires="gia-redis.service gia-signal.service",
+        after=f"{unit_prefix}-redis.service {unit_prefix}-signal.service",
+        requires=f"{unit_prefix}-redis.service {unit_prefix}-signal.service",
        one_shot=True,
    )

    collectstatic_unit = gia_container(
        "GIA Collectstatic",
-        "collectstatic_gia",
+        with_stack("collectstatic_gia"),
        "sh -c '. /venv/bin/activate && python manage.py collectstatic --noinput'",
        include_uwsgi=False,
        include_whatsapp=False,
-        after="gia-migration.service",
-        requires="gia-migration.service",
+        after=f"{unit_prefix}-migration.service",
+        requires=f"{unit_prefix}-migration.service",
        one_shot=True,
    )

    app_unit = gia_container(
        "GIA App",
-        "gia",
+        with_stack("gia"),
        "sh -c 'if [ \\\"$OPERATION\\\" = \\\"uwsgi\\\" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi'",
        include_uwsgi=True,
        include_whatsapp=True,
-        after="gia-collectstatic.service gia-redis.service gia-signal.service",
-        requires="gia-collectstatic.service gia-redis.service gia-signal.service",
+        after=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
+        requires=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
    )

    asgi_unit = gia_container(
        "GIA ASGI",
-        "asgi_gia",
+        with_stack("asgi_gia"),
        "sh -c 'rm -f /var/run/asgi-gia.sock && . /venv/bin/activate && python -m pip install --disable-pip-version-check -q uvicorn && python -m uvicorn app.asgi:application --uds /var/run/asgi-gia.sock --workers 1'",
        include_uwsgi=False,
        include_whatsapp=True,
-        after="gia-collectstatic.service gia-redis.service gia-signal.service",
-        requires="gia-collectstatic.service gia-redis.service gia-signal.service",
+        after=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
+        requires=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
    )

    ur_unit = gia_container(
        "GIA Unified Router",
-        "ur_gia",
+        with_stack("ur_gia"),
        "sh -c '. /venv/bin/activate && python manage.py ur'",
        include_uwsgi=True,
        include_whatsapp=True,
-        after="gia-collectstatic.service gia-redis.service gia-signal.service",
-        requires="gia-collectstatic.service gia-redis.service gia-signal.service",
+        after=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
+        requires=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
    )

    scheduling_unit = gia_container(
        "GIA Scheduling",
-        "scheduling_gia",
+        with_stack("scheduling_gia"),
        "sh -c '. /venv/bin/activate && python manage.py scheduling'",
        include_uwsgi=True,
        include_whatsapp=False,
-        after="gia-collectstatic.service gia-redis.service gia-signal.service",
-        requires="gia-collectstatic.service gia-redis.service gia-signal.service",
+        after=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
+        requires=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
    )

-    write_unit(out_dir / "gia.pod", pod_unit)
-    write_unit(out_dir / "gia.target", target_unit)
-    write_unit(out_dir / "gia-redis.container", redis_unit)
-    write_unit(out_dir / "gia-signal.container", signal_unit)
-    write_unit(out_dir / "gia-migration.container", migration_unit)
-    write_unit(out_dir / "gia-collectstatic.container", collectstatic_unit)
-    write_unit(out_dir / "gia-app.container", app_unit)
-    write_unit(out_dir / "gia-asgi.container", asgi_unit)
-    write_unit(out_dir / "gia-ur.container", ur_unit)
-    write_unit(out_dir / "gia-scheduling.container", scheduling_unit)
+    codex_worker_unit = gia_container(
+        "GIA Codex Worker",
+        with_stack("codex_worker_gia"),
+        "sh -c '. /venv/bin/activate && python manage.py codex_worker'",
+        include_uwsgi=True,
+        include_whatsapp=False,
+        after=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
+        requires=f"{unit_prefix}-collectstatic.service {unit_prefix}-redis.service {unit_prefix}-signal.service",
+    )
+
+    write_unit(out_dir / f"{unit_prefix}.pod", pod_unit)
+    write_unit(out_dir / f"{unit_prefix}.target", target_unit)
+    write_unit(out_dir / f"{unit_prefix}-redis.container", redis_unit)
+    write_unit(out_dir / f"{unit_prefix}-signal.container", signal_unit)
+    write_unit(out_dir / f"{unit_prefix}-migration.container", migration_unit)
+    write_unit(out_dir / f"{unit_prefix}-collectstatic.container", collectstatic_unit)
+    write_unit(out_dir / f"{unit_prefix}-app.container", app_unit)
+    write_unit(out_dir / f"{unit_prefix}-asgi.container", asgi_unit)
+    write_unit(out_dir / f"{unit_prefix}-ur.container", ur_unit)
+    write_unit(out_dir / f"{unit_prefix}-scheduling.container", scheduling_unit)
+    write_unit(out_dir / f"{unit_prefix}-codex-worker.container", codex_worker_unit)

    print(f"Wrote Quadlet units to: {out_dir}")
    return 0
--- a/scripts/quadlet/watchdog.sh
+++ b/scripts/quadlet/watchdog.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_ENV="${STACK_ENV:-$ROOT_DIR/stack.env}"
+STACK_ID="${GIA_STACK_ID:-${STACK_ID:-}}"
+STACK_ID="$(echo "$STACK_ID" | tr -cs 'a-zA-Z0-9._-' '-' | sed 's/^-*//; s/-*$//')"
+SLEEP_SECONDS="${WATCHDOG_SLEEP_SECONDS:-15}"
+NTFY_TOPIC="${NTFY_TOPIC:-${NOTIFY_TOPIC:-}}"
+NTFY_URL_BASE="${NTFY_URL_BASE:-https://ntfy.sh}"
+HOST_TAG="${HOSTNAME:-$(hostname 2>/dev/null || echo unknown-host)}"
+
+if [[ -f "$STACK_ENV" ]]; then
+  set -a
+  . "$STACK_ENV"
+  set +a
+fi
+
+name_with_stack() {
+  local base="$1"
+  if [[ -n "$STACK_ID" ]]; then
+    echo "${base}_${STACK_ID}"
+  else
+    echo "$base"
+  fi
+}
+
+notify() {
+  local title="$1"
+  local msg="$2"
+  if [[ -z "$NTFY_TOPIC" ]]; then
+    return 0
+  fi
+  if ! command -v curl >/dev/null 2>&1; then
+    return 0
+  fi
+  curl -sS -X POST "${NTFY_URL_BASE%/}/$NTFY_TOPIC" \
+    -H "Title: $title" \
+    -H "Tags: warning" \
+    -d "$msg" >/dev/null || true
+}
+
+CONTAINERS=(
+  "$(name_with_stack "gia")"
+  "$(name_with_stack "asgi_gia")"
+  "$(name_with_stack "ur_gia")"
+  "$(name_with_stack "scheduling_gia")"
+  "$(name_with_stack "codex_worker_gia")"
+)
+
+declare -A LAST_STATE
+for name in "${CONTAINERS[@]}"; do
+  LAST_STATE["$name"]="unknown"
+done
+
+while true; do
+  for name in "${CONTAINERS[@]}"; do
+    running="false"
+    if inspect_out="$(podman inspect -f '{{.State.Running}}' "$name" 2>/dev/null)"; then
+      running="$(echo "$inspect_out" | tr -d '\n' | tr 'A-Z' 'a-z')"
+    fi
+
+    if [[ "$running" == "true" ]]; then
+      if [[ "${LAST_STATE[$name]}" != "up" ]]; then
+        notify "GIA recovered: $name" "[$HOST_TAG] container $name is now running"
+      fi
+      LAST_STATE["$name"]="up"
+      continue
+    fi
+
+    restart_out=""
+    if restart_out="$(podman restart "$name" 2>&1)"; then
+      LAST_STATE["$name"]="recovering"
+      notify "GIA restarted: $name" "[$HOST_TAG] container $name was not running and restart succeeded"
+    else
+      LAST_STATE["$name"]="down"
+      notify "GIA restart failed: $name" "[$HOST_TAG] restart failed for $name: $restart_out"
+    fi
+  done
+
+  sleep "$SLEEP_SECONDS"
+done