From d22924f6aad90866902fa9dcb6baa63a309249fe Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 1 Mar 2026 17:27:48 +0000 Subject: [PATCH] Fix containerised restarts --- Containerfile.dev | 56 ++++++++++++++++++++++++++ app/settings.py | 2 +- core/templates/pages/osint-search.html | 10 ----- enter.sh | 1 + scripts/quadlet/manage.sh | 50 ++++++++++++++++++++--- scripts/quadlet/render_units.py | 22 +++++++++- uwsgi.ini | 0 7 files changed, 122 insertions(+), 19 deletions(-) create mode 100644 Containerfile.dev create mode 120000 enter.sh create mode 100755 uwsgi.ini diff --git a/Containerfile.dev b/Containerfile.dev new file mode 100644 index 0000000..5d30a2e --- /dev/null +++ b/Containerfile.dev @@ -0,0 +1,56 @@ +FROM python:3.11-bookworm + +ARG USER_ID=1000 +ARG GROUP_ID=1000 +ARG USER_NAME=dev + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash-completion \ + build-essential \ + cargo \ + ca-certificates \ + curl \ + fd-find \ + fzf \ + git \ + golang \ + jq \ + less \ + libffi-dev \ + libssl-dev \ + make \ + neovim \ + nodejs \ + npm \ + procps \ + ripgrep \ + rsync \ + rustc \ + sqlite3 \ + tar \ + tmux \ + unzip \ + wget \ + which \ + zip && \ + rm -rf /var/lib/apt/lists/* && \ + ln -sf /usr/bin/fdfind /usr/local/bin/fd + +RUN groupadd -g "${GROUP_ID}" "${USER_NAME}" 2>/dev/null || true && \ + useradd -m -u "${USER_ID}" -g "${GROUP_ID}" -s /bin/bash "${USER_NAME}" + +USER ${USER_NAME} +WORKDIR /home/${USER_NAME} + +# Build a project virtualenv and preinstall dependencies. +COPY --chown=${USER_NAME}:${USER_NAME} requirements.txt /tmp/requirements.txt +RUN bash -lc 'set -e; \ + python3.11 -m venv /home/${USER_NAME}/.venv/gia && \ + /home/${USER_NAME}/.venv/gia/bin/pip install --upgrade pip setuptools wheel && \ + grep -Ev "^(git\\+https://git\\.zm\\.is/|aiograpi$)" /tmp/requirements.txt > /tmp/requirements.build.txt && \ + /home/${USER_NAME}/.venv/gia/bin/pip install -r /tmp/requirements.build.txt' + +ENV VIRTUAL_ENV=/home/${USER_NAME}/.venv/gia +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +CMD ["/bin/bash"] diff --git a/app/settings.py b/app/settings.py index a01d9b2..697adc4 100644 --- a/app/settings.py +++ b/app/settings.py @@ -130,7 +130,7 @@ WSGI_APPLICATION = "app.wsgi.application" DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": "/conf/db.sqlite3", + "NAME": os.getenv("APP_DATABASE_PATH", "/conf/db.sqlite3"), } } diff --git a/core/templates/pages/osint-search.html b/core/templates/pages/osint-search.html index 4ea94c2..9b9b634 100644 --- a/core/templates/pages/osint-search.html +++ b/core/templates/pages/osint-search.html @@ -10,16 +10,6 @@ Search across OSINT objects with sortable, paginated results.

-
- -
{% include "partials/osint/search-panel.html" %} diff --git a/enter.sh b/enter.sh new file mode 120000 index 0000000..fbc303c --- /dev/null +++ b/enter.sh @@ -0,0 +1 @@ +/code/xf/entersh/enter.sh \ No newline at end of file diff --git a/scripts/quadlet/manage.sh b/scripts/quadlet/manage.sh index 6ebf128..25b389f 100755 --- a/scripts/quadlet/manage.sh +++ b/scripts/quadlet/manage.sh @@ -16,6 +16,7 @@ SCHED_CONTAINER="scheduling_gia" REDIS_DATA_DIR="${QUADLET_REDIS_DATA_DIR:-$ROOT_DIR/.podman/gia_redis_data}" WHATSAPP_DATA_DIR="${QUADLET_WHATSAPP_DATA_DIR:-$ROOT_DIR/.podman/gia_whatsapp_data}" +SQLITE_DATA_DIR="${QUADLET_SQLITE_DATA_DIR:-$ROOT_DIR/.podman/gia_sqlite_data}" VRUN_DIR="/code/vrun" load_env() { @@ -24,6 +25,19 @@ load_env() { set +a } +is_remote() { + [[ -n "${CONTAINER_HOST:-}" ]] || [[ -n "${PODMAN_HOST:-}" ]] || [[ -n "${DOCKER_HOST:-}" ]] +} + +resolve_path() { + local path="$1" + if [[ "$path" = /* ]]; then + echo "$path" + else + echo "$ROOT_DIR/$path" + fi +} + require_podman() { if ! command -v podman >/dev/null 2>&1; then echo "podman not found" >&2 @@ -32,11 +46,12 @@ require_podman() { } ensure_dirs() { - mkdir -p "$REDIS_DATA_DIR" "$WHATSAPP_DATA_DIR" "$VRUN_DIR" "$ROOT_DIR/signal-cli-config" + mkdir -p "$REDIS_DATA_DIR" "$WHATSAPP_DATA_DIR" "$SQLITE_DATA_DIR" "$VRUN_DIR" "$ROOT_DIR/signal-cli-config" + chmod 0777 "$SQLITE_DATA_DIR" 2>/dev/null || true # Container runs as uid 1000 (xf); rootless Podman remaps uids so plain # chown won't work — podman unshare translates to the correct host uid. if [[ -n "${QUADLET_SKIP_UNSHARE:-}" ]] || [[ -n "${CONTAINER_HOST:-}" ]] || [[ -n "${PODMAN_HOST:-}" ]] || [[ -n "${DOCKER_HOST:-}" ]]; then - echo "Skipping podman unshare chown (QUADLET_SKIP_UNSHARE or remote host detected)" + echo "Skipping podman unshare chown -- running in sandbox" else podman unshare chown 1000:1000 "$WHATSAPP_DATA_DIR" 2>/dev/null || true fi @@ -68,10 +83,12 @@ run_worker_container() { --replace --name "$name" --pod "$POD_NAME" + --user "$(id -u):$(id -g)" --env-file "$STACK_ENV" --env "SIGNAL_HTTP_URL=http://127.0.0.1:8080" + --env "APP_DATABASE_PATH=$APP_DATABASE_PATH" -v "$REPO_DIR:/code" - -v "$APP_DATABASE_FILE:/conf/db.sqlite3" + -v "$SQLITE_DATA_DIR:/conf" -v "$VRUN_DIR:/var/run" ) if [[ "$with_uwsgi" == "1" ]]; then @@ -93,10 +110,12 @@ run_oneshot_container() { --replace --name "$name" --pod "$POD_NAME" + --user "$(id -u):$(id -g)" --env-file "$STACK_ENV" --env "SIGNAL_HTTP_URL=http://127.0.0.1:8080" + --env "APP_DATABASE_PATH=$APP_DATABASE_PATH" -v "$REPO_DIR:/code" - -v "$APP_DATABASE_FILE:/conf/db.sqlite3" + -v "$SQLITE_DATA_DIR:/conf" -v "$VRUN_DIR:/var/run" ) if [[ "$with_whatsapp" == "1" ]]; then @@ -120,7 +139,18 @@ down_stack() { start_stack() { require_podman load_env + REPO_DIR="$(resolve_path "$REPO_DIR")" + APP_DATABASE_FILE="$(resolve_path "$APP_DATABASE_FILE")" + APP_DATABASE_BASENAME="$(basename "$APP_DATABASE_FILE")" + APP_DATABASE_PATH="/conf/$APP_DATABASE_BASENAME" + HOST_DATABASE_FILE="$SQLITE_DATA_DIR/$APP_DATABASE_BASENAME" + APP_LOCAL_SETTINGS="$(resolve_path "$APP_LOCAL_SETTINGS")" ensure_dirs + if [[ "$APP_DATABASE_FILE" != "$HOST_DATABASE_FILE" ]] && [[ -f "$APP_DATABASE_FILE" ]] && [[ ! -f "$HOST_DATABASE_FILE" ]]; then + cp "$APP_DATABASE_FILE" "$HOST_DATABASE_FILE" + fi + touch "$HOST_DATABASE_FILE" + chmod 0666 "$HOST_DATABASE_FILE" 2>/dev/null || true down_stack podman pod create --name "$POD_NAME" -p "${APP_PORT:-5006}:8000" -p "8080:8080" >/dev/null @@ -165,7 +195,11 @@ case "${1:-}" in up) start_stack trap 'down_stack; exit 0' INT TERM - podman logs -f "$APP_CONTAINER" "$ASGI_CONTAINER" "$UR_CONTAINER" "$SCHED_CONTAINER" "$REDIS_CONTAINER" "$SIGNAL_CONTAINER" || true + if is_remote; then + podman logs -f "$APP_CONTAINER" || true + else + podman logs -f "$APP_CONTAINER" "$ASGI_CONTAINER" "$UR_CONTAINER" "$SCHED_CONTAINER" "$REDIS_CONTAINER" "$SIGNAL_CONTAINER" || true + fi ;; down) require_podman @@ -181,7 +215,11 @@ case "${1:-}" in ;; logs) require_podman - podman logs -f "$APP_CONTAINER" "$ASGI_CONTAINER" "$UR_CONTAINER" "$SCHED_CONTAINER" "$REDIS_CONTAINER" "$SIGNAL_CONTAINER" + if is_remote; then + podman logs -f "$APP_CONTAINER" + else + podman logs -f "$APP_CONTAINER" "$ASGI_CONTAINER" "$UR_CONTAINER" "$SCHED_CONTAINER" "$REDIS_CONTAINER" "$SIGNAL_CONTAINER" + fi ;; *) echo "Usage: $0 {install|up|down|restart|status|logs}" >&2 diff --git a/scripts/quadlet/render_units.py b/scripts/quadlet/render_units.py index 31a8c2e..0dc04cc 100755 --- a/scripts/quadlet/render_units.py +++ b/scripts/quadlet/render_units.py @@ -4,6 +4,7 @@ from __future__ import annotations from pathlib import Path import argparse import os +import shutil def parse_env(path: Path) -> dict[str, str]: @@ -46,7 +47,17 @@ def main() -> int: env = parse_env(stack_env_path) repo_dir = abs_from(repo_root, env.get("REPO_DIR", "."), ".") + host_uid = int(os.getuid()) + host_gid = int(os.getgid()) app_db_file = abs_from(repo_root, env.get("APP_DATABASE_FILE", "./db.sqlite3"), "./db.sqlite3") + app_db_basename = app_db_file.name + sqlite_data_dir = abs_from( + repo_root, + env.get("QUADLET_SQLITE_DATA_DIR", "./.podman/gia_sqlite_data"), + "./.podman/gia_sqlite_data", + ) + host_db_file = (sqlite_data_dir / app_db_basename).resolve() + app_db_path_in_container = f"/conf/{app_db_basename}" redis_data_dir = abs_from(repo_root, env.get("QUADLET_REDIS_DATA_DIR", "./.podman/gia_redis_data"), "./.podman/gia_redis_data") whatsapp_data_dir = abs_from(repo_root, env.get("QUADLET_WHATSAPP_DATA_DIR", "./.podman/gia_whatsapp_data"), "./.podman/gia_whatsapp_data") @@ -56,8 +67,13 @@ def main() -> int: uwsgi_ini = (repo_dir / "docker" / "uwsgi.ini").resolve() redis_conf = (repo_dir / "docker" / "redis.conf").resolve() - for p in (redis_data_dir, whatsapp_data_dir, vrun_dir, signal_cli_dir): + for p in (redis_data_dir, whatsapp_data_dir, sqlite_data_dir, vrun_dir, signal_cli_dir): p.mkdir(parents=True, exist_ok=True) + sqlite_data_dir.chmod(0o777) + if app_db_file.resolve() != host_db_file and app_db_file.exists() and not host_db_file.exists(): + shutil.copy2(app_db_file, host_db_file) + host_db_file.touch(exist_ok=True) + host_db_file.chmod(0o666) out_dir = Path(args.output_dir).expanduser().resolve() out_dir.mkdir(parents=True, exist_ok=True) @@ -143,10 +159,12 @@ WantedBy=gia.target "Image=localhost/xf/gia:prod", f"ContainerName={container_name}", "Pod=gia.pod", + f"User={host_uid}:{host_gid}", f"EnvironmentFile={env_file}", "Environment=SIGNAL_HTTP_URL=http://127.0.0.1:8080", + f"Environment=APP_DATABASE_PATH={app_db_path_in_container}", f"Volume={repo_dir}:/code", - f"Volume={app_db_file}:/conf/db.sqlite3", + f"Volume={sqlite_data_dir}:/conf", f"Volume={vrun_dir}:/var/run", ] if include_uwsgi: diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100755 index 0000000..e69de29