From fd4cecee05c45c63c804f9db8fc697c646edfabf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 29 Nov 2022 07:20:21 +0000 Subject: [PATCH] Switch to UWSGI and improve Docker definitions --- Dockerfile | 28 ++++++++++ Makefile | 20 +++++++ docker-compose.yml | 88 +++++++++++++++++++++++-------- docker/Dockerfile | 18 ------- docker/docker-compose.prod.yml | 76 -------------------------- docker/nginx/conf.d/dev.conf | 23 ++++++++ docker/nginx/conf.d/uwsgi.conf | 24 +++++++++ docker/prod/Dockerfile | 21 -------- docker/prod/requirements.prod.txt | 19 ------- docker/requirements.dev.txt | 18 ------- docker/{prod => }/uwsgi.ini | 5 +- requirements.txt | 1 + stack.env | 10 ++-- 13 files changed, 171 insertions(+), 180 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile delete mode 100644 docker/Dockerfile delete mode 100644 docker/docker-compose.prod.yml create mode 100644 docker/nginx/conf.d/dev.conf create mode 100644 docker/nginx/conf.d/uwsgi.conf delete mode 100644 docker/prod/Dockerfile delete mode 100644 docker/prod/requirements.prod.txt delete mode 100644 docker/requirements.dev.txt rename docker/{prod => }/uwsgi.ini (84%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..439fb21 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 +FROM python:3 +ARG OPERATION + +RUN useradd -d /code pathogen +RUN mkdir -p /code +RUN chown -R pathogen:pathogen /code + +RUN mkdir -p /conf/static +RUN chown -R pathogen:pathogen /conf + +RUN mkdir /venv +RUN chown pathogen:pathogen /venv + +USER pathogen +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +WORKDIR /code +COPY requirements.txt /code/ +RUN python -m venv /venv +RUN . /venv/bin/activate && pip install -r requirements.txt + +# CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini + +CMD if [ "$OPERATION" = "uwsgi" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi + +# CMD . /venv/bin/activate && uvicorn --reload --reload-include *.html --workers 2 --uds /var/run/socks/app.sock app.asgi:application +# CMD . /venv/bin/activate && gunicorn -b 0.0.0.0:8000 --reload app.asgi:application -k uvicorn.workers.UvicornWorker \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5b10324 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +run: + docker-compose --env-file=stack.env up -d + +build: + docker-compose --env-file=stack.env build + +stop: + docker-compose --env-file=stack.env down + +log: + docker-compose --env-file=stack.env logs -f + +migrate: + docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py migrate" + +makemigrations: + docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py makemigrations" + +auth: + docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py createsuperuser" diff --git a/docker-compose.yml b/docker-compose.yml index e46123d..04830f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,30 @@ -version: "2" - +version: "2.2" + services: app: image: pathogen/neptune:latest container_name: neptune - build: ./docker + build: + context: . + args: + OPERATION: ${OPERATION} volumes: - ${PORTAINER_GIT_DIR}:/code - - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py - - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3 - ports: - - "${NEPTUNE_PORT}:8000" + - ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini + - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py + - ${APP_DATABASE_FILE}:/code/db.sqlite3 + - neptune_static:${STATIC_ROOT} env_file: - - .env + - stack.env volumes_from: - - tmp + - tmp depends_on: redis: condition: service_healthy migration: condition: service_started + collectstatic: + condition: service_started networks: - default - pathogen @@ -28,25 +33,63 @@ services: migration: image: pathogen/neptune:latest container_name: migration_neptune + build: + context: . + args: + OPERATION: ${OPERATION} command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput' volumes: - ${PORTAINER_GIT_DIR}:/code - - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py - - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3 + - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py + - ${APP_DATABASE_FILE}:/code/db.sqlite3 + - neptune_static:${STATIC_ROOT} volumes_from: - tmp depends_on: redis: condition: service_healthy - # pyroscope: - # image: pyroscope/pyroscope - # environment: - # - PYROSCOPE_LOG_LEVEL=debug - # ports: - # - '4040:4040' - # command: - # - 'server' + collectstatic: + image: pathogen/neptune:latest + container_name: collectstatic_neptune + build: + context: . + args: + OPERATION: ${OPERATION} + command: sh -c '. /venv/bin/activate && python manage.py collectstatic --noinput' + volumes: + - ${PORTAINER_GIT_DIR}:/code + - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py + - ${APP_DATABASE_FILE}:/code/db.sqlite3 + - neptune_static:${STATIC_ROOT} + env_file: + - stack.env + depends_on: + redis: + condition: service_healthy + + nginx: + image: nginx:latest + container_name: nginx_neptune + ports: + - ${APP_PORT}:9999 + ulimits: + nproc: 65535 + nofile: + soft: 65535 + hard: 65535 + volumes: + - ${PORTAINER_GIT_DIR}:/code + - ${PORTAINER_GIT_DIR}/docker/nginx/conf.d/${OPERATION}.conf:/etc/nginx/conf.d/default.conf + - neptune_static:${STATIC_ROOT} + volumes_from: + - tmp + networks: + - default + - pathogen + depends_on: + app: + condition: service_started tmp: image: busybox @@ -65,7 +108,7 @@ services: soft: 65535 hard: 65535 volumes: - - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf + - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf volumes_from: - tmp healthcheck: @@ -80,4 +123,7 @@ networks: pathogen: external: true elastic: - external: true \ No newline at end of file + external: true + +volumes: + neptune_static: {} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 29cec43..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM python:3 - -RUN useradd -d /code pathogen -RUN mkdir /code -RUN chown pathogen:pathogen /code - -RUN mkdir /venv -RUN chown pathogen:pathogen /venv - -USER pathogen -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -WORKDIR /code -COPY requirements.dev.txt /code/ -RUN python -m venv /venv -RUN . /venv/bin/activate && pip install -r requirements.dev.txt -CMD . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000 \ No newline at end of file diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml deleted file mode 100644 index 1c31e49..0000000 --- a/docker/docker-compose.prod.yml +++ /dev/null @@ -1,76 +0,0 @@ -version: "2.2" - -services: - app: - image: pathogen/neptune:latest - container_name: neptune - build: ./docker/prod - volumes: - - ${PORTAINER_GIT_DIR}:/code - - ${PORTAINER_GIT_DIR}/docker/prod/uwsgi.ini:/conf/uwsgi.ini - - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py - - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3 - ports: - - "${NEPTUNE_PORT}:8000" # uwsgi socket - env_file: - - ../stack.env - volumes_from: - - tmp - depends_on: - redis: - condition: service_healthy - migration: - condition: service_started - networks: - - default - - pathogen - - elastic - - migration: - image: pathogen/neptune:latest - container_name: migration_neptune - build: ./docker/prod - command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput' - volumes: - - ${PORTAINER_GIT_DIR}:/code - - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py - - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3 - volumes_from: - - tmp - depends_on: - redis: - condition: service_healthy - - tmp: - image: busybox - container_name: tmp_neptune - command: chmod -R 777 /var/run/socks - volumes: - - /var/run/socks - - redis: - image: redis - container_name: redis_neptune - command: redis-server /etc/redis.conf - ulimits: - nproc: 65535 - nofile: - soft: 65535 - hard: 65535 - volumes: - - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf - volumes_from: - - tmp - healthcheck: - test: "redis-cli -s /var/run/socks/redis.sock ping" - interval: 2s - timeout: 2s - retries: 15 - -networks: - default: - driver: bridge - pathogen: - external: true - elastic: - external: true \ No newline at end of file diff --git a/docker/nginx/conf.d/dev.conf b/docker/nginx/conf.d/dev.conf new file mode 100644 index 0000000..bd3ab31 --- /dev/null +++ b/docker/nginx/conf.d/dev.conf @@ -0,0 +1,23 @@ +upstream django { + #server app:8000; + #server unix:///var/run/socks/app.sock; + server app:8000; +} + +server { + listen 9999; + + location = /favicon.ico { access_log off; log_not_found off; } + + location /static/ { + root /conf; + } + + location / { + proxy_pass http://django; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + } + +} \ No newline at end of file diff --git a/docker/nginx/conf.d/uwsgi.conf b/docker/nginx/conf.d/uwsgi.conf new file mode 100644 index 0000000..1e19ae9 --- /dev/null +++ b/docker/nginx/conf.d/uwsgi.conf @@ -0,0 +1,24 @@ +upstream django { + server app:8000; + #server unix:///var/run/socks/app.sock; +} + +server { + listen 9999; + + location = /favicon.ico { access_log off; log_not_found off; } + + location /static/ { + root /conf; + } + + location / { + include /etc/nginx/uwsgi_params; # the uwsgi_params file you installed + uwsgi_pass django; + uwsgi_param Host $host; + uwsgi_param X-Real-IP $remote_addr; + uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for; + uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto; + } + +} \ No newline at end of file diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile deleted file mode 100644 index c8386d1..0000000 --- a/docker/prod/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM python:3 - -RUN useradd -d /code pathogen -RUN mkdir /code -RUN chown pathogen:pathogen /code - -RUN mkdir /conf -RUN chown pathogen:pathogen /conf - -RUN mkdir /venv -RUN chown pathogen:pathogen /venv - -USER pathogen -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -WORKDIR /code -COPY requirements.prod.txt /code/ -RUN python -m venv /venv -RUN . /venv/bin/activate && pip install -r requirements.prod.txt -CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini \ No newline at end of file diff --git a/docker/prod/requirements.prod.txt b/docker/prod/requirements.prod.txt deleted file mode 100644 index 7614fa7..0000000 --- a/docker/prod/requirements.prod.txt +++ /dev/null @@ -1,19 +0,0 @@ -wheel -django -django-crispy-forms -crispy-bulma -elasticsearch -stripe -django-rest-framework -numpy -uwsgi -django-tables2 -django-tables2-bulma-template -django-htmx -cryptography -siphashc -redis -sortedcontainers -django-debug-toolbar -django-debug-toolbar-template-profiler -orjson diff --git a/docker/requirements.dev.txt b/docker/requirements.dev.txt deleted file mode 100644 index 4471d43..0000000 --- a/docker/requirements.dev.txt +++ /dev/null @@ -1,18 +0,0 @@ -wheel -django -django-crispy-forms -crispy-bulma -elasticsearch -stripe -django-rest-framework -numpy -django-tables2 -django-tables2-bulma-template -django-htmx -cryptography -siphashc -redis -sortedcontainers -django-debug-toolbar -django-debug-toolbar-template-profiler -orjson diff --git a/docker/prod/uwsgi.ini b/docker/uwsgi.ini similarity index 84% rename from docker/prod/uwsgi.ini rename to docker/uwsgi.ini index d70a784..4f952d0 100644 --- a/docker/prod/uwsgi.ini +++ b/docker/uwsgi.ini @@ -5,9 +5,8 @@ env=DJANGO_SETTINGS_MODULE=app.settings master=1 pidfile=/tmp/project-master.pid socket=0.0.0.0:8000 -processes=5 harakiri=20 -max-requests=5000 +max-requests=100000 vacuum=1 home=/venv - +processes=12 diff --git a/requirements.txt b/requirements.txt index a3b253b..e3e698a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ wheel +uwsgi django pre-commit django-crispy-forms diff --git a/stack.env b/stack.env index faddad4..ec82cbd 100644 --- a/stack.env +++ b/stack.env @@ -1,4 +1,6 @@ -NEPTUNE_PORT=5000 -PORTAINER_GIT_DIR=.. -NEPTUNE_LOCAL_SETTINGS=../app/local_settings.py -NEPTUNE_DATABASE_FILE=../db.sqlite3 \ No newline at end of file +APP_PORT=5000 +PORTAINER_GIT_DIR=. +APP_LOCAL_SETTINGS=./app/local_settings.py +APP_DATABASE_FILE=./db.sqlite3 +STATIC_ROOT=/conf/static +OPERATION=uwsgi \ No newline at end of file