From 3ba932209fc33206a28483d07fb0e108d1b16a2b Mon Sep 17 00:00:00 2001 From: jaseg Date: Mon, 4 Jul 2022 23:57:28 +0200 Subject: gerboweb: add deployment stuff --- gerboweb/Containerfile.develop | 12 +++ gerboweb/ansible/gerboweb-job-processor.service.j2 | 9 +++ gerboweb/ansible/gerboweb.cfg.j2 | 4 + gerboweb/ansible/gerboweb.yml | 86 ++++++++++++++++++++++ gerboweb/ansible/render.sh.j2 | 17 +++++ gerboweb/ansible/tmpfiles-gerboweb.conf.j2 | 1 + gerboweb/ansible/uwsgi-gerboweb.ini | 10 +++ gerboweb/ansible/vector.sh.j2 | 18 +++++ gerboweb/develop-startup.sh | 22 ++++++ gerboweb/gerboweb-develop.cfg | 4 + gerboweb/run-develop.sh | 7 ++ 11 files changed, 190 insertions(+) create mode 100644 gerboweb/Containerfile.develop create mode 100644 gerboweb/ansible/gerboweb-job-processor.service.j2 create mode 100644 gerboweb/ansible/gerboweb.cfg.j2 create mode 100644 gerboweb/ansible/gerboweb.yml create mode 100755 gerboweb/ansible/render.sh.j2 create mode 100644 gerboweb/ansible/tmpfiles-gerboweb.conf.j2 create mode 100644 gerboweb/ansible/uwsgi-gerboweb.ini create mode 100755 gerboweb/ansible/vector.sh.j2 create mode 100755 gerboweb/develop-startup.sh create mode 100644 gerboweb/gerboweb-develop.cfg create mode 100755 gerboweb/run-develop.sh (limited to 'gerboweb') diff --git a/gerboweb/Containerfile.develop b/gerboweb/Containerfile.develop new file mode 100644 index 0000000..92f9558 --- /dev/null +++ b/gerboweb/Containerfile.develop @@ -0,0 +1,12 @@ +FROM docker.io/archlinux:latest +MAINTAINER gerbolyze@jaseg.de +RUN pacman --noconfirm -Syu +RUN pacman --noconfirm -Sy pugixml opencv pango cairo git python make clang rustup cargo python-pip base-devel gerbv rsync tmux +RUN rustup install stable +RUN rustup default stable +RUN cargo install usvg resvg +RUN mkdir /app /gerbolyze +RUN python3 -m pip install pip==21.3.1 +RUN python3 -m pip install flask numpy lxml wasmtime svg_flatten_wasi resvg_wasi flask_wtf +COPY develop-startup.sh /app/ +ENTRYPOINT /app/develop-startup.sh diff --git a/gerboweb/ansible/gerboweb-job-processor.service.j2 b/gerboweb/ansible/gerboweb-job-processor.service.j2 new file mode 100644 index 0000000..517d8b8 --- /dev/null +++ b/gerboweb/ansible/gerboweb-job-processor.service.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Gerboweb gerber job processor + +[Service] +WorkingDirectory=/var/lib/gerboweb +ExecStart=/usr/bin/python3 job_processor.py {{gerboweb_cache}}/job_queue.sqlite3 + +[Install] +WantedBy=uwsgi-app@gerboweb.service diff --git a/gerboweb/ansible/gerboweb.cfg.j2 b/gerboweb/ansible/gerboweb.cfg.j2 new file mode 100644 index 0000000..eec4dcb --- /dev/null +++ b/gerboweb/ansible/gerboweb.cfg.j2 @@ -0,0 +1,4 @@ +MAX_CONTENT_LENGTH=50000000 +SECRET_KEY="{{lookup('password', 'gerboweb_flask_secret.txt length=32')}}" +UPLOAD_PATH="{{gerboweb_cache}}/upload" +JOB_QUEUE_DB="{{gerboweb_cache}}/job_queue.sqlite3" diff --git a/gerboweb/ansible/gerboweb.yml b/gerboweb/ansible/gerboweb.yml new file mode 100644 index 0000000..590d01b --- /dev/null +++ b/gerboweb/ansible/gerboweb.yml @@ -0,0 +1,86 @@ +--- +- name: Setup gerboweb + hosts: all + tasks: + - name: Set local facts + set_fact: + gerboweb_cache: /var/cache/gerboweb + + - name: Install packages into gerbolyze container + pacman: + name: python3,python-numpy,python-pip,imagemagick,pugixml,git,python,make,clang,rustup,cargo + state: present + + - name: Setup usvg + shell: cargo install usvg resvg + args: + creates: /usr/bin/usvg + + - name: Copy webapp sources + git: + repo: ../.. + dest: /var/lib/gerboweb + + - name: Create uwsgi worker user and group + user: + name: uwsgi-gerboweb + create_home: no + group: uwsgi + password: '!' + shell: /sbin/nologin + system: yes + + - name: Template webapp config + template: + src: gerboweb.cfg.j2 + dest: /var/lib/gerboweb/gerboweb_prod.cfg + owner: uwsgi-gerboweb + group: root + mode: 0660 + + - name: Copy uwsgi config + copy: + src: uwsgi-gerboweb.ini + dest: /etc/uwsgi.d/gerboweb.ini + owner: uwsgi-gerboweb + group: uwsgi + mode: 0440 + + - name: Copy job processor systemd service config + template: + src: gerboweb-job-processor.service.j2 + dest: /etc/systemd/system/gerboweb-job-processor.service + + - name: Enable uwsgi systemd socket + systemd: + daemon-reload: yes + name: uwsgi-app@gerboweb.socket + enabled: yes + + - name: Copy gerboweb cache dir tmpfiles.d config + template: + src: tmpfiles-gerboweb.conf.j2 + dest: /etc/tmpfiles.d/gerboweb.conf + owner: root + group: root + mode: 0644 + register: tmpfiles_config + + - name: Kick systemd tmpfiles service to create cache dir + command: systemd-tmpfiles --create + when: tmpfiles_config is changed + + - name: Create job queue db + file: + path: "{{gerboweb_cache}}/job_queue.sqlite3" + owner: root + group: uwsgi + mode: 0660 + state: touch + + - name: Enable and launch job processor + systemd: + name: gerboweb-job-processor.service + enabled: yes + state: restarted + diff --git a/gerboweb/ansible/render.sh.j2 b/gerboweb/ansible/render.sh.j2 new file mode 100755 index 0000000..beed8f1 --- /dev/null +++ b/gerboweb/ansible/render.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/sh + +[ $# != 1 ] && exit 1 +ID=$1 +egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2 + +systemd-nspawn \ + -D {{gerboweb_root}} \ + -x --bind={{gerboweb_cache}}/upload/$ID:/mnt \ + /bin/sh -c "set -euo pipefail +unzip -j -d /tmp/gerber /mnt/gerber.zip +rm -f /mnt/template_top.svg /mnt/template_bottom.svg /mnt/template_top.preview.png /mnt/template_bottom.preview.png +date; echo 'Rendering' +gerbolyze template --top /mnt/template_top.svg --bottom /mnt/template_bottom.svg /tmp/gerber +date; echo 'Scaling down' +convert /mnt/template_top.svg -resize 500x500 -negate -brightness-contrast 30x30 -colorspace gray /mnt/template_top.preview.png +convert /mnt/template_bottom.svg -resize 500x500 -negate -brightness-contrast 30x30 -colorspace gray /mnt/template_bottom.preview.png diff --git a/gerboweb/ansible/tmpfiles-gerboweb.conf.j2 b/gerboweb/ansible/tmpfiles-gerboweb.conf.j2 new file mode 100644 index 0000000..18469b7 --- /dev/null +++ b/gerboweb/ansible/tmpfiles-gerboweb.conf.j2 @@ -0,0 +1 @@ +d {{gerboweb_cache}} 770 uwsgi-gerboweb uwsgi 2d diff --git a/gerboweb/ansible/uwsgi-gerboweb.ini b/gerboweb/ansible/uwsgi-gerboweb.ini new file mode 100644 index 0000000..155d01a --- /dev/null +++ b/gerboweb/ansible/uwsgi-gerboweb.ini @@ -0,0 +1,10 @@ +[uwsgi] +master = True +cheap = True +die-on-idle = False +manage-script-name = True +plugins = python3 +chdir = /var/lib/gerboweb +mount = /=gerboweb:app +env = GERBOWEB_SETTINGS=gerboweb_prod.cfg + diff --git a/gerboweb/ansible/vector.sh.j2 b/gerboweb/ansible/vector.sh.j2 new file mode 100755 index 0000000..b007967 --- /dev/null +++ b/gerboweb/ansible/vector.sh.j2 @@ -0,0 +1,18 @@ +#!/bin/sh + +[ $# != 2 ] && exit 1 +ID=$1 +egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2 +LAYER=$2 +egrep -x -q '^(top|bottom)$'<<<"$LAYER" || exit 2 + +systemd-nspawn \ + -D {{gerboweb_root}} \ + -x --bind={{gerboweb_cache}}/upload/$ID:/mnt \ + /bin/sh -c "set -euo pipefail +cd /tmp +unzip -j -d gerber_in /mnt/gerber.zip +gerbolyze paste "--"$LAYER /mnt/overlay.svg gerber_in gerber +rm -f /mnt/gerber_out.zip +zip -r /mnt/gerber_out.zip gerber" + diff --git a/gerboweb/develop-startup.sh b/gerboweb/develop-startup.sh new file mode 100755 index 0000000..8723ab0 --- /dev/null +++ b/gerboweb/develop-startup.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +cd /gerbolyze/gerbonara +echo "### Setting up gerbonara ###" +# newer pip is buggy and just crashes so we pinned an old version. +# python packaging infrastructure is such an incoherent, buggy mess +# also ignore the running pip as root warning, it's dumb and here we actually want to do just that. +python3 -m pip --disable-pip-version-check install . +cd /gerbolyze +echo "### Setting up gerbolyze ###" +python3 -m pip --disable-pip-version-check install . + +export PATH=$PATH:$HOME/.cargo/bin +cd /gerbolyze/gerboweb +echo "### Launching app ###" +tmux new-session -d -s dev env GERBOWEB_SETTINGS=gerboweb-develop.cfg FLASK_APP=gerboweb.py flask run -h 0.0.0.0 +tmux bind -n C-q kill-session +tmux rename-window gerboweb +tmux split-window -t 0 -v python3 job_processor.py /var/cache/job_queue.sqlite3 +tmux attach diff --git a/gerboweb/gerboweb-develop.cfg b/gerboweb/gerboweb-develop.cfg new file mode 100644 index 0000000..4e039db --- /dev/null +++ b/gerboweb/gerboweb-develop.cfg @@ -0,0 +1,4 @@ +MAX_CONTENT_LENGTH=50000000 +SECRET_KEY="development mode" +UPLOAD_PATH="/var/cache/upload" +JOB_QUEUE_DB="/var/cache/job_queue.sqlite3" diff --git a/gerboweb/run-develop.sh b/gerboweb/run-develop.sh new file mode 100755 index 0000000..f40c18f --- /dev/null +++ b/gerboweb/run-develop.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +cd "$(dirname $0)" +podman build -f Containerfile.develop --tag gerbolyze-develop +podman run -p 127.0.0.1:5000:5000 -v ..:/gerbolyze -ti gerbolyze-develop -- cgit