From aaacaed22082df3ec338b1bc795ebe3013c450c8 Mon Sep 17 00:00:00 2001 From: jaseg Date: Sun, 29 Sep 2019 16:28:11 +0200 Subject: deploy: add pogojig --- gerboweb/deploy/bootstrap_arch_container.yml | 4 +- gerboweb/deploy/checkouts/pogojig | 1 + gerboweb/deploy/nginx.conf | 128 +++++++++++++++++++++-- gerboweb/deploy/playbook.yml | 11 ++ gerboweb/deploy/pogojig-job-processor.service.j2 | 9 ++ gerboweb/deploy/pogojig.cfg.j2 | 4 + gerboweb/deploy/pogojig_flask_secret.txt | 1 + gerboweb/deploy/pogojig_generate.sh.j2 | 25 +++++ gerboweb/deploy/setup_openjscad.yml | 9 ++ gerboweb/deploy/setup_pogojig.yml | 118 +++++++++++++++++++++ gerboweb/deploy/setup_tracespace.yml | 9 ++ gerboweb/deploy/setup_webserver.yml | 25 +++-- gerboweb/deploy/tmpfiles-pogojig.conf.j2 | 1 + gerboweb/deploy/uwsgi-pogojig.ini | 10 ++ 14 files changed, 333 insertions(+), 22 deletions(-) create mode 160000 gerboweb/deploy/checkouts/pogojig create mode 100644 gerboweb/deploy/pogojig-job-processor.service.j2 create mode 100644 gerboweb/deploy/pogojig.cfg.j2 create mode 100644 gerboweb/deploy/pogojig_flask_secret.txt create mode 100755 gerboweb/deploy/pogojig_generate.sh.j2 create mode 100644 gerboweb/deploy/setup_openjscad.yml create mode 100644 gerboweb/deploy/setup_pogojig.yml create mode 100644 gerboweb/deploy/setup_tracespace.yml create mode 100644 gerboweb/deploy/tmpfiles-pogojig.conf.j2 create mode 100644 gerboweb/deploy/uwsgi-pogojig.ini (limited to 'gerboweb/deploy') diff --git a/gerboweb/deploy/bootstrap_arch_container.yml b/gerboweb/deploy/bootstrap_arch_container.yml index 4126bbd..11bbf3d 100644 --- a/gerboweb/deploy/bootstrap_arch_container.yml +++ b/gerboweb/deploy/bootstrap_arch_container.yml @@ -13,9 +13,9 @@ - name: Download arch bootstrap image get_url: - url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz + url: http://mirror.rackspace.com/archlinux/iso/2019.09.01/archlinux-bootstrap-2019.09.01-x86_64.tar.gz dest: /tmp/arch-bootstrap.tar.xz - checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756 + checksum: sha256:9fc9f178db6f5c188be8884c0abf10c69418e7cd38a4389e866fac5d9961297d when: create_container is changed - name: Create container image filesystem diff --git a/gerboweb/deploy/checkouts/pogojig b/gerboweb/deploy/checkouts/pogojig new file mode 160000 index 0000000..04b34f7 --- /dev/null +++ b/gerboweb/deploy/checkouts/pogojig @@ -0,0 +1 @@ +Subproject commit 04b34f73c1e0023ce4abdb9f1c9f0d9175247624 diff --git a/gerboweb/deploy/nginx.conf b/gerboweb/deploy/nginx.conf index b6f13c0..6cf28cf 100644 --- a/gerboweb/deploy/nginx.conf +++ b/gerboweb/deploy/nginx.conf @@ -36,9 +36,9 @@ http { include /etc/nginx/conf.d/*.conf; server { - listen 80 default_server; - listen [::]:80 default_server; - server_name gerbolyze.jaseg.net; + listen 80; + listen [::]:80; + server_name .jaseg.net; return 301 https://$host$request_uri; } @@ -84,13 +84,6 @@ http { } } - server { - listen 80; - listen [::]:80; - server_name blog.jaseg.net blog.jaseg.net; - return 301 https://$host$request_uri; - } - server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -136,5 +129,120 @@ http { } } + server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name pogojig.jaseg.net; + root /usr/share/nginx/html; + + ssl_certificate "/etc/letsencrypt/live/pogojig.jaseg.net/fullchain.pem"; + ssl_certificate_key "/etc/letsencrypt/live/pogojig.jaseg.net/privkey.pem"; + ssl_dhparam "/etc/letsencrypt/ssl-dhparams.pem"; + include /etc/letsencrypt/options-ssl-nginx.conf; + + ssl_stapling on; + ssl_stapling_verify on; + + resolver 67.207.67.2 67.207.67.3 valid=300s; + resolver_timeout 10s; + + add_header Strict-Transport-Security "max-age=86400"; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location ^~ /pogospace/ { + root /var/lib/pogojig/pogospace; + } + + location / { + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/pogojig.socket; + } + + error_page 404 /404.html; + location = /40x.html { + root /usr/share/nginx/html; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } + + server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name tracespace.jaseg.net; + root /usr/share/nginx/html; + + ssl_certificate "/etc/letsencrypt/live/tracespace.jaseg.net/fullchain.pem"; + ssl_certificate_key "/etc/letsencrypt/live/tracespace.jaseg.net/privkey.pem"; + ssl_dhparam "/etc/letsencrypt/ssl-dhparams.pem"; + include /etc/letsencrypt/options-ssl-nginx.conf; + + ssl_stapling on; + ssl_stapling_verify on; + + resolver 67.207.67.2 67.207.67.3 valid=300s; + resolver_timeout 10s; + + add_header Strict-Transport-Security "max-age=86400"; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + root /var/www/tracespace.jaseg.net; + } + + error_page 404 /404.html; + location = /40x.html { + root /usr/share/nginx/html; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } + + server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name openjscad.jaseg.net; + root /usr/share/nginx/html; + + ssl_certificate "/etc/letsencrypt/live/openjscad.jaseg.net/fullchain.pem"; + ssl_certificate_key "/etc/letsencrypt/live/openjscad.jaseg.net/privkey.pem"; + ssl_dhparam "/etc/letsencrypt/ssl-dhparams.pem"; + include /etc/letsencrypt/options-ssl-nginx.conf; + + ssl_stapling on; + ssl_stapling_verify on; + + resolver 67.207.67.2 67.207.67.3 valid=300s; + resolver_timeout 10s; + + add_header Strict-Transport-Security "max-age=86400"; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + root /var/www/openjscad.jaseg.net; + } + + error_page 404 /404.html; + location = /40x.html { + root /usr/share/nginx/html; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } } diff --git a/gerboweb/deploy/playbook.yml b/gerboweb/deploy/playbook.yml index afea884..6d558d3 100644 --- a/gerboweb/deploy/playbook.yml +++ b/gerboweb/deploy/playbook.yml @@ -48,6 +48,7 @@ containers: - gerboweb - clippy + - pogojig - name: Setup web server include_tasks: setup_webserver.yml @@ -60,3 +61,13 @@ - name: Setup secure download include_tasks: setup_secure_download.yml + + - name: Setup tracespace + include_tasks: setup_tracespace.yml + + - name: Setup openjscad + include_tasks: setup_openjscad.yml + + - name: Setup pogojig + include_tasks: setup_pogojig.yml + diff --git a/gerboweb/deploy/pogojig-job-processor.service.j2 b/gerboweb/deploy/pogojig-job-processor.service.j2 new file mode 100644 index 0000000..5ca9a8b --- /dev/null +++ b/gerboweb/deploy/pogojig-job-processor.service.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Pogojig render job processor + +[Service] +WorkingDirectory=/var/lib/pogojig +ExecStart=/usr/bin/python3 job_processor.py {{pogojig_cache}}/job_queue.sqlite3 + +[Install] +WantedBy=uwsgi-app@pogojig.service diff --git a/gerboweb/deploy/pogojig.cfg.j2 b/gerboweb/deploy/pogojig.cfg.j2 new file mode 100644 index 0000000..3dd7160 --- /dev/null +++ b/gerboweb/deploy/pogojig.cfg.j2 @@ -0,0 +1,4 @@ +MAX_CONTENT_LENGTH=10000000 +SECRET_KEY="{{lookup('password', 'pogojig_flask_secret.txt length=32')}}" +UPLOAD_PATH="{{pogojig_cache}}/upload" +JOB_QUEUE_DB="{{pogojig_cache}}/job_queue.sqlite3" diff --git a/gerboweb/deploy/pogojig_flask_secret.txt b/gerboweb/deploy/pogojig_flask_secret.txt new file mode 100644 index 0000000..fc1e245 --- /dev/null +++ b/gerboweb/deploy/pogojig_flask_secret.txt @@ -0,0 +1 @@ +RhshDKyRVcg.5r4ulH:0f8Vhut0HoFS3 diff --git a/gerboweb/deploy/pogojig_generate.sh.j2 b/gerboweb/deploy/pogojig_generate.sh.j2 new file mode 100755 index 0000000..c1cc023 --- /dev/null +++ b/gerboweb/deploy/pogojig_generate.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/sh + +[ $# != 1 ] && exit 1 +ID=$1 +egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2 + +systemd-nspawn \ + -D {{pogojig_root}} \ + -x --bind={{pogojig_cache}}/upload/$ID:/mnt \ + /bin/sh -c "set -euo pipefail +cd /mnt + +date; echo 'Cleaning up previous output' +rm -rf pcb_shape.dxf jig.stl kicad kicad.zip sources.zip + +date; echo 'Rendering' +cp -r /var/lib/pogojig_renderer sources +cp input.svg sources/ +make -C sources + +date; echo 'Packing source bundle' +cp -r sources/out/pcb_shape.dxf sources/out/jig.stl sources/out/kicad ./ +zip -r sources.zip sources +zip -r kicad.zip kicad +rm -rf sources" diff --git a/gerboweb/deploy/setup_openjscad.yml b/gerboweb/deploy/setup_openjscad.yml new file mode 100644 index 0000000..dea4ad2 --- /dev/null +++ b/gerboweb/deploy/setup_openjscad.yml @@ -0,0 +1,9 @@ +--- +- name: Copy openjscad webapp sources + synchronize: + # FIXME: make this path configurable + src: ~/openjscad_dist/ + dest: /var/www/openjscad.jaseg.net/ + group: no + owner: no + diff --git a/gerboweb/deploy/setup_pogojig.yml b/gerboweb/deploy/setup_pogojig.yml new file mode 100644 index 0000000..8a4c827 --- /dev/null +++ b/gerboweb/deploy/setup_pogojig.yml @@ -0,0 +1,118 @@ +--- +- name: Set local facts + set_fact: + pogojig_cache: /var/cache/pogojig + +- name: Copy render script + template: + src: pogojig_generate.sh.j2 + dest: /usr/local/sbin/pogojig_generate.sh + mode: ug+x + +- name: Install packages into pogojig container + shell: arch-chroot "{{pogojig_root}}" pacman -Syu --noconfirm python3 python-pip imagemagick unzip zip openscad inkscape make python-lxml xorg-server-xvfb + +- name: Install python dependencies into pogojig container + shell: arch-chroot "{{pogojig_root}}" pip install -U --upgrade-strategy=eager ezdxf xvfbwrapper + +- name: Install pogojig + synchronize: + # FIXME: make this path configurable + src: checkouts/pogojig/renderer/ + dest: "{{pogojig_root}}/var/lib/pogojig_renderer" + group: no + +- name: Copy webapp sources + synchronize: + # FIXME: make this path configurable + src: checkouts/pogojig/webapp/ + dest: /var/lib/pogojig + group: no + owner: no + +- name: Create web home for modified tracespace + file: + path: /var/lib/pogojig/pogospace + state: directory + owner: nginx + group: nginx + mode: 0550 + +- name: Unpack modified tracespace sources + unarchive: + src: resource/pogojig-tracespace.tar.gz + dest: /var/lib/pogojig/pogospace + extra_opts: [--strip-components=1] + owner: nginx + group: nginx + +- name: Create uwsgi worker user and group + user: + name: uwsgi-pogojig + create_home: no + group: uwsgi + password: '!' + shell: /sbin/nologin + system: yes + +- name: Template webapp config + template: + src: pogojig.cfg.j2 + dest: /var/lib/pogojig/pogojig_prod.cfg + owner: uwsgi-pogojig + group: root + mode: 0660 + +- name: Copy uwsgi config + copy: + src: uwsgi-pogojig.ini + dest: /etc/uwsgi.d/pogojig.ini + owner: uwsgi-pogojig + group: uwsgi + mode: 440 + +- name: Copy job processor systemd service config + template: + src: pogojig-job-processor.service.j2 + dest: /etc/systemd/system/pogojig-job-processor.service + +- name: Enable uwsgi systemd socket + systemd: + daemon-reload: yes + name: uwsgi-app@pogojig.socket + enabled: yes + +# FIXME the socket doesn't seem to work properly +- name: Enable uwsgi systemd service + systemd: + daemon-reload: yes + name: uwsgi-app@pogojig.service + enabled: yes + +- name: Copy pogojig cache dir tmpfiles.d config + template: + src: tmpfiles-pogojig.conf.j2 + dest: /etc/tmpfiles.d/pogojig.conf + owner: root + group: root + mode: 0644 + register: pogojig_tmpfiles_config + +- name: Kick systemd tmpfiles service to create cache dir + command: systemd-tmpfiles --create + when: pogojig_tmpfiles_config is changed + +- name: Create job queue db + file: + path: "{{pogojig_cache}}/job_queue.sqlite3" + owner: root + group: uwsgi + mode: 0660 + state: touch + +- name: Enable and launch job processor + systemd: + name: pogojig-job-processor.service + enabled: yes + state: restarted + diff --git a/gerboweb/deploy/setup_tracespace.yml b/gerboweb/deploy/setup_tracespace.yml new file mode 100644 index 0000000..2975967 --- /dev/null +++ b/gerboweb/deploy/setup_tracespace.yml @@ -0,0 +1,9 @@ +--- +- name: Copy tracespace webapp sources + synchronize: + # FIXME: make this path configurable + src: ~/tracespace_dist/ + dest: /var/www/tracespace.jaseg.net/ + group: no + owner: no + diff --git a/gerboweb/deploy/setup_webserver.yml b/gerboweb/deploy/setup_webserver.yml index ea821d8..671a699 100644 --- a/gerboweb/deploy/setup_webserver.yml +++ b/gerboweb/deploy/setup_webserver.yml @@ -9,13 +9,17 @@ groups: uwsgi append: yes -- name: Create blog.jaseg.net content dir +- name: Create subdomain content dirs file: - path: /var/www/blog.jaseg.net + path: /var/www/{{item}} state: directory owner: nginx group: nginx mode: 0550 + loop: + - blog.jaseg.net + - tracespace.jaseg.net + - openjscad.jaseg.net - name: Copy uwsgi systemd socket config copy: @@ -38,15 +42,16 @@ enabled: yes state: restarted -- name: Create letsencrypt certificate for gerbolyze.jaseg.net - command: certbot --nginx certonly -d gerbolyze.jaseg.net -n --agree-tos --email gerboweb@jaseg.net +- name: Create subdomain letsencrypt certificates + command: certbot --nginx certonly -d {{item}} -n --agree-tos --email {{item}}-letsencrypt@jaseg.net args: - creates: /etc/letsencrypt/live/gerbolyze.jaseg.net/fullchain.pem - -- name: Create letsencrypt certificate for blog.jaseg.net - command: certbot --nginx certonly -d blog.jaseg.net -n --agree-tos --email blog@jaseg.net - args: - creates: /etc/letsencrypt/live/blog.jaseg.net/fullchain.pem + creates: /etc/letsencrypt/live/{{item}}/fullchain.pem + loop: + - blog.jaseg.net + - gerbolyze.jaseg.net + - tracespace.jaseg.net + - openjscad.jaseg.net + - pogojig.jaseg.net - name: Copy final nginx config copy: diff --git a/gerboweb/deploy/tmpfiles-pogojig.conf.j2 b/gerboweb/deploy/tmpfiles-pogojig.conf.j2 new file mode 100644 index 0000000..4e9fef1 --- /dev/null +++ b/gerboweb/deploy/tmpfiles-pogojig.conf.j2 @@ -0,0 +1 @@ +d {{pogojig_cache}} 770 uwsgi-pogojig uwsgi 2d diff --git a/gerboweb/deploy/uwsgi-pogojig.ini b/gerboweb/deploy/uwsgi-pogojig.ini new file mode 100644 index 0000000..003702d --- /dev/null +++ b/gerboweb/deploy/uwsgi-pogojig.ini @@ -0,0 +1,10 @@ +[uwsgi] +master = True +cheap = True +die-on-idle = False +manage-script-name = True +plugins = python3 +chdir = /var/lib/pogojig +mount = /=pogojig:app +env = POGOJIG_SETTINGS=pogojig_prod.cfg + -- cgit