aboutsummaryrefslogtreecommitdiff
path: root/gerboweb/deploy
diff options
context:
space:
mode:
authorjaseg <code@jaseg.net>2019-04-02 04:36:10 +0900
committerjaseg <git@jaseg.net>2019-04-02 04:36:10 +0900
commitcbb98bcf0ee0e9e123f054377ff68b513efe6bf6 (patch)
treed44deea64134600a6b385c3c4a773aadc20812cf /gerboweb/deploy
parenta846d39bc88e0c03402b20790b04762666bc055f (diff)
downloadgerbolyze-cbb98bcf0ee0e9e123f054377ff68b513efe6bf6.tar.gz
gerbolyze-cbb98bcf0ee0e9e123f054377ff68b513efe6bf6.tar.bz2
gerbolyze-cbb98bcf0ee0e9e123f054377ff68b513efe6bf6.zip
gerboweb: Modularize deployment playbooks a bit
Diffstat (limited to 'gerboweb/deploy')
-rw-r--r--gerboweb/deploy/.gitignore2
-rw-r--r--gerboweb/deploy/bootstrap_arch_container.yml60
-rw-r--r--gerboweb/deploy/gerboweb-job-processor.service.j2 (renamed from gerboweb/deploy/gerboweb-job-processor.service)2
-rw-r--r--gerboweb/deploy/gerboweb.cfg.j24
-rw-r--r--gerboweb/deploy/playbook.yml201
-rwxr-xr-xgerboweb/deploy/render.sh.j2 (renamed from gerboweb/deploy/render.sh)4
-rw-r--r--gerboweb/deploy/setup_containers.yml25
-rw-r--r--gerboweb/deploy/setup_gerboweb.yml95
-rw-r--r--gerboweb/deploy/setup_webserver.yml52
-rw-r--r--gerboweb/deploy/tmpfiles-gerboweb.conf1
-rw-r--r--gerboweb/deploy/tmpfiles-gerboweb.conf.j21
-rw-r--r--gerboweb/deploy/uwsgi-gerboweb.ini2
-rwxr-xr-xgerboweb/deploy/vector.sh.j2 (renamed from gerboweb/deploy/vector.sh)4
13 files changed, 257 insertions, 196 deletions
diff --git a/gerboweb/deploy/.gitignore b/gerboweb/deploy/.gitignore
new file mode 100644
index 0000000..97b80a1
--- /dev/null
+++ b/gerboweb/deploy/.gitignore
@@ -0,0 +1,2 @@
+gerboweb_flask_secret.txt
+playbook.retry
diff --git a/gerboweb/deploy/bootstrap_arch_container.yml b/gerboweb/deploy/bootstrap_arch_container.yml
new file mode 100644
index 0000000..bd534e8
--- /dev/null
+++ b/gerboweb/deploy/bootstrap_arch_container.yml
@@ -0,0 +1,60 @@
+---
+- name: Set local path facts
+ set_fact:
+ image: "/var/cache/containers/{{ container }}.img"
+ root: "/var/cache/containers/{{ container }}_root"
+ "{{container}}_root": "/var/cache/containers/{{ container }}_root"
+
+- name: Create container image file
+ command: truncate -s 4G "{{image}}"
+ args:
+ creates: "{{image}}"
+ register: create_container
+
+- name: Download arch bootstrap image
+ get_url:
+ url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz
+ dest: /tmp/arch-bootstrap.tar.xz
+ checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756
+ when: create_container is changed
+
+- name: Create container image filesystem
+ filesystem:
+ dev: "{{image}}"
+ fstype: btrfs
+
+- name: Create container image fstab entry
+ mount:
+ src: "{{image}}"
+ path: "{{root}}"
+ state: mounted
+ fstype: btrfs
+ opts: loop
+
+- name: Unpack bootstrap image
+ unarchive:
+ remote_src: yes
+ src: /tmp/arch-bootstrap.tar.xz
+ dest: "{{root}}"
+ extra_opts: --strip-components=1
+ creates: "{{root}}/etc"
+
+- name: Copy mirrorlist into container
+ copy:
+ src: mirrorlist
+ dest: "{{root}}/etc/pacman.d/mirrorlist"
+
+- name: Initialize container pacman keyring
+ shell: arch-chroot "{{root}}" pacman-key --init && arch-chroot "{{root}}" pacman-key --populate archlinux
+ args:
+ creates: "{{root}}/etc/pacman.d/gnupg"
+
+- name: Fixup pacman.conf for pacman to work in chroot without its own root fs
+ lineinfile:
+ path: "{{root}}/etc/pacman.conf"
+ regexp: '^CheckSpace'
+ line: '#CheckSpace'
+
+- name: Update container and install software
+ shell: arch-chroot "{{root}}" pacman -Syu --noconfirm
+
diff --git a/gerboweb/deploy/gerboweb-job-processor.service b/gerboweb/deploy/gerboweb-job-processor.service.j2
index 8569317..517d8b8 100644
--- a/gerboweb/deploy/gerboweb-job-processor.service
+++ b/gerboweb/deploy/gerboweb-job-processor.service.j2
@@ -3,7 +3,7 @@ Description=Gerboweb gerber job processor
[Service]
WorkingDirectory=/var/lib/gerboweb
-ExecStart=/usr/bin/python3 job_processor.py /var/cache/gerboweb/job_queue.sqlite3
+ExecStart=/usr/bin/python3 job_processor.py {{gerboweb_cache}}/job_queue.sqlite3
[Install]
WantedBy=uwsgi-app@gerboweb.service
diff --git a/gerboweb/deploy/gerboweb.cfg.j2 b/gerboweb/deploy/gerboweb.cfg.j2
new file mode 100644
index 0000000..994cd08
--- /dev/null
+++ b/gerboweb/deploy/gerboweb.cfg.j2
@@ -0,0 +1,4 @@
+MAX_CONTENT_LENGTH=10000000
+SECRET_KEY="{{lookup('password', 'gerboweb_flask_secret.txt length=32')}}"
+UPLOAD_PATH="{{gerboweb_cache}}/upload"
+JOB_QUEUE_DB="{{gerboweb_cache}}/job_queue.sqlite3"
diff --git a/gerboweb/deploy/playbook.yml b/gerboweb/deploy/playbook.yml
index a0ff505..23544c4 100644
--- a/gerboweb/deploy/playbook.yml
+++ b/gerboweb/deploy/playbook.yml
@@ -7,12 +7,12 @@
- name: Install common admin tools
dnf:
- name: htop,tmux,fish,mosh,neovim
+ name: htop,tmux,fish,mosh,neovim,sqlite
state: latest
- name: Install host requisites
dnf:
- name: btrfs-progs,arch-install-scripts,nginx,uwsgi,python3-flask,python3-flask-wtf,systemd-container,uwsgi-plugin-python3,certbot,python3-certbot-nginx,libselinux-python
+ name: nginx,uwsgi,python3-flask,python3-flask-wtf,uwsgi-plugin-python3,certbot,python3-certbot-nginx,libselinux-python
state: latest
- name: Disable password-based root login
@@ -28,192 +28,15 @@
state: restarted
when: disable_root_pw_ssh is changed
- - name: Create container image file
- command: truncate -s 4G /var/cache/gerbolyze_container.img
- args:
- creates: /var/cache/gerbolyze_container.img
- register: create_container
+ - name: Create containers
+ include_tasks: setup_containers.yml
+ vars:
+ containers:
+ - gerboweb
+ - clippy
- - name: Download arch bootstrap image
- get_url:
- url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz
- dest: /tmp/arch-bootstrap.tar.xz
- checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756
- when: create_container is changed
-
- - name: Create container image filesystem
- filesystem:
- dev: /var/cache/gerbolyze_container.img
- fstype: btrfs
-
- - name: Create container image fstab entry
- mount:
- src: /var/cache/gerbolyze_container.img
- path: /var/cache/gerbolyze_container
- state: mounted
- fstype: btrfs
- opts: loop
-
- - name: Unpack bootstrap image
- unarchive:
- remote_src: yes
- src: /tmp/arch-bootstrap.tar.xz
- dest: /var/cache/gerbolyze_container
- extra_opts: --strip-components=1
- creates: /var/cache/gerbolyze_container/etc
-
- - name: Copy mirrorlist into container
- copy:
- src: mirrorlist
- dest: /var/cache/gerbolyze_container/etc/pacman.d/mirrorlist
-
- - name: Copy render script
- copy:
- src: render.sh
- dest: /usr/local/sbin/gerbolyze_render.sh
- mode: ug+x
-
- - name: Copy vector script
- copy:
- src: vector.sh
- dest: /usr/local/sbin/gerbolyze_vector.sh
- mode: ug+x
-
- - name: Initialize container pacman keyring
- shell: arch-chroot /var/cache/gerbolyze_container pacman-key --init && arch-chroot /var/cache/gerbolyze_container pacman-key --populate archlinux
- args:
- creates: /var/cache/gerbolyze_container/etc/pacman.d/gnupg
-
- - name: Fixup pacman.conf for pacman to work in chroot without its own root fs
- lineinfile:
- path: /var/cache/gerbolyze_container/etc/pacman.conf
- regexp: '^CheckSpace'
- line: '#CheckSpace'
-
- - name: Update container and install software
- shell: arch-chroot /var/cache/gerbolyze_container pacman -Syu --noconfirm python3 opencv hdf5 gtk3 python-numpy python-pip imagemagick unzip zip
-
- # TODO maybe install directly from local git checkout?
- - name: Install gerbolyze
- shell: arch-chroot /var/cache/gerbolyze_container pip install -U --upgrade-strategy=eager gerbolyze
-
- - name: Cleanup bootstrap image
- file:
- path: /tmp/arch-bootstrap.tar.xz
- state: absent
-
- - name: Copy webapp sources
- synchronize:
- # FIXME: make this path configurable
- src: ~/gerbolyze/gerboweb/
- dest: /var/lib/gerboweb/
- group: no
- owner: no
-
- - name: Copy first stage nginx config
- copy:
- src: nginx_nossl.conf
- dest: /etc/nginx/nginx.conf
-
- - name: Create uwsgi worker user and group
- user:
- name: uwsgi-gerboweb
- create_home: no
- group: uwsgi
- password: '!'
- shell: /sbin/nologin
- system: yes
-
- - name: Add nginx user to uwsgi group for access to uwsgi socket
- user:
- name: nginx
- groups: uwsgi
- append: yes
-
- - name: Copy uwsgi config
- copy:
- src: uwsgi-gerboweb.ini
- dest: /etc/uwsgi.d/gerboweb.ini
- owner: uwsgi-gerboweb
- group: uwsgi
- mode: 440
-
- - name: Copy uwsgi systemd socket config
- copy:
- src: uwsgi-app@.socket
- dest: /etc/systemd/system/
-
- - name: Copy uwsgi systemd service config
- copy:
- src: uwsgi-app@.service
- dest: /etc/systemd/system/
-
- - name: Copy job processor systemd service config
- copy:
- src: gerboweb-job-processor.service
- dest: /etc/systemd/system/
-
- - name: Set SELinux to permissive mode # FIXME
- selinux:
- state: permissive
- policy: targeted
-
- - name: Enable uwsgi systemd socket
- systemd:
- daemon-reload: yes
- name: uwsgi-app@gerboweb.socket
- enabled: yes
-
- - name: Copy gerboweb cache dir tmpfiles.d config
- copy:
- src: tmpfiles-gerboweb.conf
- dest: /etc/tmpfiles.d/gerboweb.conf
- owner: root
- group: root
- mode: 0644
- register: tmpfiles_config
-
- - name: Kick systemd tmpfiles service to create cache dir
- command: systemd-tmpfiles --create
- when: tmpfiles_config is changed
-
- - name: Create job queue db
- file:
- path: /var/cache/gerboweb/job_queue.sqlite3
- owner: root
- group: uwsgi
- mode: 0660
- state: touch
-
- - name: Enable and launch job processor
- systemd:
- name: gerboweb-job-processor.service
- enabled: yes
- state: restarted
-
- - name: Enable and launch nginx systemd service
- systemd:
- name: nginx.service
- enabled: yes
- state: restarted
-
- - name: Create letsencrypt certificate
- command: certbot --nginx certonly -d gerbolyze.jaseg.net -n --agree-tos --email gerboweb@jaseg.net
- args:
- creates: /etc/letsencrypt/live/gerbolyze.jaseg.net/fullchain.pem
-
- - name: Copy final nginx config
- copy:
- src: nginx.conf
- dest: /etc/nginx/nginx.conf
-
- - name: Restart nginx to load new cert
- systemd:
- name: nginx.service
- state: restarted
-
- - name: Enable certbot renewal timer
- systemd:
- name: certbot-renew.timer
- enabled: yes
+ - name: Setup web server
+ include_tasks: setup_webserver.yml
+ - name: Setup gerboweb
+ include_tasks: setup_gerboweb.yml
diff --git a/gerboweb/deploy/render.sh b/gerboweb/deploy/render.sh.j2
index eefe7f0..ceb837d 100755
--- a/gerboweb/deploy/render.sh
+++ b/gerboweb/deploy/render.sh.j2
@@ -5,8 +5,8 @@ ID=$1
egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2
systemd-nspawn \
- -D /var/cache/gerbolyze_container \
- -x --bind=/var/cache/gerboweb/upload/$ID:/mnt \
+ -D {{gerboweb_root}} \
+ -x --bind={{gerboweb_cache}}/upload/$ID:/mnt \
/bin/sh -c "set -euo pipefail
unzip -j -d /tmp/gerber /mnt/gerber.zip
rm -f /mnt/render_top.png /mnt/render_bottom.png /mnt/render_top.small.png /mnt/render_bottom.small.png
diff --git a/gerboweb/deploy/setup_containers.yml b/gerboweb/deploy/setup_containers.yml
new file mode 100644
index 0000000..dd0a5ca
--- /dev/null
+++ b/gerboweb/deploy/setup_containers.yml
@@ -0,0 +1,25 @@
+---
+- name: Install host requisites
+ dnf:
+ name: btrfs-progs,arch-install-scripts,systemd-container,libselinux-python
+ state: latest
+
+- name: Create container dir
+ file:
+ path: /var/cache/containers
+ owner: root
+ group: root
+ mode: 0775
+ state: directory
+
+- name: Create individual containers
+ include_tasks: bootstrap_arch_container.yml
+ with_items: "{{ containers }}"
+ loop_control:
+ loop_var: container
+
+- name: Cleanup bootstrap image
+ file:
+ path: /tmp/arch-bootstrap.tar.xz
+ state: absent
+
diff --git a/gerboweb/deploy/setup_gerboweb.yml b/gerboweb/deploy/setup_gerboweb.yml
new file mode 100644
index 0000000..e1a49fb
--- /dev/null
+++ b/gerboweb/deploy/setup_gerboweb.yml
@@ -0,0 +1,95 @@
+---
+- name: Set local facts
+ set_fact:
+ gerboweb_cache: /var/cache/gerboweb
+
+- name: Copy render script
+ template:
+ src: render.sh.j2
+ dest: /usr/local/sbin/gerbolyze_render.sh
+ mode: ug+x
+
+- name: Copy vector script
+ template:
+ src: vector.sh.j2
+ dest: /usr/local/sbin/gerbolyze_vector.sh
+ mode: ug+x
+
+- name: Install packages into gerbolyze container
+ shell: arch-chroot "{{gerboweb_root}}" pacman -Syu --noconfirm python3 opencv hdf5 gtk3 python-numpy python-pip imagemagick unzip zip
+
+ # TODO maybe install directly from local git checkout?
+- name: Install gerbolyze
+ shell: arch-chroot "{{gerboweb_root}}" pip install -U --upgrade-strategy=eager gerbolyze
+
+- name: Copy webapp sources
+ synchronize:
+ # FIXME: make this path configurable
+ src: ~/gerbolyze/gerboweb/
+ dest: /var/lib/gerboweb/
+ group: no
+ owner: no
+
+- name: Create uwsgi worker user and group
+ user:
+ name: uwsgi-gerboweb
+ create_home: no
+ group: uwsgi
+ password: '!'
+ shell: /sbin/nologin
+ system: yes
+
+- name: Template webapp config
+ template:
+ src: gerboweb.cfg.j2
+ dest: /var/lib/gerboweb/gerboweb_prod.cfg
+ owner: uwsgi-gerboweb
+ group: root
+ mode: 0660
+
+- name: Copy uwsgi config
+ copy:
+ src: uwsgi-gerboweb.ini
+ dest: /etc/uwsgi.d/gerboweb.ini
+ owner: uwsgi-gerboweb
+ group: uwsgi
+ mode: 440
+
+- name: Copy job processor systemd service config
+ template:
+ src: gerboweb-job-processor.service.j2
+ dest: /etc/systemd/system/gerboweb-job-processor.service
+
+- name: Enable uwsgi systemd socket
+ systemd:
+ daemon-reload: yes
+ name: uwsgi-app@gerboweb.socket
+ enabled: yes
+
+- name: Copy gerboweb cache dir tmpfiles.d config
+ template:
+ src: tmpfiles-gerboweb.conf.j2
+ dest: /etc/tmpfiles.d/gerboweb.conf
+ owner: root
+ group: root
+ mode: 0644
+ register: tmpfiles_config
+
+- name: Kick systemd tmpfiles service to create cache dir
+ command: systemd-tmpfiles --create
+ when: tmpfiles_config is changed
+
+- name: Create job queue db
+ file:
+ path: "{{gerboweb_cache}}/job_queue.sqlite3"
+ owner: root
+ group: uwsgi
+ mode: 0660
+ state: touch
+
+- name: Enable and launch job processor
+ systemd:
+ name: gerboweb-job-processor.service
+ enabled: yes
+ state: restarted
+
diff --git a/gerboweb/deploy/setup_webserver.yml b/gerboweb/deploy/setup_webserver.yml
new file mode 100644
index 0000000..7dc65c5
--- /dev/null
+++ b/gerboweb/deploy/setup_webserver.yml
@@ -0,0 +1,52 @@
+- name: Copy first stage nginx config
+ copy:
+ src: nginx_nossl.conf
+ dest: /etc/nginx/nginx.conf
+
+- name: Add nginx user to uwsgi group for access to uwsgi socket
+ user:
+ name: nginx
+ groups: uwsgi
+ append: yes
+
+- name: Copy uwsgi systemd socket config
+ copy:
+ src: uwsgi-app@.socket
+ dest: /etc/systemd/system/
+
+- name: Copy uwsgi systemd service config
+ copy:
+ src: uwsgi-app@.service
+ dest: /etc/systemd/system/
+
+- name: Set SELinux to permissive mode # FIXME this is to let nginx talk to uwsgi
+ selinux:
+ state: permissive
+ policy: targeted
+
+- name: Enable and launch nginx systemd service
+ systemd:
+ name: nginx.service
+ enabled: yes
+ state: restarted
+
+- name: Create letsencrypt certificate
+ command: certbot --nginx certonly -d gerbolyze.jaseg.net -n --agree-tos --email gerboweb@jaseg.net
+ args:
+ creates: /etc/letsencrypt/live/gerbolyze.jaseg.net/fullchain.pem
+
+- name: Copy final nginx config
+ copy:
+ src: nginx.conf
+ dest: /etc/nginx/nginx.conf
+
+- name: Restart nginx to load new cert
+ systemd:
+ name: nginx.service
+ state: restarted
+
+- name: Enable certbot renewal timer
+ systemd:
+ name: certbot-renew.timer
+ enabled: yes
+
diff --git a/gerboweb/deploy/tmpfiles-gerboweb.conf b/gerboweb/deploy/tmpfiles-gerboweb.conf
deleted file mode 100644
index 1f11122..0000000
--- a/gerboweb/deploy/tmpfiles-gerboweb.conf
+++ /dev/null
@@ -1 +0,0 @@
-d /var/cache/gerboweb 770 uwsgi-gerboweb uwsgi 2d
diff --git a/gerboweb/deploy/tmpfiles-gerboweb.conf.j2 b/gerboweb/deploy/tmpfiles-gerboweb.conf.j2
new file mode 100644
index 0000000..18469b7
--- /dev/null
+++ b/gerboweb/deploy/tmpfiles-gerboweb.conf.j2
@@ -0,0 +1 @@
+d {{gerboweb_cache}} 770 uwsgi-gerboweb uwsgi 2d
diff --git a/gerboweb/deploy/uwsgi-gerboweb.ini b/gerboweb/deploy/uwsgi-gerboweb.ini
index 3c8addd..ec52f90 100644
--- a/gerboweb/deploy/uwsgi-gerboweb.ini
+++ b/gerboweb/deploy/uwsgi-gerboweb.ini
@@ -9,5 +9,5 @@ manage-script-name = True
plugins = python3
chdir = /var/lib/gerboweb
mount = /=gerboweb:app
-env = GERBOWEB_SETTINGS=gerboweb.cfg
+env = GERBOWEB_SETTINGS=gerboweb_prod.cfg
diff --git a/gerboweb/deploy/vector.sh b/gerboweb/deploy/vector.sh.j2
index 5d239d5..b17116e 100755
--- a/gerboweb/deploy/vector.sh
+++ b/gerboweb/deploy/vector.sh.j2
@@ -7,8 +7,8 @@ LAYER=$2
egrep -x -q '^(top|bottom)$'<<<"$LAYER" || exit 2
systemd-nspawn \
- -D /var/cache/gerbolyze_container \
- -x --bind=/var/cache/gerboweb/upload/$ID:/mnt \
+ -D {{gerboweb_root}} \
+ -x --bind={{gerboweb_cache}}/upload/$ID:/mnt \
/bin/sh -c "set -euo pipefail
cd /tmp
unzip -j -d gerber_in /mnt/gerber.zip