diff options
| -rw-r--r-- | gerboweb/deploy/.gitignore | 2 | ||||
| -rw-r--r-- | gerboweb/deploy/bootstrap_arch_container.yml | 60 | ||||
| -rw-r--r-- | gerboweb/deploy/gerboweb-job-processor.service.j2 (renamed from gerboweb/deploy/gerboweb-job-processor.service) | 2 | ||||
| -rw-r--r-- | gerboweb/deploy/gerboweb.cfg.j2 | 4 | ||||
| -rw-r--r-- | gerboweb/deploy/playbook.yml | 201 | ||||
| -rwxr-xr-x | gerboweb/deploy/render.sh.j2 (renamed from gerboweb/deploy/render.sh) | 4 | ||||
| -rw-r--r-- | gerboweb/deploy/setup_containers.yml | 25 | ||||
| -rw-r--r-- | gerboweb/deploy/setup_gerboweb.yml | 95 | ||||
| -rw-r--r-- | gerboweb/deploy/setup_webserver.yml | 52 | ||||
| -rw-r--r-- | gerboweb/deploy/tmpfiles-gerboweb.conf | 1 | ||||
| -rw-r--r-- | gerboweb/deploy/tmpfiles-gerboweb.conf.j2 | 1 | ||||
| -rw-r--r-- | gerboweb/deploy/uwsgi-gerboweb.ini | 2 | ||||
| -rwxr-xr-x | gerboweb/deploy/vector.sh.j2 (renamed from gerboweb/deploy/vector.sh) | 4 |
13 files changed, 257 insertions, 196 deletions
diff --git a/gerboweb/deploy/.gitignore b/gerboweb/deploy/.gitignore new file mode 100644 index 0000000..97b80a1 --- /dev/null +++ b/gerboweb/deploy/.gitignore @@ -0,0 +1,2 @@ +gerboweb_flask_secret.txt +playbook.retry diff --git a/gerboweb/deploy/bootstrap_arch_container.yml b/gerboweb/deploy/bootstrap_arch_container.yml new file mode 100644 index 0000000..bd534e8 --- /dev/null +++ b/gerboweb/deploy/bootstrap_arch_container.yml @@ -0,0 +1,60 @@ +--- +- name: Set local path facts + set_fact: + image: "/var/cache/containers/{{ container }}.img" + root: "/var/cache/containers/{{ container }}_root" + "{{container}}_root": "/var/cache/containers/{{ container }}_root" + +- name: Create container image file + command: truncate -s 4G "{{image}}" + args: + creates: "{{image}}" + register: create_container + +- name: Download arch bootstrap image + get_url: + url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz + dest: /tmp/arch-bootstrap.tar.xz + checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756 + when: create_container is changed + +- name: Create container image filesystem + filesystem: + dev: "{{image}}" + fstype: btrfs + +- name: Create container image fstab entry + mount: + src: "{{image}}" + path: "{{root}}" + state: mounted + fstype: btrfs + opts: loop + +- name: Unpack bootstrap image + unarchive: + remote_src: yes + src: /tmp/arch-bootstrap.tar.xz + dest: "{{root}}" + extra_opts: --strip-components=1 + creates: "{{root}}/etc" + +- name: Copy mirrorlist into container + copy: + src: mirrorlist + dest: "{{root}}/etc/pacman.d/mirrorlist" + +- name: Initialize container pacman keyring + shell: arch-chroot "{{root}}" pacman-key --init && arch-chroot "{{root}}" pacman-key --populate archlinux + args: + creates: "{{root}}/etc/pacman.d/gnupg" + +- name: Fixup pacman.conf for pacman to work in chroot without its own root fs + lineinfile: + path: "{{root}}/etc/pacman.conf" + regexp: '^CheckSpace' + line: '#CheckSpace' + +- name: Update container and install software + shell: arch-chroot "{{root}}" pacman -Syu --noconfirm + diff --git a/gerboweb/deploy/gerboweb-job-processor.service b/gerboweb/deploy/gerboweb-job-processor.service.j2 index 8569317..517d8b8 100644 --- a/gerboweb/deploy/gerboweb-job-processor.service +++ b/gerboweb/deploy/gerboweb-job-processor.service.j2 @@ -3,7 +3,7 @@ Description=Gerboweb gerber job processor [Service] WorkingDirectory=/var/lib/gerboweb -ExecStart=/usr/bin/python3 job_processor.py /var/cache/gerboweb/job_queue.sqlite3 +ExecStart=/usr/bin/python3 job_processor.py {{gerboweb_cache}}/job_queue.sqlite3 [Install] WantedBy=uwsgi-app@gerboweb.service diff --git a/gerboweb/deploy/gerboweb.cfg.j2 b/gerboweb/deploy/gerboweb.cfg.j2 new file mode 100644 index 0000000..994cd08 --- /dev/null +++ b/gerboweb/deploy/gerboweb.cfg.j2 @@ -0,0 +1,4 @@ +MAX_CONTENT_LENGTH=10000000 +SECRET_KEY="{{lookup('password', 'gerboweb_flask_secret.txt length=32')}}" +UPLOAD_PATH="{{gerboweb_cache}}/upload" +JOB_QUEUE_DB="{{gerboweb_cache}}/job_queue.sqlite3" diff --git a/gerboweb/deploy/playbook.yml b/gerboweb/deploy/playbook.yml index a0ff505..23544c4 100644 --- a/gerboweb/deploy/playbook.yml +++ b/gerboweb/deploy/playbook.yml @@ -7,12 +7,12 @@ - name: Install common admin tools dnf: - name: htop,tmux,fish,mosh,neovim + name: htop,tmux,fish,mosh,neovim,sqlite state: latest - name: Install host requisites dnf: - name: btrfs-progs,arch-install-scripts,nginx,uwsgi,python3-flask,python3-flask-wtf,systemd-container,uwsgi-plugin-python3,certbot,python3-certbot-nginx,libselinux-python + name: nginx,uwsgi,python3-flask,python3-flask-wtf,uwsgi-plugin-python3,certbot,python3-certbot-nginx,libselinux-python state: latest - name: Disable password-based root login @@ -28,192 +28,15 @@ state: restarted when: disable_root_pw_ssh is changed - - name: Create container image file - command: truncate -s 4G /var/cache/gerbolyze_container.img - args: - creates: /var/cache/gerbolyze_container.img - register: create_container + - name: Create containers + include_tasks: setup_containers.yml + vars: + containers: + - gerboweb + - clippy - - name: Download arch bootstrap image - get_url: - url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz - dest: /tmp/arch-bootstrap.tar.xz - checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756 - when: create_container is changed - - - name: Create container image filesystem - filesystem: - dev: /var/cache/gerbolyze_container.img - fstype: btrfs - - - name: Create container image fstab entry - mount: - src: /var/cache/gerbolyze_container.img - path: /var/cache/gerbolyze_container - state: mounted - fstype: btrfs - opts: loop - - - name: Unpack bootstrap image - unarchive: - remote_src: yes - src: /tmp/arch-bootstrap.tar.xz - dest: /var/cache/gerbolyze_container - extra_opts: --strip-components=1 - creates: /var/cache/gerbolyze_container/etc - - - name: Copy mirrorlist into container - copy: - src: mirrorlist - dest: /var/cache/gerbolyze_container/etc/pacman.d/mirrorlist - - - name: Copy render script - copy: - src: render.sh - dest: /usr/local/sbin/gerbolyze_render.sh - mode: ug+x - - - name: Copy vector script - copy: - src: vector.sh - dest: /usr/local/sbin/gerbolyze_vector.sh - mode: ug+x - - - name: Initialize container pacman keyring - shell: arch-chroot /var/cache/gerbolyze_container pacman-key --init && arch-chroot /var/cache/gerbolyze_container pacman-key --populate archlinux - args: - creates: /var/cache/gerbolyze_container/etc/pacman.d/gnupg - - - name: Fixup pacman.conf for pacman to work in chroot without its own root fs - lineinfile: - path: /var/cache/gerbolyze_container/etc/pacman.conf - regexp: '^CheckSpace' - line: '#CheckSpace' - - - name: Update container and install software - shell: arch-chroot /var/cache/gerbolyze_container pacman -Syu --noconfirm python3 opencv hdf5 gtk3 python-numpy python-pip imagemagick unzip zip - - # TODO maybe install directly from local git checkout? - - name: Install gerbolyze - shell: arch-chroot /var/cache/gerbolyze_container pip install -U --upgrade-strategy=eager gerbolyze - - - name: Cleanup bootstrap image - file: - path: /tmp/arch-bootstrap.tar.xz - state: absent - - - name: Copy webapp sources - synchronize: - # FIXME: make this path configurable - src: ~/gerbolyze/gerboweb/ - dest: /var/lib/gerboweb/ - group: no - owner: no - - - name: Copy first stage nginx config - copy: - src: nginx_nossl.conf - dest: /etc/nginx/nginx.conf - - - name: Create uwsgi worker user and group - user: - name: uwsgi-gerboweb - create_home: no - group: uwsgi - password: '!' - shell: /sbin/nologin - system: yes - - - name: Add nginx user to uwsgi group for access to uwsgi socket - user: - name: nginx - groups: uwsgi - append: yes - - - name: Copy uwsgi config - copy: - src: uwsgi-gerboweb.ini - dest: /etc/uwsgi.d/gerboweb.ini - owner: uwsgi-gerboweb - group: uwsgi - mode: 440 - - - name: Copy uwsgi systemd socket config - copy: - src: uwsgi-app@.socket - dest: /etc/systemd/system/ - - - name: Copy uwsgi systemd service config - copy: - src: uwsgi-app@.service - dest: /etc/systemd/system/ - - - name: Copy job processor systemd service config - copy: - src: gerboweb-job-processor.service - dest: /etc/systemd/system/ - - - name: Set SELinux to permissive mode # FIXME - selinux: - state: permissive - policy: targeted - - - name: Enable uwsgi systemd socket - systemd: - daemon-reload: yes - name: uwsgi-app@gerboweb.socket - enabled: yes - - - name: Copy gerboweb cache dir tmpfiles.d config - copy: - src: tmpfiles-gerboweb.conf - dest: /etc/tmpfiles.d/gerboweb.conf - owner: root - group: root - mode: 0644 - register: tmpfiles_config - - - name: Kick systemd tmpfiles service to create cache dir - command: systemd-tmpfiles --create - when: tmpfiles_config is changed - - - name: Create job queue db - file: - path: /var/cache/gerboweb/job_queue.sqlite3 - owner: root - group: uwsgi - mode: 0660 - state: touch - - - name: Enable and launch job processor - systemd: - name: gerboweb-job-processor.service - enabled: yes - state: restarted - - - name: Enable and launch nginx systemd service - systemd: - name: nginx.service - enabled: yes - state: restarted - - - name: Create letsencrypt certificate - command: certbot --nginx certonly -d gerbolyze.jaseg.net -n --agree-tos --email gerboweb@jaseg.net - args: - creates: /etc/letsencrypt/live/gerbolyze.jaseg.net/fullchain.pem - - - name: Copy final nginx config - copy: - src: nginx.conf - dest: /etc/nginx/nginx.conf - - - name: Restart nginx to load new cert - systemd: - name: nginx.service - state: restarted - - - name: Enable certbot renewal timer - systemd: - name: certbot-renew.timer - enabled: yes + - name: Setup web server + include_tasks: setup_webserver.yml + - name: Setup gerboweb + include_tasks: setup_gerboweb.yml diff --git a/gerboweb/deploy/render.sh b/gerboweb/deploy/render.sh.j2 index eefe7f0..ceb837d 100755 --- a/gerboweb/deploy/render.sh +++ b/gerboweb/deploy/render.sh.j2 @@ -5,8 +5,8 @@ ID=$1 egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2 systemd-nspawn \ - -D /var/cache/gerbolyze_container \ - -x --bind=/var/cache/gerboweb/upload/$ID:/mnt \ + -D {{gerboweb_root}} \ + -x --bind={{gerboweb_cache}}/upload/$ID:/mnt \ /bin/sh -c "set -euo pipefail unzip -j -d /tmp/gerber /mnt/gerber.zip rm -f /mnt/render_top.png /mnt/render_bottom.png /mnt/render_top.small.png /mnt/render_bottom.small.png diff --git a/gerboweb/deploy/setup_containers.yml b/gerboweb/deploy/setup_containers.yml new file mode 100644 index 0000000..dd0a5ca --- /dev/null +++ b/gerboweb/deploy/setup_containers.yml @@ -0,0 +1,25 @@ +--- +- name: Install host requisites + dnf: + name: btrfs-progs,arch-install-scripts,systemd-container,libselinux-python + state: latest + +- name: Create container dir + file: + path: /var/cache/containers + owner: root + group: root + mode: 0775 + state: directory + +- name: Create individual containers + include_tasks: bootstrap_arch_container.yml + with_items: "{{ containers }}" + loop_control: + loop_var: container + +- name: Cleanup bootstrap image + file: + path: /tmp/arch-bootstrap.tar.xz + state: absent + diff --git a/gerboweb/deploy/setup_gerboweb.yml b/gerboweb/deploy/setup_gerboweb.yml new file mode 100644 index 0000000..e1a49fb --- /dev/null +++ b/gerboweb/deploy/setup_gerboweb.yml @@ -0,0 +1,95 @@ +--- +- name: Set local facts + set_fact: + gerboweb_cache: /var/cache/gerboweb + +- name: Copy render script + template: + src: render.sh.j2 + dest: /usr/local/sbin/gerbolyze_render.sh + mode: ug+x + +- name: Copy vector script + template: + src: vector.sh.j2 + dest: /usr/local/sbin/gerbolyze_vector.sh + mode: ug+x + +- name: Install packages into gerbolyze container + shell: arch-chroot "{{gerboweb_root}}" pacman -Syu --noconfirm python3 opencv hdf5 gtk3 python-numpy python-pip imagemagick unzip zip + + # TODO maybe install directly from local git checkout? +- name: Install gerbolyze + shell: arch-chroot "{{gerboweb_root}}" pip install -U --upgrade-strategy=eager gerbolyze + +- name: Copy webapp sources + synchronize: + # FIXME: make this path configurable + src: ~/gerbolyze/gerboweb/ + dest: /var/lib/gerboweb/ + group: no + owner: no + +- name: Create uwsgi worker user and group + user: + name: uwsgi-gerboweb + create_home: no + group: uwsgi + password: '!' + shell: /sbin/nologin + system: yes + +- name: Template webapp config + template: + src: gerboweb.cfg.j2 + dest: /var/lib/gerboweb/gerboweb_prod.cfg + owner: uwsgi-gerboweb + group: root + mode: 0660 + +- name: Copy uwsgi config + copy: + src: uwsgi-gerboweb.ini + dest: /etc/uwsgi.d/gerboweb.ini + owner: uwsgi-gerboweb + group: uwsgi + mode: 440 + +- name: Copy job processor systemd service config + template: + src: gerboweb-job-processor.service.j2 + dest: /etc/systemd/system/gerboweb-job-processor.service + +- name: Enable uwsgi systemd socket + systemd: + daemon-reload: yes + name: uwsgi-app@gerboweb.socket + enabled: yes + +- name: Copy gerboweb cache dir tmpfiles.d config + template: + src: tmpfiles-gerboweb.conf.j2 + dest: /etc/tmpfiles.d/gerboweb.conf + owner: root + group: root + mode: 0644 + register: tmpfiles_config + +- name: Kick systemd tmpfiles service to create cache dir + command: systemd-tmpfiles --create + when: tmpfiles_config is changed + +- name: Create job queue db + file: + path: "{{gerboweb_cache}}/job_queue.sqlite3" + owner: root + group: uwsgi + mode: 0660 + state: touch + +- name: Enable and launch job processor + systemd: + name: gerboweb-job-processor.service + enabled: yes + state: restarted + diff --git a/gerboweb/deploy/setup_webserver.yml b/gerboweb/deploy/setup_webserver.yml new file mode 100644 index 0000000..7dc65c5 --- /dev/null +++ b/gerboweb/deploy/setup_webserver.yml @@ -0,0 +1,52 @@ +- name: Copy first stage nginx config + copy: + src: nginx_nossl.conf + dest: /etc/nginx/nginx.conf + +- name: Add nginx user to uwsgi group for access to uwsgi socket + user: + name: nginx + groups: uwsgi + append: yes + +- name: Copy uwsgi systemd socket config + copy: + src: uwsgi-app@.socket + dest: /etc/systemd/system/ + +- name: Copy uwsgi systemd service config + copy: + src: uwsgi-app@.service + dest: /etc/systemd/system/ + +- name: Set SELinux to permissive mode # FIXME this is to let nginx talk to uwsgi + selinux: + state: permissive + policy: targeted + +- name: Enable and launch nginx systemd service + systemd: + name: nginx.service + enabled: yes + state: restarted + +- name: Create letsencrypt certificate + command: certbot --nginx certonly -d gerbolyze.jaseg.net -n --agree-tos --email gerboweb@jaseg.net + args: + creates: /etc/letsencrypt/live/gerbolyze.jaseg.net/fullchain.pem + +- name: Copy final nginx config + copy: + src: nginx.conf + dest: /etc/nginx/nginx.conf + +- name: Restart nginx to load new cert + systemd: + name: nginx.service + state: restarted + +- name: Enable certbot renewal timer + systemd: + name: certbot-renew.timer + enabled: yes + diff --git a/gerboweb/deploy/tmpfiles-gerboweb.conf b/gerboweb/deploy/tmpfiles-gerboweb.conf deleted file mode 100644 index 1f11122..0000000 --- a/gerboweb/deploy/tmpfiles-gerboweb.conf +++ /dev/null @@ -1 +0,0 @@ -d /var/cache/gerboweb 770 uwsgi-gerboweb uwsgi 2d diff --git a/gerboweb/deploy/tmpfiles-gerboweb.conf.j2 b/gerboweb/deploy/tmpfiles-gerboweb.conf.j2 new file mode 100644 index 0000000..18469b7 --- /dev/null +++ b/gerboweb/deploy/tmpfiles-gerboweb.conf.j2 @@ -0,0 +1 @@ +d {{gerboweb_cache}} 770 uwsgi-gerboweb uwsgi 2d diff --git a/gerboweb/deploy/uwsgi-gerboweb.ini b/gerboweb/deploy/uwsgi-gerboweb.ini index 3c8addd..ec52f90 100644 --- a/gerboweb/deploy/uwsgi-gerboweb.ini +++ b/gerboweb/deploy/uwsgi-gerboweb.ini @@ -9,5 +9,5 @@ manage-script-name = True plugins = python3 chdir = /var/lib/gerboweb mount = /=gerboweb:app -env = GERBOWEB_SETTINGS=gerboweb.cfg +env = GERBOWEB_SETTINGS=gerboweb_prod.cfg diff --git a/gerboweb/deploy/vector.sh b/gerboweb/deploy/vector.sh.j2 index 5d239d5..b17116e 100755 --- a/gerboweb/deploy/vector.sh +++ b/gerboweb/deploy/vector.sh.j2 @@ -7,8 +7,8 @@ LAYER=$2 egrep -x -q '^(top|bottom)$'<<<"$LAYER" || exit 2 systemd-nspawn \ - -D /var/cache/gerbolyze_container \ - -x --bind=/var/cache/gerboweb/upload/$ID:/mnt \ + -D {{gerboweb_root}} \ + -x --bind={{gerboweb_cache}}/upload/$ID:/mnt \ /bin/sh -c "set -euo pipefail cd /tmp unzip -j -d gerber_in /mnt/gerber.zip |