From dabe1d8809348cf82c82b9a871f0509e1403a8cd Mon Sep 17 00:00:00 2001 From: jaseg Date: Wed, 27 Mar 2019 23:08:20 +0900 Subject: Deployment to digitalocean works --- gerboweb/deploy/gerboweb-job-processor.service | 9 ++ gerboweb/deploy/inventory.yml | 7 ++ gerboweb/deploy/nginx.conf | 98 +++++++++++++++++++ gerboweb/deploy/playbook.yml | 125 +++++++++++++++++++------ gerboweb/deploy/tmpfiles-gerboweb.conf | 1 + gerboweb/deploy/uwsgi-app@.service | 15 +++ gerboweb/deploy/uwsgi-app@.socket | 11 +++ gerboweb/deploy/uwsgi-gerboweb.ini | 14 +++ gerboweb/gerboweb.py | 4 +- 9 files changed, 252 insertions(+), 32 deletions(-) create mode 100644 gerboweb/deploy/gerboweb-job-processor.service create mode 100644 gerboweb/deploy/inventory.yml create mode 100644 gerboweb/deploy/nginx.conf create mode 100644 gerboweb/deploy/tmpfiles-gerboweb.conf create mode 100644 gerboweb/deploy/uwsgi-app@.service create mode 100644 gerboweb/deploy/uwsgi-app@.socket create mode 100644 gerboweb/deploy/uwsgi-gerboweb.ini (limited to 'gerboweb') diff --git a/gerboweb/deploy/gerboweb-job-processor.service b/gerboweb/deploy/gerboweb-job-processor.service new file mode 100644 index 0000000..8569317 --- /dev/null +++ b/gerboweb/deploy/gerboweb-job-processor.service @@ -0,0 +1,9 @@ +[Unit] +Description=Gerboweb gerber job processor + +[Service] +WorkingDirectory=/var/lib/gerboweb +ExecStart=/usr/bin/python3 job_processor.py /var/cache/gerboweb/job_queue.sqlite3 + +[Install] +WantedBy=uwsgi-app@gerboweb.service diff --git a/gerboweb/deploy/inventory.yml b/gerboweb/deploy/inventory.yml new file mode 100644 index 0000000..34e2e99 --- /dev/null +++ b/gerboweb/deploy/inventory.yml @@ -0,0 +1,7 @@ +--- +all: + hosts: + wendelstein: + ansible_host: wendelstein.jaseg.net + ansible_ssh_identity_file: ~/.ssh/id_ed25519 + ansible_user: root diff --git a/gerboweb/deploy/nginx.conf b/gerboweb/deploy/nginx.conf new file mode 100644 index 0000000..22b3be2 --- /dev/null +++ b/gerboweb/deploy/nginx.conf @@ -0,0 +1,98 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 4096; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name gerbolyze.jaseg.net; + root /usr/share/nginx/html; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location ^~ /static/ { + root /var/lib/gerboweb/static; + } + + location / { + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/gerboweb.socket; + } + + error_page 404 /404.html; + location = /40x.html { + root /usr/share/nginx/html; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers PROFILE=SYSTEM; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} + diff --git a/gerboweb/deploy/playbook.yml b/gerboweb/deploy/playbook.yml index a510772..eb4f367 100644 --- a/gerboweb/deploy/playbook.yml +++ b/gerboweb/deploy/playbook.yml @@ -1,33 +1,30 @@ - name: Gerbolyze container setup playbook - hosts: localhost - connection: local + hosts: all tasks: + - name: Create container image file + command: truncate -s 4G /var/cache/gerbolyze_container.img + args: + creates: /var/cache/gerbolyze_container.img + register: create_container + - name: Download arch bootstrap image get_url: url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz dest: /tmp/arch-bootstrap.tar.xz checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756 + when: create_container is changed - name: Install host requisites - become: yes dnf: - name: btrfs-progs,arch-install-scripts + name: btrfs-progs,arch-install-scripts,nginx,uwsgi,python3-flask,python3-flask-wtf,systemd-container,uwsgi-plugin-python3 state: latest - - name: Create container image file - become: yes - shell: truncate -s 4G /var/cache/gerbolyze_container.img - args: - creates: /var/cache/gerbolyze_container.img - - name: Create container image filesystem - become: yes filesystem: dev: /var/cache/gerbolyze_container.img fstype: btrfs - name: Create container image fstab entry - become: yes mount: src: /var/cache/gerbolyze_container.img path: /var/cache/gerbolyze_container @@ -36,7 +33,6 @@ opts: loop - name: Unpack bootstrap image - become: yes unarchive: remote_src: yes src: /tmp/arch-bootstrap.tar.xz @@ -45,56 +41,125 @@ creates: /var/cache/gerbolyze_container/etc - name: Copy mirrorlist into container - become: yes copy: src: mirrorlist dest: /var/cache/gerbolyze_container/etc/pacman.d/mirrorlist - name: Copy render script - become: yes copy: src: render.sh dest: /usr/local/sbin/gerbolyze_render.sh mode: ug+x - name: Copy vector script - become: yes copy: src: vector.sh dest: /usr/local/sbin/gerbolyze_vector.sh mode: ug+x - name: Initialize container pacman keyring - become: yes shell: arch-chroot /var/cache/gerbolyze_container pacman-key --init && arch-chroot /var/cache/gerbolyze_container pacman-key --populate archlinux args: creates: /var/cache/gerbolyze_container/etc/pacman.d/gnupg - name: Fixup pacman.conf for pacman to work in chroot without its own root fs - become: yes lineinfile: path: /var/cache/gerbolyze_container/etc/pacman.conf regexp: '^CheckSpace' line: '#CheckSpace' - name: Update container and install software - become: yes shell: arch-chroot /var/cache/gerbolyze_container pacman -Syu --noconfirm python3 opencv hdf5 gtk3 python-numpy python-pip imagemagick unzip zip # TODO maybe install directly from local git checkout? - name: Install gerbolyze - become: yes shell: arch-chroot /var/cache/gerbolyze_container pip install -U --upgrade-strategy=eager gerbolyze - # - name: Cleanup bootstrap image - # file: - # path: /tmp/arch-bootstrap.tar.xz - # state: absent - - - name: Create app cache directory + - name: Cleanup bootstrap image file: - path: /var/cache/gerboweb - owner: user # FIXME debug - group: user # FIXME debug - mode: 0770 + path: /tmp/arch-bootstrap.tar.xz + state: absent + + - name: Copy webapp sources + synchronize: + # FIXME: make this path configurable + src: ~/gerbolyze/gerboweb/ + dest: /var/lib/gerboweb/ + group: no + owner: no + + - name: Copy nginx config + copy: + src: nginx.conf + dest: /etc/nginx/nginx.conf + + - name: Create uwsgi worker user and group + user: + name: uwsgi-gerboweb + create_home: no + group: uwsgi + password: '!' + shell: /sbin/nologin + system: yes + + - name: Add nginx user to uwsgi group for access to uwsgi socket + user: + name: nginx + groups: uwsgi + append: yes + + - name: Copy uwsgi config + copy: + src: uwsgi-gerboweb.ini + dest: /etc/uwsgi.d/gerboweb.ini + owner: uwsgi-gerboweb + group: uwsgi + mode: 440 + + - name: Copy uwsgi systemd socket config + copy: + src: uwsgi-app@.socket + dest: /etc/systemd/system/ + + - name: Copy uwsgi systemd service config + copy: + src: uwsgi-app@.service + dest: /etc/systemd/system/ + + - name: Copy job processor systemd service config + copy: + src: gerboweb-job-processor.service + dest: /etc/systemd/system/ + + - name: Enable uwsgi systemd socket + systemd: + daemon-reload: yes + name: uwsgi-app@gerboweb.socket + enabled: yes + + - name: Enable and launch uwsgi systemd service + systemd: + name: uwsgi-app@gerboweb.service + enabled: yes + state: restarted + + - name: Enable and launch job processor + systemd: + name: gerboweb-job-processor.service + enabled: yes + state: restarted + + - name: Enable and launch nginx systemd service + systemd: + name: nginx.service + enabled: yes + state: restarted + + - name: Copy gerboweb cache dir tmpfiles.d config + copy: + src: tmpfiles-gerboweb.conf + dest: /etc/tmpfiles.d/gerboweb.conf + owner: root + group: root + mode: 0644 diff --git a/gerboweb/deploy/tmpfiles-gerboweb.conf b/gerboweb/deploy/tmpfiles-gerboweb.conf new file mode 100644 index 0000000..33264cf --- /dev/null +++ b/gerboweb/deploy/tmpfiles-gerboweb.conf @@ -0,0 +1 @@ +d /var/cache/gerboweb 760 uwsgi-gerboweb uwsgi 2d diff --git a/gerboweb/deploy/uwsgi-app@.service b/gerboweb/deploy/uwsgi-app@.service new file mode 100644 index 0000000..61bf1e4 --- /dev/null +++ b/gerboweb/deploy/uwsgi-app@.service @@ -0,0 +1,15 @@ +[Unit] +Description=%i uWSGI app +After=syslog.target + +[Service] +ExecStart=/usr/sbin/uwsgi \ + --ini /etc/uwsgi.d/%i.ini \ + --socket /run/uwsgi/%i.socket +User=uwsgi-%i +Group=uwsgi +Restart=on-failure +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all diff --git a/gerboweb/deploy/uwsgi-app@.socket b/gerboweb/deploy/uwsgi-app@.socket new file mode 100644 index 0000000..ae06d71 --- /dev/null +++ b/gerboweb/deploy/uwsgi-app@.socket @@ -0,0 +1,11 @@ +[Unit] +Description=Socket for uWSGI app %i + +[Socket] +ListenStream=/run/uwsgi/%i.socket +SocketUser=uwsgi-%i +SocketGroup=nginx +SocketMode=0660 + +[Install] +WantedBy=sockets.target diff --git a/gerboweb/deploy/uwsgi-gerboweb.ini b/gerboweb/deploy/uwsgi-gerboweb.ini new file mode 100644 index 0000000..748af71 --- /dev/null +++ b/gerboweb/deploy/uwsgi-gerboweb.ini @@ -0,0 +1,14 @@ +[uwsgi] +chmod-socket = 660 +master = True +cheap = True +idle = 600 +die-on-idle = True # If app is not used often, it will exit and be launched + # again by systemd requested by users. + +manage-script-name = True +plugins = python3 +chdir = /var/lib/gerboweb +mount = /=gerboweb:app +env = GERBOWEB_SETTINGS=gerboweb.cfg + diff --git a/gerboweb/gerboweb.py b/gerboweb/gerboweb.py index bf2921a..6b579f0 100644 --- a/gerboweb/gerboweb.py +++ b/gerboweb/gerboweb.py @@ -14,7 +14,7 @@ import sqlite3 from flask import Flask, url_for, redirect, session, make_response, render_template, request, send_file, abort from flask_wtf import FlaskForm -from flask_wtf.file import FileField +from flask_wtf.file import FileField, FileRequired from wtforms.fields import RadioField from wtforms.validators import DataRequired from werkzeug.utils import secure_filename @@ -28,7 +28,7 @@ class UploadForm(FlaskForm): upload_file = FileField(validators=[DataRequired()]) class OverlayForm(UploadForm): - upload_file = FileField(validators=[DataRequired()]) + upload_file = FileField(validators=[FileRequired()]) side = RadioField('Side', choices=[('top', 'Top'), ('bottom', 'Bottom')]) class ResetForm(FlaskForm): -- cgit