From a54abadf01e9e543b416ed09e8500ed71170342c Mon Sep 17 00:00:00 2001 From: jaseg Date: Wed, 3 Apr 2019 23:53:04 +0900 Subject: gerbolyze fixes, clippy experiments --- gerboweb/deploy/bootstrap_arch_container.yml | 6 +-- gerboweb/deploy/clippy-nspawn.service | 36 +++++++++++++++ gerboweb/deploy/clippy.service.j2 | 9 ++++ gerboweb/deploy/playbook.yml | 5 +- gerboweb/deploy/setup_clippy.yml | 69 ++++++++++++++++++++++++++++ gerboweb/deploy/setup_containers.yml | 8 ---- gerboweb/deploy/uwsgi-app@.service | 3 +- gerboweb/deploy/uwsgi-gerboweb.ini | 5 +- gerboweb/gerboweb.py | 8 ++-- gerboweb/job_queue.py | 2 +- 10 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 gerboweb/deploy/clippy-nspawn.service create mode 100644 gerboweb/deploy/clippy.service.j2 create mode 100644 gerboweb/deploy/setup_clippy.yml diff --git a/gerboweb/deploy/bootstrap_arch_container.yml b/gerboweb/deploy/bootstrap_arch_container.yml index bd534e8..4126bbd 100644 --- a/gerboweb/deploy/bootstrap_arch_container.yml +++ b/gerboweb/deploy/bootstrap_arch_container.yml @@ -1,9 +1,9 @@ --- - name: Set local path facts set_fact: - image: "/var/cache/containers/{{ container }}.img" - root: "/var/cache/containers/{{ container }}_root" - "{{container}}_root": "/var/cache/containers/{{ container }}_root" + image: "/var/lib/machines/{{ container }}.img" + root: "/var/lib/machines/{{ container }}" + "{{container}}_root": "/var/lib/machines/{{ container }}" - name: Create container image file command: truncate -s 4G "{{image}}" diff --git a/gerboweb/deploy/clippy-nspawn.service b/gerboweb/deploy/clippy-nspawn.service new file mode 100644 index 0000000..66b8e85 --- /dev/null +++ b/gerboweb/deploy/clippy-nspawn.service @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: LGPL-2.1+ +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Clippy container +PartOf=machines.target +Before=machines.target +After=network.target systemd-resolved.service +RequiresMountsFor=/var/lib/machines + +[Service] +ExecStart=/usr/bin/systemd-nspawn --quiet --keep-unit --ephemeral --boot --network-veth --port=23:2342 -U --settings=override --machine=clippy +KillMode=mixed +Type=notify +RestartForceExitStatus=133 +SuccessExitStatus=133 +WatchdogSec=3min +Slice=machine.slice +Delegate=yes +TasksMax=512 + +# Enforce a strict device policy, similar to the one nspawn configures when it +# allocates its own scope unit. Make sure to keep these policies in sync if you +# change them! +DevicePolicy=closed +DeviceAllow=/dev/net/tun rwm +DeviceAllow=char-pts rw + +[Install] +WantedBy=machines.target diff --git a/gerboweb/deploy/clippy.service.j2 b/gerboweb/deploy/clippy.service.j2 new file mode 100644 index 0000000..22b3d7d --- /dev/null +++ b/gerboweb/deploy/clippy.service.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Clippy listener daemon + +[Service] +WorkingDirectory=/var/lib/clippy.git +ExecStart=/usr/bin/python3 clippy.py -s -x 60x30 -e + +[Install] +WantedBy=multi-user.target diff --git a/gerboweb/deploy/playbook.yml b/gerboweb/deploy/playbook.yml index 23544c4..60fe499 100644 --- a/gerboweb/deploy/playbook.yml +++ b/gerboweb/deploy/playbook.yml @@ -12,7 +12,7 @@ - name: Install host requisites dnf: - name: nginx,uwsgi,python3-flask,python3-flask-wtf,uwsgi-plugin-python3,certbot,python3-certbot-nginx,libselinux-python + name: nginx,uwsgi,python3-flask,python3-flask-wtf,uwsgi-plugin-python3,certbot,python3-certbot-nginx,libselinux-python,git state: latest - name: Disable password-based root login @@ -40,3 +40,6 @@ - name: Setup gerboweb include_tasks: setup_gerboweb.yml + + - name: Setup clippy + include_tasks: setup_clippy.yml diff --git a/gerboweb/deploy/setup_clippy.yml b/gerboweb/deploy/setup_clippy.yml new file mode 100644 index 0000000..e4416e0 --- /dev/null +++ b/gerboweb/deploy/setup_clippy.yml @@ -0,0 +1,69 @@ +--- +- name: Clone pixelterm git + git: + repo: https://github.com/jaseg/pixelterm + dest: "{{clippy_root}}/var/lib/pixelterm.git" + +- name: Clone clippy git + git: + repo: https://github.com/jaseg/clippy + dest: "{{clippy_root}}/var/lib/clippy.git" + +- name: Setup required packages for clippy + command: arch-chroot "{{clippy_root}}" pacman -Syu --noconfirm python3 python-pip python-numpy python-pillow + +- name: Setup pixelterm + command: arch-chroot "{{clippy_root}}" sh -c "cd /var/lib/pixelterm.git && python3 setup.py install" + +- name: Setup container clippy systemd service file + template: + src: clippy.service.j2 + dest: "{{clippy_root}}/etc/systemd/system/clippy.service" + owner: root + group: root + mode: 0664 + +- name: Enable systemd machines target + systemd: + name: machines.target + enabled: yes + +- name: Copy over clippy container auto boot service file + copy: + src: clippy-nspawn.service + dest: /etc/systemd/system/clippy-nspawn.service + owner: root + group: root + mode: 0664 + +- name: Enable clippy container auto boot + systemd: + daemon-reload: yes + name: clippy-nspawn.service + enabled: yes + +- name: Restart clippy container + shell: | + systemctl stop clippy-nspawn + sleep 1 + systemctl start clippy-nspawn + for x in $(seq 0 30); do + systemctl -M clippy is-system-running && exit + sleep 1 + done + +- name: Enable clippy systemd service in container + command: systemctl enable -M clippy clippy.service + +- name: Restart clippy systemd service in container + command: systemctl restart -M clippy clippy.service + +#- name: Enable host networkd +# systemd: +# name: systemd-networkd +# enabled: yes +# state: started + +- name: Enable clippy container networkd + command: systemctl enable -M clippy systemd-networkd + diff --git a/gerboweb/deploy/setup_containers.yml b/gerboweb/deploy/setup_containers.yml index dd0a5ca..4738f1e 100644 --- a/gerboweb/deploy/setup_containers.yml +++ b/gerboweb/deploy/setup_containers.yml @@ -4,14 +4,6 @@ name: btrfs-progs,arch-install-scripts,systemd-container,libselinux-python state: latest -- name: Create container dir - file: - path: /var/cache/containers - owner: root - group: root - mode: 0775 - state: directory - - name: Create individual containers include_tasks: bootstrap_arch_container.yml with_items: "{{ containers }}" diff --git a/gerboweb/deploy/uwsgi-app@.service b/gerboweb/deploy/uwsgi-app@.service index 61bf1e4..8398456 100644 --- a/gerboweb/deploy/uwsgi-app@.service +++ b/gerboweb/deploy/uwsgi-app@.service @@ -5,7 +5,8 @@ After=syslog.target [Service] ExecStart=/usr/sbin/uwsgi \ --ini /etc/uwsgi.d/%i.ini \ - --socket /run/uwsgi/%i.socket + --chmod-socket=660 \ + --socket=/run/uwsgi/%i.socket User=uwsgi-%i Group=uwsgi Restart=on-failure diff --git a/gerboweb/deploy/uwsgi-gerboweb.ini b/gerboweb/deploy/uwsgi-gerboweb.ini index ec52f90..155d01a 100644 --- a/gerboweb/deploy/uwsgi-gerboweb.ini +++ b/gerboweb/deploy/uwsgi-gerboweb.ini @@ -1,10 +1,7 @@ [uwsgi] master = True cheap = True -idle = 600 -die-on-idle = True # If app is not used often, it will exit and be launched - # again by systemd requested by users. - +die-on-idle = False manage-script-name = True plugins = python3 chdir = /var/lib/gerboweb diff --git a/gerboweb/gerboweb.py b/gerboweb/gerboweb.py index 7e952a4..2633e7b 100644 --- a/gerboweb/gerboweb.py +++ b/gerboweb/gerboweb.py @@ -62,13 +62,13 @@ def index(): 'overlay_form': OverlayForm(), 'reset_form': ResetForm() } - for job in ('vector_job', 'render_job'): - if job in session: - job = job_queue[session[job]] + for job_type in ('vector_job', 'render_job'): + if job_type in session: + job = job_queue[session[job_type]] if job.finished: if job.result != 0: flash(f'Error processing gerber files', 'success') # FIXME make this an error, add CSS - del session[job] + del session[job_type] r = make_response(render_template('index.html', has_renders = path.isfile(tempfile_path('gerber.zip')), diff --git a/gerboweb/job_queue.py b/gerboweb/job_queue.py index 76f17dc..62ba398 100644 --- a/gerboweb/job_queue.py +++ b/gerboweb/job_queue.py @@ -13,7 +13,7 @@ class JobQueue: type TEXT, params TEXT, client TEXT, - result TEXT DEFAULT NULL, + result INTEGER DEFAULT NULL, created DATETIME DEFAULT CURRENT_TIMESTAMP, consumed DATETIME DEFAULT NULL, aborted DATETIME DEFAULT NULL, -- cgit