From 874adce8f4efdda653c1e60d5b353a3bc816af93 Mon Sep 17 00:00:00 2001 From: jaseg Date: Wed, 27 Mar 2019 18:28:57 +0900 Subject: gerboweb: Initial commit The functionality is there, no design yet --- gerboweb/deploy/mirrorlist | 474 ++++++++++++++++++++++++++++++++++++++++++ gerboweb/deploy/playbook.yml | 100 +++++++++ gerboweb/deploy/render.sh | 20 ++ gerboweb/deploy/vector.sh | 18 ++ gerboweb/gerboweb.cfg | 4 + gerboweb/gerboweb.py | 147 +++++++++++++ gerboweb/job_processor.py | 40 ++++ gerboweb/job_queue.py | 88 ++++++++ gerboweb/templates/index.html | 86 ++++++++ 9 files changed, 977 insertions(+) create mode 100644 gerboweb/deploy/mirrorlist create mode 100644 gerboweb/deploy/playbook.yml create mode 100755 gerboweb/deploy/render.sh create mode 100755 gerboweb/deploy/vector.sh create mode 100644 gerboweb/gerboweb.cfg create mode 100644 gerboweb/gerboweb.py create mode 100644 gerboweb/job_processor.py create mode 100644 gerboweb/job_queue.py create mode 100644 gerboweb/templates/index.html diff --git a/gerboweb/deploy/mirrorlist b/gerboweb/deploy/mirrorlist new file mode 100644 index 0000000..a2fd58c --- /dev/null +++ b/gerboweb/deploy/mirrorlist @@ -0,0 +1,474 @@ +## +## Arch Linux repository mirrorlist +## Generated on 2017-06-06 +## + +## Worldwide +#Server = https://archlinux.surlyjake.com/archlinux/$repo/os/$arch +#Server = http://mirrors.evowise.com/archlinux/$repo/os/$arch +Server = http://mirror.rackspace.com/archlinux/$repo/os/$arch + +## Australia +#Server = https://mirror.aarnet.edu.au/pub/archlinux/$repo/os/$arch +#Server = http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch +#Server = http://ftp.iinet.net.au/pub/archlinux/$repo/os/$arch +#Server = http://mirror.internode.on.net/pub/archlinux/$repo/os/$arch +#Server = http://ftp.swin.edu.au/archlinux/$repo/os/$arch +#Server = http://archlinux.uberglobalmirror.com/$repo/os/$arch + +## Austria +#Server = http://mirror.digitalnova.at/archlinux/$repo/os/$arch +#Server = http://mirror.easyname.at/archlinux/$repo/os/$arch +#Server = http://mirror1.htu.tugraz.at/archlinux/$repo/os/$arch + +## Belarus +#Server = http://ftp.byfly.by/pub/archlinux/$repo/os/$arch +#Server = http://mirror.datacenter.by/pub/archlinux/$repo/os/$arch + +## Belgium +#Server = http://archlinux.cu.be/$repo/os/$arch +#Server = http://archlinux.mirror.kangaroot.net/$repo/os/$arch + +## Bosnia and Herzegovina +#Server = http://burek.archlinux.ba/$repo/os/$arch +#Server = http://archlinux.mirror.ba/$repo/os/$arch + +## Brazil +#Server = http://br.mirror.archlinux-br.org/$repo/os/$arch +#Server = http://archlinux.c3sl.ufpr.br/$repo/os/$arch +#Server = http://linorg.usp.br/archlinux/$repo/os/$arch +#Server = http://pet.inf.ufsc.br/mirrors/archlinux/$repo/os/$arch +#Server = http://archlinux.pop-es.rnp.br/$repo/os/$arch + +## Bulgaria +#Server = http://mirror.host.ag/archlinux/$repo/os/$arch +#Server = http://mirrors.netix.net/archlinux/$repo/os/$arch +#Server = http://mirror.telepoint.bg/archlinux/$repo/os/$arch +#Server = http://mirrors.uni-plovdiv.net/archlinux/$repo/os/$arch +#Server = https://mirrors.uni-plovdiv.net/archlinux/$repo/os/$arch + +## Canada +#Server = http://mirror.cedille.club/archlinux/$repo/os/$arch +#Server = http://archlinux.mirror.colo-serv.net/$repo/os/$arch +#Server = http://mirror.csclub.uwaterloo.ca/archlinux/$repo/os/$arch +#Server = https://mirror.csclub.uwaterloo.ca/archlinux/$repo/os/$arch +#Server = http://mirror.frgl.pw/archlinux/$repo/os/$arch +#Server = https://mirror.frgl.pw/archlinux/$repo/os/$arch +#Server = http://mirror.its.dal.ca/archlinux/$repo/os/$arch +#Server = http://muug.ca/mirror/archlinux/$repo/os/$arch +#Server = https://muug.ca/mirror/archlinux/$repo/os/$arch +#Server = http://archlinux.mirror.rafal.ca/$repo/os/$arch + +## Chile +#Server = http://mirror.archlinux.cl/$repo/os/$arch + +## China +#Server = http://mirrors.163.com/archlinux/$repo/os/$arch +#Server = http://mirror.lzu.edu.cn/archlinux/$repo/os/$arch +#Server = http://mirrors.neusoft.edu.cn/archlinux/$repo/os/$arch +#Server = https://mirrors.skyshe.cn/archlinux/$repo/os/$arch +#Server = http://mirrors.tuna.tsinghua.edu.cn/archlinux/$repo/os/$arch +#Server = https://mirrors.tuna.tsinghua.edu.cn/archlinux/$repo/os/$arch +#Server = http://mirrors.ustc.edu.cn/archlinux/$repo/os/$arch +#Server = https://mirrors.ustc.edu.cn/archlinux/$repo/os/$arch +#Server = http://mirrors.xjtu.edu.cn/archlinux/$repo/os/$arch +#Server = https://mirrors.xjtu.edu.cn/archlinux/$repo/os/$arch +#Server = http://mirrors.zju.edu.cn/archlinux/$repo/os/$arch + +## Colombia +#Server = http://mirror.edatel.net.co/archlinux/$repo/os/$arch +#Server = http://mirror.upb.edu.co/archlinux/$repo/os/$arch + +## Croatia +#Server = http://archlinux.iskon.hr/$repo/os/$arch + +## Czech Republic +#Server = http://mirror.dkm.cz/archlinux/$repo/os/$arch +#Server = https://mirror.dkm.cz/archlinux/$repo/os/$arch +#Server = http://ftp.fi.muni.cz/pub/linux/arch/$repo/os/$arch +#Server = http://ftp.linux.cz/pub/linux/arch/$repo/os/$arch +#Server = http://gluttony.sin.cvut.cz/arch/$repo/os/$arch +#Server = https://gluttony.sin.cvut.cz/arch/$repo/os/$arch +#Server = http://mirrors.nic.cz/archlinux/$repo/os/$arch +#Server = http://ftp.sh.cvut.cz/arch/$repo/os/$arch +#Server = https://ftp.sh.cvut.cz/arch/$repo/os/$arch +#Server = http://mirror.vpsfree.cz/archlinux/$repo/os/$arch + +## Denmark +#Server = http://mirrors.dotsrc.org/archlinux/$repo/os/$arch +#Server = https://mirrors.dotsrc.org/archlinux/$repo/os/$arch +#Server = http://ftp.klid.dk/ftp/archlinux/$repo/os/$arch +#Server = http://mirror.one.com/archlinux/$repo/os/$arch +#Server = https://mirror.one.com/archlinux/$repo/os/$arch + +## Ecuador +#Server = http://mirror.cedia.org.ec/archlinux/$repo/os/$arch +#Server = http://mirror.espoch.edu.ec/archlinux/$repo/os/$arch +#Server = http://mirror.uta.edu.ec/archlinux/$repo/os/$arch + +## Finland +#Server = http://arch.mirror.far.fi/$repo/os/$arch + +## France +#Server = http://archlinux.de-labrusse.fr/$repo/os/$arch +#Server = http://mirror.archlinux.ikoula.com/archlinux/$repo/os/$arch +#Server = http://archlinux.vi-di.fr/$repo/os/$arch +#Server = https://archlinux.vi-di.fr/$repo/os/$arch +#Server = http://mirror.armbrust.me/archlinux/$repo/os/$arch +#Server = https://mirror.armbrust.me/archlinux/$repo/os/$arch +#Server = https://archlinux.ec-tech.fr/$repo/os/$arch +#Server = http://fooo.biz/archlinux/$repo/os/$arch +#Server = https://fooo.biz/archlinux/$repo/os/$arch +#Server = http://mirror.gerhard.re/archlinux/$repo/os/$arch +#Server = http://mirror.ibcp.fr/pub/archlinux/$repo/os/$arch +#Server = http://mirror.lastmikoi.net/archlinux/$repo/os/$arch +#Server = http://archlinux.mailtunnel.eu/$repo/os/$arch +#Server = https://www.mailtunnel.eu/archlinux/$repo/os/$arch +#Server = http://mir.archlinux.fr/$repo/os/$arch +#Server = http://archlinux.mirrors.ovh.net/archlinux/$repo/os/$arch +#Server = http://archlinux.mirror.pkern.at/$repo/os/$arch +#Server = https://archlinux.mirror.pkern.at/$repo/os/$arch +#Server = http://archlinux.polymorf.fr/$repo/os/$arch +#Server = http://mirrors.standaloneinstaller.com/archlinux/$repo/os/$arch +#Server = http://arch.tamcore.eu/$repo/os/$arch +#Server = http://mirror.tyborek.pl/arch/$repo/os/$arch +#Server = https://mirror.tyborek.pl/arch/$repo/os/$arch +#Server = http://ftp.u-strasbg.fr/linux/distributions/archlinux/$repo/os/$arch +#Server = https://mirror.wormhole.eu/archlinux/$repo/os/$arch +#Server = http://arch.yourlabs.org/$repo/os/$arch + +## Germany +#Server = http://mirror.23media.de/archlinux/$repo/os/$arch +#Server = https://arch.32g.eu/$repo/os/$arch +#Server = http://artfiles.org/archlinux.org/$repo/os/$arch +#Server = https://fabric-mirror.vps.hosteurope.de/archlinux/$repo/os/$arch +#Server = https://mirror.bethselamin.de/$repo/os/$arch +#Server = http://mirror.euserv.net/linux/archlinux/$repo/os/$arch +#Server = http://mirror.f4st.host/archlinux/$repo/os/$arch +#Server = https://mirror.f4st.host/archlinux/$repo/os/$arch +#Server = http://ftp.fau.de/archlinux/$repo/os/$arch +#Server = https://ftp.fau.de/archlinux/$repo/os/$arch +#Server = http://mirror.fluxent.de/archlinux/$repo/os/$arch +#Server = https://mirror.fluxent.de/archlinux/$repo/os/$arch +#Server = http://mirror.gnomus.de/$repo/os/$arch +#Server = http://www.gutscheindrache.com/mirror/archlinux/$repo/os/$arch +#Server = http://ftp.gwdg.de/pub/linux/archlinux/$repo/os/$arch +#Server = http://mirror.hactar.xyz/$repo/os/$arch +#Server = https://mirror.hactar.xyz/$repo/os/$arch +#Server = http://archlinux.honkgong.info/$repo/os/$arch +#Server = http://ftp.hosteurope.de/mirror/ftp.archlinux.org/$repo/os/$arch +#Server = http://ftp-stud.hs-esslingen.de/pub/Mirrors/archlinux/$repo/os/$arch +#Server = http://archlinux.mirror.iphh.net/$repo/os/$arch +#Server = http://repo.itmettke.de/archlinux/$repo/os/$arch +#Server = https://repo.itmettke.de/archlinux/$repo/os/$arch +#Server = https://mirror.jankoppe.de/archlinux/$repo/os/$arch +#Server = http://arch.jensgutermuth.de/$repo/os/$arch +#Server = https://arch.jensgutermuth.de/$repo/os/$arch +#Server = http://mirror.js-webcoding.de/pub/archlinux/$repo/os/$arch +#Server = https://mirror.js-webcoding.de/pub/archlinux/$repo/os/$arch +#Server = http://k42.ch/mirror/archlinux/$repo/os/$arch +#Server = https://k42.ch/mirror/archlinux/$repo/os/$arch +#Server = http://mirror.de.leaseweb.net/archlinux/$repo/os/$arch +Server = https://mirror.de.leaseweb.net/archlinux/$repo/os/$arch +#Server = http://mirror.loli.forsale/arch/$repo/os/$arch +#Server = https://mirror.loli.forsale/arch/$repo/os/$arch +#Server = http://mirror.metalgamer.eu/archlinux/$repo/os/$arch +#Server = https://mirror.metalgamer.eu/archlinux/$repo/os/$arch +#Server = http://mirror.michael-eckert.net/archlinux/$repo/os/$arch +#Server = https://mirror.michael-eckert.net/archlinux/$repo/os/$arch +#Server = http://mirrors.n-ix.net/archlinux/$repo/os/$arch +#Server = https://mirrors.n-ix.net/archlinux/$repo/os/$arch +#Server = http://mirror.netcologne.de/archlinux/$repo/os/$arch +Server = https://mirror.netcologne.de/archlinux/$repo/os/$arch +#Server = http://mirrors.niyawe.de/archlinux/$repo/os/$arch +#Server = https://mirrors.niyawe.de/archlinux/$repo/os/$arch +#Server = http://archlinux.nullpointer.io/$repo/os/$arch +#Server = https://archlinux.nullpointer.io/$repo/os/$arch +#Server = http://mirror.pseudoform.org/$repo/os/$arch +#Server = https://mirror.pseudoform.org/$repo/os/$arch +#Server = https://www.ratenzahlung.de/mirror/archlinux/$repo/os/$arch +#Server = http://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch +#Server = http://linux.rz.rub.de/archlinux/$repo/os/$arch +#Server = http://mirror.selfnet.de/archlinux/$repo/os/$arch +#Server = http://ftp.spline.inf.fu-berlin.de/mirrors/archlinux/$repo/os/$arch +#Server = https://ftp.spline.inf.fu-berlin.de/mirrors/archlinux/$repo/os/$arch +#Server = http://archlinux.thaller.ws/$repo/os/$arch +#Server = https://archlinux.thaller.ws/$repo/os/$arch +#Server = http://archlinux.thelinuxnetworx.rocks/$repo/os/$arch +#Server = https://archlinux.thelinuxnetworx.rocks/$repo/os/$arch +#Server = http://archmirror.tomforb.es/$repo/os/$arch +#Server = https://archmirror.tomforb.es/$repo/os/$arch +#Server = http://ftp.tu-chemnitz.de/pub/linux/archlinux/$repo/os/$arch +#Server = http://mirror.ubrco.de/archlinux/$repo/os/$arch +#Server = https://mirror.ubrco.de/archlinux/$repo/os/$arch +#Server = http://ftp.uni-bayreuth.de/linux/archlinux/$repo/os/$arch +#Server = http://ftp.uni-hannover.de/archlinux/$repo/os/$arch +#Server = http://ftp.uni-kl.de/pub/linux/archlinux/$repo/os/$arch +#Server = http://mirror.united-gameserver.de/archlinux/$repo/os/$arch +#Server = http://mirror.vfn-nrw.de/archlinux/$repo/os/$arch +#Server = https://mirror.vfn-nrw.de/archlinux/$repo/os/$arch + +## Greece +#Server = http://ftp.cc.uoc.gr/mirrors/linux/archlinux/$repo/os/$arch +#Server = http://foss.aueb.gr/mirrors/linux/archlinux/$repo/os/$arch +#Server = https://foss.aueb.gr/mirrors/linux/archlinux/$repo/os/$arch +#Server = http://mirrors.myaegean.gr/linux/archlinux/$repo/os/$arch +#Server = http://ftp.ntua.gr/pub/linux/archlinux/$repo/os/$arch +#Server = http://ftp.otenet.gr/linux/archlinux/$repo/os/$arch + +## Hong Kong +#Server = http://arch-mirror.wtako.net/$repo/os/$arch +#Server = https://arch-mirror.wtako.net/$repo/os/$arch + +## Hungary +#Server = http://ftp.energia.mta.hu/pub/mirrors/ftp.archlinux.org/$repo/os/$arch +#Server = http://archmirror.hbit.sztaki.hu/archlinux/$repo/os/$arch + +## Iceland +#Server = http://mirror.system.is/arch/$repo/os/$arch +#Server = https://mirror.system.is/arch/$repo/os/$arch + +## India +#Server = http://mirror.cse.iitk.ac.in/archlinux/$repo/os/$arch +#Server = http://ftp.iitm.ac.in/archlinux/$repo/os/$arch + +## Indonesia +#Server = http://mirror.devilzc0de.org/archlinux/$repo/os/$arch +#Server = http://mirror.poliwangi.ac.id/archlinux/$repo/os/$arch +#Server = http://suro.ubaya.ac.id/archlinux/$repo/os/$arch + +## Iran +#Server = http://repo.sadjad.ac.ir/arch/$repo/os/$arch +#Server = https://repo.sadjad.ac.ir/arch/$repo/os/$arch + +## Ireland +#Server = http://ftp.heanet.ie/mirrors/ftp.archlinux.org/$repo/os/$arch +#Server = https://ftp.heanet.ie/mirrors/ftp.archlinux.org/$repo/os/$arch + +## Israel +#Server = http://mirror.isoc.org.il/pub/archlinux/$repo/os/$arch + +## Italy +#Server = http://archlinux.prometeolibero.eu/archlinux/$repo/os/$arch +#Server = https://archlinux.prometeolibero.eu/archlinux/$repo/os/$arch +#Server = https://archlinux.beccacervello.it/archlinux/$repo/os/$arch +#Server = http://mi.mirror.garr.it/mirrors/archlinux/$repo/os/$arch +#Server = http://mirrors.prometeus.net/archlinux/$repo/os/$arch +#Server = http://archlinux.students.cs.unibo.it/$repo/os/$arch + +## Japan +#Server = http://ftp.tsukuba.wide.ad.jp/Linux/archlinux/$repo/os/$arch +Server = http://ftp.jaist.ac.jp/pub/Linux/ArchLinux/$repo/os/$arch + +## Kazakhstan +#Server = http://mirror.neolabs.kz/archlinux/$repo/os/$arch + +## Latvia +#Server = http://archlinux.koyanet.lv/archlinux/$repo/os/$arch + +## Lithuania +#Server = http://mirrors.atviras.lt/archlinux/$repo/os/$arch +#Server = https://mirrors.atviras.lt/archlinux/$repo/os/$arch + +## Luxembourg +#Server = http://archlinux.mirror.root.lu/$repo/os/$arch + +## Macedonia +#Server = http://arch.softver.org.mk/archlinux/$repo/os/$arch +#Server = http://mirror.t-home.mk/archlinux/$repo/os/$arch +#Server = https://mirror.t-home.mk/archlinux/$repo/os/$arch + +## Netherlands +#Server = http://arch.apt-get.eu/$repo/os/$arch +#Server = http://mirror.i3d.net/pub/archlinux/$repo/os/$arch +#Server = https://mirror.i3d.net/pub/archlinux/$repo/os/$arch +#Server = http://mirror.nl.leaseweb.net/archlinux/$repo/os/$arch +#Server = https://mirror.nl.leaseweb.net/archlinux/$repo/os/$arch +#Server = http://mirror.netrouting.net/archlinux/$repo/os/$arch +#Server = http://ftp.nluug.nl/os/Linux/distr/archlinux/$repo/os/$arch +#Server = http://ftp.snt.utwente.nl/pub/os/linux/archlinux/$repo/os/$arch +#Server = http://archlinux.mirror.wearetriple.com/$repo/os/$arch +#Server = https://archlinux.mirror.wearetriple.com/$repo/os/$arch + +## New Caledonia +#Server = http://mirror.lagoon.nc/pub/archlinux/$repo/os/$arch +#Server = http://archlinux.nautile.nc/archlinux/$repo/os/$arch + +## New Zealand +#Server = https://mirror.smith.geek.nz/archlinux/$repo/os/$arch + +## Norway +#Server = http://mirror.archlinux.no/$repo/os/$arch +#Server = http://archlinux.uib.no/$repo/os/$arch +#Server = http://mirror.neuf.no/archlinux/$repo/os/$arch +#Server = https://mirror.neuf.no/archlinux/$repo/os/$arch + +## Philippines +#Server = http://mirror.rise.ph/archlinux/$repo/os/$arch + +## Poland +#Server = http://mirror.chmuri.net/archmirror/$repo/os/$arch +#Server = http://arch.midov.pl/arch/$repo/os/$arch +#Server = http://mirror.onet.pl/pub/mirrors/archlinux/$repo/os/$arch +#Server = http://piotrkosoft.net/pub/mirrors/ftp.archlinux.org/$repo/os/$arch +#Server = http://ftp.vectranet.pl/archlinux/$repo/os/$arch + +## Portugal +#Server = http://glua.ua.pt/pub/archlinux/$repo/os/$arch +#Server = https://glua.ua.pt/pub/archlinux/$repo/os/$arch +#Server = http://ftp.rnl.tecnico.ulisboa.pt/pub/archlinux/$repo/os/$arch + +## Qatar +#Server = http://mirror.qnren.qa/archlinux/$repo/os/$arch + +## Romania +#Server = http://mirror.archlinux.ro/archlinux/$repo/os/$arch +#Server = http://archlinux.mirrors.linux.ro/$repo/os/$arch +#Server = http://mirrors.m247.ro/archlinux/$repo/os/$arch +#Server = http://mirrors.pidginhost.com/arch/$repo/os/$arch + +## Russia +#Server = http://mirror.aur.rocks/$repo/os/$arch +#Server = https://mirror.aur.rocks/$repo/os/$arch +#Server = http://mirror.rol.ru/archlinux/$repo/os/$arch +#Server = https://mirror.rol.ru/archlinux/$repo/os/$arch +#Server = http://mirror.yandex.ru/archlinux/$repo/os/$arch +#Server = https://mirror.yandex.ru/archlinux/$repo/os/$arch + +## Serbia +#Server = http://mirror.pmf.kg.ac.rs/archlinux/$repo/os/$arch + +## Singapore +#Server = http://mirror.0x.sg/archlinux/$repo/os/$arch +#Server = http://download.nus.edu.sg/mirror/arch/$repo/os/$arch + +## Slovakia +#Server = http://mirror.lnx.sk/pub/linux/archlinux/$repo/os/$arch +#Server = https://mirror.lnx.sk/pub/linux/archlinux/$repo/os/$arch +#Server = http://tux.rainside.sk/archlinux/$repo/os/$arch + +## Slovenia +#Server = http://archimonde.ts.si/archlinux/$repo/os/$arch +#Server = https://archimonde.ts.si/archlinux/$repo/os/$arch + +## South Africa +#Server = http://za.mirror.archlinux-br.org/$repo/os/$arch +#Server = http://ftp.wa.co.za/pub/archlinux/$repo/os/$arch +#Server = http://mirror.is.co.za/mirror/archlinux.org/$repo/os/$arch +#Server = http://mirror.wbs.co.za/archlinux/$repo/os/$arch + +## South Korea +#Server = http://ftp.kaist.ac.kr/ArchLinux/$repo/os/$arch +#Server = http://mirror.premi.st/archlinux/$repo/os/$arch + +## Spain +#Server = http://osl.ugr.es/archlinux/$repo/os/$arch +#Server = http://sunsite.rediris.es/mirror/archlinux/$repo/os/$arch + +## Sweden +#Server = http://ftp.acc.umu.se/mirror/archlinux/$repo/os/$arch +#Server = https://ftp.acc.umu.se/mirror/archlinux/$repo/os/$arch +#Server = http://archlinux.dynamict.se/$repo/os/$arch +#Server = https://archlinux.dynamict.se/$repo/os/$arch +#Server = http://ftp.lysator.liu.se/pub/archlinux/$repo/os/$arch +#Server = https://ftp.lysator.liu.se/pub/archlinux/$repo/os/$arch +#Server = http://ftp.myrveln.se/pub/linux/archlinux/$repo/os/$arch +#Server = https://ftp.myrveln.se/pub/linux/archlinux/$repo/os/$arch +#Server = https://mirror.osbeck.com/archlinux/$repo/os/$arch +#Server = http://ftp.portlane.com/pub/os/linux/archlinux/$repo/os/$arch + +## Switzerland +#Server = http://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +#Server = https://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +#Server = http://archlinux.puzzle.ch/$repo/os/$arch + +## Taiwan +#Server = http://archlinux.cs.nctu.edu.tw/$repo/os/$arch +#Server = http://shadow.ind.ntou.edu.tw/archlinux/$repo/os/$arch +#Server = http://ftp.tku.edu.tw/Linux/ArchLinux/$repo/os/$arch +#Server = http://ftp.yzu.edu.tw/Linux/archlinux/$repo/os/$arch + +## Thailand +#Server = http://mirror.adminbannok.com/archlinux/$repo/os/$arch +#Server = http://mirror.kku.ac.th/archlinux/$repo/os/$arch +#Server = https://mirror.kku.ac.th/archlinux/$repo/os/$arch + +## Turkey +#Server = http://ftp.linux.org.tr/archlinux/$repo/os/$arch + +## Ukraine +#Server = http://archlinux.ip-connect.vn.ua/$repo/os/$arch +#Server = https://archlinux.ip-connect.vn.ua/$repo/os/$arch +#Server = http://mirrors.nix.org.ua/linux/archlinux/$repo/os/$arch +#Server = https://mirrors.nix.org.ua/linux/archlinux/$repo/os/$arch + +## United Kingdom +#Server = http://mirror.bytemark.co.uk/archlinux/$repo/os/$arch +#Server = http://mirrors.manchester.m247.com/arch-linux/$repo/os/$arch +#Server = http://www.mirrorservice.org/sites/ftp.archlinux.org/$repo/os/$arch +#Server = http://arch.serverspace.co.uk/arch/$repo/os/$arch +#Server = http://archlinux.mirrors.uk2.net/$repo/os/$arch + +## United States +#Server = http://mirrors.acm.wpi.edu/archlinux/$repo/os/$arch +#Server = http://mirrors.advancedhosters.com/archlinux/$repo/os/$arch +#Server = http://mirrors.aggregate.org/archlinux/$repo/os/$arch +#Server = http://ca.us.mirror.archlinux-br.org/$repo/os/$arch +#Server = http://il.us.mirror.archlinux-br.org/$repo/os/$arch +#Server = http://archlinux.surlyjake.com/archlinux/$repo/os/$arch +#Server = http://arlm.tyzoid.com/$repo/os/$arch +#Server = http://mirror.as65535.net/archlinux/$repo/os/$arch +#Server = http://mirrors.cat.pdx.edu/archlinux/$repo/os/$arch +#Server = http://mirror.cc.columbia.edu/pub/linux/archlinux/$repo/os/$arch +#Server = http://arch.mirror.constant.com/$repo/os/$arch +#Server = https://arch.mirror.constant.com/$repo/os/$arch +#Server = http://cosmos.cites.illinois.edu/pub/archlinux/$repo/os/$arch +#Server = http://mirror.cs.pitt.edu/archlinux/$repo/os/$arch +#Server = http://mirror.cs.vt.edu/pub/ArchLinux/$repo/os/$arch +#Server = http://mirror.epiphyte.network/archlinux/$repo/os/$arch +#Server = https://mirror.epiphyte.network/archlinux/$repo/os/$arch +#Server = http://mirror.es.its.nyu.edu/archlinux/$repo/os/$arch +#Server = http://mirrors.gigenet.com/archlinux/$repo/os/$arch +#Server = http://mirror.grig.io/archlinux/$repo/os/$arch +#Server = https://mirror.grig.io/archlinux/$repo/os/$arch +#Server = http://www.gtlib.gatech.edu/pub/archlinux/$repo/os/$arch +#Server = http://mirror1.hackingand.coffee/arch/$repo/os/$arch +#Server = http://mirror2.hackingand.coffee/arch/$repo/os/$arch +#Server = http://mirror3.hackingand.coffee/arch/$repo/os/$arch +#Server = http://mirror.htnshost.com/archlinux/$repo/os/$arch +#Server = http://mirror.jmu.edu/pub/archlinux/$repo/os/$arch +#Server = http://mirrors.kernel.org/archlinux/$repo/os/$arch +#Server = https://mirrors.kernel.org/archlinux/$repo/os/$arch +#Server = http://mirror.us.leaseweb.net/archlinux/$repo/os/$arch +#Server = https://mirror.us.leaseweb.net/archlinux/$repo/os/$arch +#Server = http://il.mirrors.linaxe.net/archlinux/$repo/os/$arch +#Server = http://mirrors.liquidweb.com/archlinux/$repo/os/$arch +#Server = http://arch.localmsp.org/arch/$repo/os/$arch +#Server = https://arch.localmsp.org/arch/$repo/os/$arch +#Server = http://mirror.lty.me/archlinux/$repo/os/$arch +#Server = https://mirror.lty.me/archlinux/$repo/os/$arch +#Server = http://mirrors.lug.mtu.edu/archlinux/$repo/os/$arch +#Server = https://mirrors.lug.mtu.edu/archlinux/$repo/os/$arch +#Server = http://mirror.math.princeton.edu/pub/archlinux/$repo/os/$arch +#Server = http://mirror.metrocast.net/archlinux/$repo/os/$arch +#Server = http://mirror.kaminski.io/archlinux/$repo/os/$arch +#Server = https://mirror.kaminski.io/archlinux/$repo/os/$arch +#Server = http://mirror.nexcess.net/archlinux/$repo/os/$arch +#Server = http://mirrors.ocf.berkeley.edu/archlinux/$repo/os/$arch +#Server = https://mirrors.ocf.berkeley.edu/archlinux/$repo/os/$arch +#Server = http://ftp.osuosl.org/pub/archlinux/$repo/os/$arch +#Server = http://arch.mirrors.pair.com/$repo/os/$arch +#Server = http://mirrors.rit.edu/archlinux/$repo/os/$arch +#Server = https://mirrors.rit.edu/archlinux/$repo/os/$arch +#Server = http://mirrors.rutgers.edu/archlinux/$repo/os/$arch +#Server = https://mirrors.rutgers.edu/archlinux/$repo/os/$arch +#Server = https://mirrors.tuxns.net/archlinux/$repo/os/$arch +#Server = http://mirror.umd.edu/archlinux/$repo/os/$arch +#Server = http://mirror.vtti.vt.edu/archlinux/$repo/os/$arch +#Server = http://mirrors.xmission.com/archlinux/$repo/os/$arch +#Server = http://mirror.yellowfiber.net/archlinux/$repo/os/$arch + +## Vietnam +#Server = http://f.archlinuxvn.org/archlinux/$repo/os/$arch +#Server = http://mirror-fpt-telecom.fpt.net/archlinux/$repo/os/$arch + diff --git a/gerboweb/deploy/playbook.yml b/gerboweb/deploy/playbook.yml new file mode 100644 index 0000000..a510772 --- /dev/null +++ b/gerboweb/deploy/playbook.yml @@ -0,0 +1,100 @@ +- name: Gerbolyze container setup playbook + hosts: localhost + connection: local + tasks: + - name: Download arch bootstrap image + get_url: + url: http://mirror.rackspace.com/archlinux/iso/2019.03.01/archlinux-bootstrap-2019.03.01-x86_64.tar.gz + dest: /tmp/arch-bootstrap.tar.xz + checksum: sha256:865c8a25312b663e724923eecf0dfc626f4cd621e2cfcb19eafc69a4fc666756 + + - name: Install host requisites + become: yes + dnf: + name: btrfs-progs,arch-install-scripts + state: latest + + - name: Create container image file + become: yes + shell: truncate -s 4G /var/cache/gerbolyze_container.img + args: + creates: /var/cache/gerbolyze_container.img + + - name: Create container image filesystem + become: yes + filesystem: + dev: /var/cache/gerbolyze_container.img + fstype: btrfs + + - name: Create container image fstab entry + become: yes + mount: + src: /var/cache/gerbolyze_container.img + path: /var/cache/gerbolyze_container + state: mounted + fstype: btrfs + opts: loop + + - name: Unpack bootstrap image + become: yes + unarchive: + remote_src: yes + src: /tmp/arch-bootstrap.tar.xz + dest: /var/cache/gerbolyze_container + extra_opts: --strip-components=1 + creates: /var/cache/gerbolyze_container/etc + + - name: Copy mirrorlist into container + become: yes + copy: + src: mirrorlist + dest: /var/cache/gerbolyze_container/etc/pacman.d/mirrorlist + + - name: Copy render script + become: yes + copy: + src: render.sh + dest: /usr/local/sbin/gerbolyze_render.sh + mode: ug+x + + - name: Copy vector script + become: yes + copy: + src: vector.sh + dest: /usr/local/sbin/gerbolyze_vector.sh + mode: ug+x + + - name: Initialize container pacman keyring + become: yes + shell: arch-chroot /var/cache/gerbolyze_container pacman-key --init && arch-chroot /var/cache/gerbolyze_container pacman-key --populate archlinux + args: + creates: /var/cache/gerbolyze_container/etc/pacman.d/gnupg + + - name: Fixup pacman.conf for pacman to work in chroot without its own root fs + become: yes + lineinfile: + path: /var/cache/gerbolyze_container/etc/pacman.conf + regexp: '^CheckSpace' + line: '#CheckSpace' + + - name: Update container and install software + become: yes + shell: arch-chroot /var/cache/gerbolyze_container pacman -Syu --noconfirm python3 opencv hdf5 gtk3 python-numpy python-pip imagemagick unzip zip + + # TODO maybe install directly from local git checkout? + - name: Install gerbolyze + become: yes + shell: arch-chroot /var/cache/gerbolyze_container pip install -U --upgrade-strategy=eager gerbolyze + + # - name: Cleanup bootstrap image + # file: + # path: /tmp/arch-bootstrap.tar.xz + # state: absent + + - name: Create app cache directory + file: + path: /var/cache/gerboweb + owner: user # FIXME debug + group: user # FIXME debug + mode: 0770 + diff --git a/gerboweb/deploy/render.sh b/gerboweb/deploy/render.sh new file mode 100755 index 0000000..c3920de --- /dev/null +++ b/gerboweb/deploy/render.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +[ $# != 1 ] && exit 1 +ID=$1 +egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2 + +systemd-nspawn \ + -D /var/cache/gerbolyze_container \ + -x --bind=/var/cache/gerboweb/upload/$ID:/mnt \ + /bin/sh -c "set -euo pipefail +unzip -j -d /tmp/gerber /mnt/gerber.zip +rm -f /mnt/render_top.png /mnt/render_bottom.png /mnt/render_top.small.png /mnt/render_bottom.small.png +date; echo 'Rendering bottom layer' +gerbolyze render top /tmp/gerber /mnt/render_top.png +date; echo 'Scaling down' +convert /mnt/render_top.png -resize 500x500 /mnt/render_top.small.png +date; echo 'Rendering top layer' +gerbolyze render bottom /tmp/gerber /mnt/render_bottom.png +date; echo 'Scaling down' +convert /mnt/render_bottom.png -resize 500x500 /mnt/render_bottom.small.png" diff --git a/gerboweb/deploy/vector.sh b/gerboweb/deploy/vector.sh new file mode 100755 index 0000000..5d239d5 --- /dev/null +++ b/gerboweb/deploy/vector.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +[ $# != 2 ] && exit 1 +ID=$1 +egrep -x -q '^[-0-9A-Za-z]{36}$'<<<"$ID" || exit 2 +LAYER=$2 +egrep -x -q '^(top|bottom)$'<<<"$LAYER" || exit 2 + +systemd-nspawn \ + -D /var/cache/gerbolyze_container \ + -x --bind=/var/cache/gerboweb/upload/$ID:/mnt \ + /bin/sh -c "set -euo pipefail +cd /tmp +unzip -j -d gerber_in /mnt/gerber.zip +gerbolyze vectorize $LAYER gerber_in gerber /mnt/overlay.png +rm -f /mnt/gerber_out.zip +zip -r /mnt/gerber_out.zip gerber" + diff --git a/gerboweb/gerboweb.cfg b/gerboweb/gerboweb.cfg new file mode 100644 index 0000000..02ea211 --- /dev/null +++ b/gerboweb/gerboweb.cfg @@ -0,0 +1,4 @@ +MAX_CONTENT_LENGTH=10000000 +SECRET_KEY="FIXME: CHANGE THIS KEY" +UPLOAD_PATH="/var/cache/gerboweb/upload" +JOB_QUEUE_DB="/var/cache/gerboweb/job_queue.sqlite3" diff --git a/gerboweb/gerboweb.py b/gerboweb/gerboweb.py new file mode 100644 index 0000000..bf2921a --- /dev/null +++ b/gerboweb/gerboweb.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# TODO create systemd unit file +# TODO create systemd tmpfiles.d config +# TODO setup ansible deployment +# TODO setup webserver user disk quota + +import tempfile +import uuid +from functools import wraps +from os import path +import os +import sqlite3 + +from flask import Flask, url_for, redirect, session, make_response, render_template, request, send_file, abort +from flask_wtf import FlaskForm +from flask_wtf.file import FileField +from wtforms.fields import RadioField +from wtforms.validators import DataRequired +from werkzeug.utils import secure_filename + +from job_queue import JobQueue + +app = Flask(__name__, static_url_path='/static') +app.config.from_envvar('GERBOWEB_SETTINGS') + +class UploadForm(FlaskForm): + upload_file = FileField(validators=[DataRequired()]) + +class OverlayForm(UploadForm): + upload_file = FileField(validators=[DataRequired()]) + side = RadioField('Side', choices=[('top', 'Top'), ('bottom', 'Bottom')]) + +class ResetForm(FlaskForm): + pass + +job_queue = JobQueue(app.config['JOB_QUEUE_DB']) + +def tempfile_path(namespace): + """ Return a path for a per-session temporary file identified by the given namespace. Create the session tempfile + dir if necessary. The application tempfile dir is controlled via the upload_path config value and not managed by + this function. """ + if not path.isdir(app.config['UPLOAD_PATH']): + os.mkdir(app.config['UPLOAD_PATH']) + sess_tmp = path.join(app.config['UPLOAD_PATH'], session['session_id']) + if not path.isdir(sess_tmp): + os.mkdir(sess_tmp) + + return path.join(sess_tmp, namespace) + +def require_session_id(fun): + @wraps(fun) + def wrapper(*args, **kwargs): + if 'session_id' not in session: + session['session_id'] = str(uuid.uuid4()) + return fun(*args, **kwargs) + return wrapper + +@app.route('/') +@require_session_id +def index(): + forms = { + 'gerber_form': UploadForm(), + 'overlay_form': OverlayForm(), + 'reset_form': ResetForm() } + + for job in ('vector_job', 'render_job'): + if job in session and job_queue[session[job]].finished: + del session[job] + + r = make_response(render_template('index.html', + has_renders = path.isfile(tempfile_path('gerber.zip')), + has_output = path.isfile(tempfile_path('overlay.png')), + **forms)) + if 'vector_job' in session or 'render_job' in session: + r.headers.set('refresh', '10') + return r + +# NOTES about the gerber and overlay file upload routines +# * The maximum upload size is limited by the MAX_CONTENT_LENGTH config setting. +# * The uploaded files are deleted after a while by systemd tmpfiles.d +# TODO: validate this setting applies *after* gzip transport compression + +@app.route('/upload/', methods=['POST']) +@require_session_id +def upload(namespace): + if namespace not in ('gerber', 'overlay'): + return abort(400, 'Invalid upload type') + + upload_form = UploadForm() if namespace == 'gerber' else OverlayForm() + if upload_form.validate_on_submit(): + f = upload_form.upload_file.data + + if namespace == 'gerber': + f.save(tempfile_path('gerber.zip')) + session['filename'] = secure_filename(f.filename) # Cache filename for later download + if 'render_job' in session: + job_queue.drop(session['render_job']) + session['render_job'] = job_queue.enqueue('render', + session_id=session['session_id'], + client=request.remote_addr) + else: # namespace == 'vector' + f.save(tempfile_path('overlay.png')) + + # Re-vectorize if either file has changed + if path.isfile(tempfile_path('gerber.zip')) and path.isfile(tempfile_path('overlay.png')): + if 'vector_job' in session: + job_queue.drop(session['vector_job']) + session['vector_job'] = job_queue.enqueue('vector', + client=request.remote_addr, + session_id=session['session_id'], + side=upload_form.side.data) + + return redirect(url_for('index')) + +@app.route('/render/preview/') +def render_preview(side): + if not side in ('top', 'bottom'): + return abort(400, 'side must be either "top" or "bottom"') + return send_file(tempfile_path(f'render_{side}.small.png')) + +@app.route('/render/download/') +def render_download(side): + if not side in ('top', 'bottom'): + return abort(400, 'side must be either "top" or "bottom"') + return send_file(tempfile_path(f'render_{side}.png'), + mimetype='image/png', + as_attachment=True, + attachment_filename=f'{path.splitext(session["filename"])[0]}_render.png') + +@app.route('/output/download') +def output_download(): + return send_file(tempfile_path('gerber_out.zip'), + mimetype='application/zip', + as_attachment=True, + attachment_filename=f'{path.splitext(session["filename"])[0]}_with_artwork.zip') + +@app.route('/session_reset', methods=['POST']) +@require_session_id +def session_reset(): + if 'render_job' in session: + session['render_job'].abort() + if 'vector_job' in session: + session['vector_job'].abort() + session.clear() + return redirect(url_for('index')) + diff --git a/gerboweb/job_processor.py b/gerboweb/job_processor.py new file mode 100644 index 0000000..c138bf4 --- /dev/null +++ b/gerboweb/job_processor.py @@ -0,0 +1,40 @@ + +import signal +import subprocess +import logging +import itertools + +from job_queue import JobQueue + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('queue', help='job queue sqlite3 database file') + parser.add_argument('--loglevel', '-l', default='info') + args = parser.parse_args() + + numeric_level = getattr(logging, args.loglevel.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: %s' % loglevel) + logging.basicConfig(level=numeric_level) + + job_queue = JobQueue(args.queue) + + signal.signal(signal.SIGALRM, lambda *args: None) # Ignore incoming alarm signals while processing jobs + signal.setitimer(signal.ITIMER_REAL, 0.001, 1) + while signal.sigwait([signal.SIGALRM, signal.SIGINT]) == signal.SIGALRM: + logging.debug('Checking for jobs') + for job in job_queue.job_iter('render'): + logging.info(f'Processing {job.type} job {job.id} session {job["session_id"]} from {job.client} submitted {job.created}') + with job: + job.result = subprocess.call(['sudo', '/usr/local/sbin/gerbolyze_render.sh', job['session_id']]) + logging.info(f'Finishied processing {job.type} job {job.id}') + + for job in job_queue.job_iter('vector'): + logging.info(f'Processing {job.type} job {job.id} session {job["session_id"]} from {job.client} submitted {job.created}') + with job: + job.result = subprocess.call(['sudo', '/usr/local/sbin/gerbolyze_vector.sh', job['session_id'], job['side']]) + logging.info(f'Finishied processing {job.type} job {job.id}') + logging.info('Caught SIGINT. Exiting.') + diff --git a/gerboweb/job_queue.py b/gerboweb/job_queue.py new file mode 100644 index 0000000..e48379d --- /dev/null +++ b/gerboweb/job_queue.py @@ -0,0 +1,88 @@ + +import json +import sqlite3 + +class JobQueue: + def __init__(self, dbfile): + self.dbfile = dbfile + self.db = sqlite3.connect(dbfile, check_same_thread=False) + self.db.row_factory = sqlite3.Row + with self.db as conn: + conn.execute('''CREATE TABLE IF NOT EXISTS jobs + (id INTEGER PRIMARY KEY, + type TEXT, + params TEXT, + client TEXT, + result TEXT DEFAULT NULL, + created DATETIME DEFAULT CURRENT_TIMESTAMP, + consumed DATETIME DEFAULT NULL, + aborted DATETIME DEFAULT NULL, + finished DATETIME DEFAULT NULL);''') + + def enqueue(self, task_type:str, client, **params): + """ Enqueue a job of the given type with the given params. Returns the new job ID. """ + with self.db as conn: + return conn.execute('INSERT INTO jobs(type, client, params) VALUES (?, ?, ?)', + (task_type, client, json.dumps(params))).lastrowid + + def check_result(slef, job_id): + with self.db as conn: + job = conn.execute('SELECT * FROM jobs WHERE id=?', (job_id,)).fetchone() + if job is None: + raise IndexError('Job id not found') + return job.result + + def drop(self, job_id): + with self.db as conn: + return conn.execute('DELETE FROM jobs WHERE id=?', (job_id,)).rowcount > 0 + + def pop(self, task_type): + """ Fetch the next job of the given type. Returns a sqlite3.Row object of the job or None if no jobs of the given + type are queued. """ + with self.db as conn: + job = conn.execute('SELECT * FROM jobs WHERE type=? AND consumed IS NULL ORDER BY created ASC LIMIT 1', + (task_type,)).fetchone() + if job is None: + return None + + # Atomically commit to this job + conn.execute('UPDATE jobs SET consumed=datetime("now") WHERE id=?', (job['id'],)) + + return Job(self.db, job) + + def job_iter(self, task_type): + return iter(lambda: self.pop(task_type), None) + + def __getitem__(self, key): + """ Return the job with the given ID, or raise a KeyError if the key cannot be found. """ + with self.db as conn: + job = conn.execute('SELECT * FROM jobs WHERE id=?', (key,)).fetchone() + if job is None: + raise KeyError(f'Unknown job ID "{key}"') + + return Job(self.db, job) + +class Job(dict): + def __init__(self, db, row): + super().__init__(json.loads(row['params'])) + self._db = db + self._row = row + self.id = row['id'] + self.type = row['type'] + self.client = row['client'] + self.created = row['created'] + self.consumed = row['consumed'] + self.finished = row['finished'] + self.result = None + + def __enter__(self): + return self + + def __exit__(self, _exc_type, _exc_val, _exc_tb): + with self._db as conn: + conn.execute('UPDATE jobs SET finished=datetime("now"), result=? WHERE id=?', (self.result, self.id,)) + + def abort(self): + with self._db as conn: + conn.execute('UPDATE jobs SET aborted=datetime("now") WHERE id=?', (self.id,)) + diff --git a/gerboweb/templates/index.html b/gerboweb/templates/index.html new file mode 100644 index 0000000..c5c8503 --- /dev/null +++ b/gerboweb/templates/index.html @@ -0,0 +1,86 @@ + + + + Gerbolyze Raster image to PCB renderer + + +
+

Raster image to PCB converter

+

+ Gerbolyze is a tool for rendering black and white raster (PNG) images directly onto gerber layers. You can + use this to put art on a PCB's silkscreen, solder mask or copper layers. The input is a black-and-white PNG + image that is vectorized and rendered into an existing gerber file. Gerbolyze works with gerber files + produced with any EDA toolchain and has been tested to work with both Altium and KiCAD. +

+
+ +
+

Step 1: Upload zipped gerber files

+

+ First, upload a zip file containing all your gerber files. The default file names used by KiCAD, Eagle + and Altium are supported. +

+ +
+ {{gerber_form.csrf_token}} + {{gerber_form.upload_file.label}} {{gerber_form.upload_file(size=20)}} + +
+
+ + {% if 'render_job' in session or has_renders %} +
+

Step 2: Download the target side's preview image

+

+ Second, download either the top or bottom preview image and use it to align and scale your own artwork + in an image editing program such as Gimp. Then upload your overlay image below. + + Note that you will have to convert grayscale images into binary images yourself. Gerbolyze can't do this + for you since there are lots of variables involved. Our Guideline on image processing gives an overview on + one way to produce agreeable binary images from grayscale source material. +

+ {% if 'render_job' in session %} + Processing... (this may take several minutes!) + {% else %} + Download + Download + {% endif %} +
+ {{reset_form.csrf_token}} + +
+
+ +
+

Step 3: Upload overlay image

+

+ Now, upload your binary overlay image as a PNG and let gerbolyze render it onto the target layer. The PNG + file should be a black and white binary file with details generally above about 10px size. Antialiased + edges are supported. +

+
+ {{overlay_form.csrf_token}} + {{overlay_form.upload_file.label}} {{overlay_form.upload_file(size=20)}} + {{overlay_form.side.label}} {{overlay_form.side()}} + +
+
+ + {% if 'vector_job' in session or has_output %} +
+

Step 4: Download the processed gerber files

+ {% if 'vector_job' in session %} + Processing... (this may take several minutes!) + {% else %} + Download + {% endif %} +
+ {{reset_form.csrf_token}} + +
+
+ {% endif %} {# vector job #} + {% endif %} {# render job #} + + -- cgit