From 49a7c6df416b18516fca776614cb0aa0f483798f Mon Sep 17 00:00:00 2001 From: jaseg Date: Sun, 31 Jan 2021 20:49:23 +0100 Subject: WIP --- gerbolyze/gerbolyze.py | 439 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 349 insertions(+), 90 deletions(-) diff --git a/gerbolyze/gerbolyze.py b/gerbolyze/gerbolyze.py index 3bfccc6..5dbbefe 100755 --- a/gerbolyze/gerbolyze.py +++ b/gerbolyze/gerbolyze.py @@ -2,6 +2,8 @@ import tempfile import os.path as path from pathlib import Path import textwrap +import subprocess +import functools import os import base64 import re @@ -12,10 +14,12 @@ import shutil import math from zipfile import ZipFile, is_zipfile import shutil -from lxml import etree +from lxml import etree import gerber from gerber.render.cairo_backend import GerberCairoContext +import gerberex +import gerberex.rs274x import numpy as np import cv2 import enum @@ -23,6 +27,118 @@ import tqdm import click from slugify import slugify +@click.command() +@click.argument('input_gerbers') +@click.argument('output_gerbers') +@click.option('-t', '--top', help='Top side SVG overlay') +@click.option('-b', '--bottom', help='Bottom side SVG overlay') +@click.option('--bbox', help='Output file bounding box. Format: "w,h" to force [w] mm by [h] mm output canvas OR ' + '"x,y,w,h" to force [w] mm by [h] mm output canvas with its bottom left corner at the given input gerber ' + 'coördinates. MUST MATCH --bbox GIVEN TO PREVIEW') +@click.option('--dilate', default=0.1, help='Default dilation for subtraction operations in mm') +@click.option('--no-subtract', 'no_subtract', flag_value=True, help='Disable subtraction') +@click.option('--subtract', help='Use user subtraction script from argument (see description above)') +@click.option('--mask-clips-silk/--silk-clips-mask', help='Set clipping order of mask and silk') +@click.option('--copper-clips-copper/--no-copper-clips-copper', help='Set whether output copper features clip input copper features') +@click.option('--trace-space', type=float, default=0.1, help='passed through to svg-flatten') +@click.option('--vectorizer', help='passed through to svg-flatten') +@click.option('--vectorizer-map', help='passed through to svg-flatten') +@click.option('--exclude-groups', help='passed through to svg-flatten') +def paste_vectors(input_gerbers, output_gerbers, top, bottom, + bbox, + dilate, no_subtract, subtract, mask_clips_silk, copper_clips_copper, + trace_space, vectorizer, vectorizer_map, exclude_groups): + #TODO: describe subtraction script + """ """ + + subtract_map = parse_subtract_script(subtract, dilate) + + if not top and not bottom: + raise click.UsageError('Either --top or --bottom must be given') + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = Path(tmpdir) + output_gerbers = Path(output_gerbers) + source = unpack_if_necessary(Path(input_gerbers), tmpdir) + matches = match_gerbers_in_dir(source) + + # FIXME add single-file, zip support + output_gerbers.mkdir(exist_ok=True) + + for side, in_svg in [('top', top), ('bottom', bottom)]: + print() + print('#########################################') + print('processing side', side, 'infile', in_svg) + print('#########################################') + print() + if not in_svg: + print('no in_svg') + continue + + if not matches[side]: + warnings.warn(f'No input gerber files found for {side} side') + continue + + try: + units, layers = load_side(matches[side]) + except SystemError as e: + raise click.UsageError(e.args) + + print('loaded layers:', list(layers.keys())) + + bounds = get_bounds(bbox, layers) + print('bounds:', bounds) + + @functools.lru_cache() + def do_dilate(layer, amount): + print('dilating', layer, 'by', amount) + outfile = Path('debug') / f'dilated-{layer}-{amount}.gbr' # FIXME + # outfile = tmpdir / 'dilated-{layer}-{amount}.gbr' + dilate_gerber(layers, layer, amount, bbox, tmpdir, outfile, units) + return gerberex.read(str(outfile)) + + for layer, input_files in layers.items(): + if layer == 'drill': + continue + + (in_grb_path, in_grb), = input_files + + print() + print('-----------------------------------------') + print('processing side', side, 'layer', layer) + print('-----------------------------------------') + print() + print('rendering layer', layer) + overlay_file = tmpdir / f'overlay-{side}-{layer}.gbr' + svg_to_gerber(in_svg, overlay_file, layer, trace_space, vectorizer, vectorizer_map, exclude_groups) + + overlay_grb = gerberex.read(str(overlay_file)) + if not overlay_grb.primitives: + print(f'Overlay layer {layer} does not contain anything. Skipping.', file=sys.stderr) + continue + + print('compositing') + comp = gerberex.GerberComposition() + foo = gerberex.rs274x.GerberFile.from_gerber_file(in_grb.cam_source) + foo.offset(-bounds[0][0], -bounds[1][0]) + comp.merge(foo) + comp.merge(overlay_grb) + dilations = subtract_map.get(layer, []) + for d_layer, amount in dilations: + print('processing dilation', d_layer, amount) + dilated = do_dilate(d_layer, amount) + comp.merge(dilated) + + this_out = output_gerbers / in_grb_path.name + print('dumping to', this_out) + comp.dump(this_out) + + for in_file in source.iterdir(): + out_cand = output_gerbers / in_file.name + if not out_cand.is_file(): + print(f'Input file {in_file.name} remained unprocessed. Copying.', file=sys.stderr) + shutil.copy(in_file, out_cand) + @click.command() @click.argument('input') @click.option('-t' ,'--top', help='Top layer output file.') @@ -53,105 +169,34 @@ def render_preview(input, top, bottom, bbox, vector, raster_dpi): 'top': Path(top) if top else None, 'bottom': Path(bottom) if bottom else None } - # If source is not a directory with gerber files (-> zip/single gerber), make it one - if not source.is_dir(): - tmp_indir = tmpdir / 'input' - tmp_indir.mkdir() - - if source.suffix.lower() == '.zip' or is_zipfile(source): - with ZipFile(source) as f: - f.extractall(path=tmp_indir) - - else: # single input file - shutil.copy(source, tmp_indir) - - source = tmp_indir - # source now is a directory with gerber files. - - bounds = None - if bbox: - elems = [ int(elem) for elem in re.split('[,/ ]', bbox) ] - if len(elems) not in (2, 4): - raise click.BadParameter( - '--bbox must be either two floating-point values like: w,h or four like: x,y,w,h') - - elems = [ float(e) for e in elems ] - - if len(elems) == 2: - bounds = [0, 0, *elems] - else: - bounds = elems - - # now transform bounds to the format pcb-tools uses. Instead of (x, y, w, h) or even (x1, y1, x2, y2), that - # is ((x1, x2), (y1, y2) - - x, y, w, h = bounds - bounds = ((x, x+w), (y, y+h)) - + source = unpack_if_necessary(source, tmpdir) matches = match_gerbers_in_dir(source) - for side in ('top', 'bottom'): - flattened = [ e for for_this_layer in matches[side].values() for e in for_this_layer ] + for side in ('top', 'bottom'): if not outfiles[side]: continue - if not flattened: + if not matches[side]: warnings.warn(f'No input gerber files found for {side} side') continue - def load(layer, path): - print('loading', layer, 'layer from:', path) - grb = gerber.load_layer(str(path)) - grb.layer_class = LAYER_CLASSES.get(layer, 'unknown') - return grb - - layers = { layer: [ load(layer, path) for path in files ] - for layer, files in matches[side].items() - if files } - - for layer, elems in layers.items(): - if len(elems) > 1 and layer != 'drill': - raise click.UsageError(f'Multiple files found for layer {layer}: {", ".join(matches[side][layer]) }') - - unitses = set(layer.cam_source.units for items in layers.values() for layer in items) - if len(unitses) != 1: - # FIXME: we should ideally be able to deal with this. We'll have to figure out a way to update a - # GerberCairoContext's scale in between layers. - raise SystemError('Input gerber files mix metric and imperial units. Please fix your export.') - units, = unitses + try: + units, layers = load_side(matches[side]) + except SystemError as e: + raise click.UsageError(e.args) # cairo-svg uses a hardcoded dpi value of 72. pcb-tools does something weird, so we have to scale things # here. scale = 1/25.4 if units == 'metric' else 1.0 # pcb-tools gerber scale - # this is fixed, we cannot tell cairo-svg to use some other value. we just have to work around it. - CAIRO_SVG_HARDCODED_DPI = 72.0 - - scale *= CAIRO_SVG_HARDCODED_DPI # cairo-svg dpi + scale *= CAIRO_SVG_HARDCODED_DPI if not vector: # adapt scale for png export scale *= raster_dpi / CAIRO_SVG_HARDCODED_DPI - # NOTE: When the user has not set explicit bounds, we automatically extract the design's bounding box from - # the input gerber files. If a folder is used as input, we use the outline gerber and barf if we can't find - # one. If only a single file is given, we simply use that file's bounding box - # - # We have to do things this way since gerber files do not have explicit bounds listed. - # - # Note that the bounding box extracted from the outline layer usually will be one outline layer stroke widht - # larger in all directions than the finished board. - if not bounds: - if 'outline' in layers: - bounds = calculate_apertureless_bounding_box(layers['outline'][0].cam_source) - - elif len(flattened) == 1: - bounds = flattened[0].cam_source.bounding_box - - else: - raise click.UsageError('Cannot find an outline file and no --bbox given.') - + bounds = get_bounds(bbox, layers) ctx = GerberCairoContext(scale=scale) for layer_name in LAYER_RENDER_ORDER: - for to_render in layers.get(layer_name, ()): + for _path, to_render in layers.get(layer_name, ()): ctx.render_layer(to_render, bounds=bounds) filetype = 'svg' if vector else 'png' @@ -172,6 +217,100 @@ def render_preview(input, top, bottom, bbox, vector, raster_dpi): with open(outfiles[side], 'w') as f: f.write(template_svg_for_png(bounds, png_data)) +# Subtraction script handling +#============================ + +DEFAULT_SUB_SCRIPT = ''' +out.silk -= in.mask +''' + +def parse_subtract_script(script, default_dilation=0.1): + if script is None: + script = DEFAULT_SUB_SCRIPT + + subtract_script = {} + lines = script.replace(';', '\n').splitlines() + for line in lines: + line = line.strip() + if not line or line.startswith('#'): + continue + + line = line.lower() + line = re.sub('\s', '', line) + + # out.copper -= in.copper+0.1 + varname = r'([a-z]+\.[a-z]+)' + floatnum = r'([+-][.0-9]+)' + match = re.fullmatch(fr'{varname}-={varname}{floatnum}?', line) + if not match: + raise ValueError(f'Cannot parse line: {line}') + + out_var, in_var, dilation = match.groups() + if not out_var.startswith('out.') or not in_var.startswith('in.'): + raise ValueError('All left-hand side values must be outputs, right-hand side values must be inputs.') + + _out, _, out_layer = out_var.partition('.') + _in, _, in_layer = in_var.partition('.') + + dilation = float(dilation) if dilation else default_dilation + + subtract_script[out_layer] = subtract_script.get(out_layer, []) + [(in_layer, dilation)] + return subtract_script + +# Parameter parsing foo +#====================== + +def parse_bbox(bbox): + if not bbox: + return None + elems = [ int(elem) for elem in re.split('[,/ ]', bbox) ] + if len(elems) not in (2, 4): + raise click.BadParameter( + '--bbox must be either two floating-point values like: w,h or four like: x,y,w,h') + + elems = [ float(e) for e in elems ] + + if len(elems) == 2: + bounds = [0, 0, *elems] + else: + bounds = elems + + # now transform bounds to the format pcb-tools uses. Instead of (x, y, w, h) or even (x1, y1, x2, y2), that + # is ((x1, x2), (y1, y2) + + x, y, w, h = bounds + return ((x, x+w), (y, y+h)) + +def bounds_from_outline(layers): + ''' NOTE: When the user has not set explicit bounds, we automatically extract the design's bounding box from the + input gerber files. If a folder is used as input, we use the outline gerber and barf if we can't find one. If only a + single file is given, we simply use that file's bounding box + + We have to do things this way since gerber files do not have explicit bounds listed. + + Note that the bounding box extracted from the outline layer usually will be one outline layer stroke widht larger in + all directions than the finished board. + ''' + if 'outline' in layers: + outline_files = layers['outline'] + _path, grb = outline_files[0] + return calculate_apertureless_bounding_box(grb.cam_source) + + elif len(layers) == 1: + first_layer, *rest = layers.values() + first_file, *rest = first_layer + _path, grb = first_file + return grb.cam_source.bounding_box + + else: + raise click.UsageError('Cannot find an outline file and no --bbox given.') + +def get_bounds(bbox, layers): + bounds = parse_bbox(bbox) + if bounds: + return bounds + return bounds_from_outline(layers) + # Utility foo # =========== @@ -181,7 +320,7 @@ def render_preview(input, top, bottom, bbox, vector, raster_dpi): LAYER_SPEC = { 'top': { 'paste': '.gtp|-F_Paste.gbr|-F.Paste.gbr|.pmc', - 'silk': '.gto|-F_SilkS.gbr|-F.SilkS.gbr|.plc', + 'silk': '.gto|-F_Silkscreen.gbr|-F_SilkS.gbr|-F.SilkS.gbr|.plc', 'mask': '.gts|-F_Mask.gbr|-F.Mask.gbr|.stc', 'copper': '.gtl|-F_Cu.gbr|-F.Cu.gbr|.cmp', 'outline': '.gko|.gm1|-Edge_Cuts.gbr|-Edge.Cuts.gbr|.gmb', @@ -189,7 +328,7 @@ LAYER_SPEC = { }, 'bottom': { 'paste': '.gbp|-B_Paste.gbr|-B.Paste.gbr|.pms', - 'silk': '.gbo|-B_SilkS.gbr|-B.SilkS.gbr|.pls', + 'silk': '.gbo|-B_Silkscreen.gbr|-B_SilkS.gbr|-B.SilkS.gbr|.pls', 'mask': '.gbs|-B_Mask.gbr|-B.Mask.gbr|.sts', 'copper': '.gbl|-B_Cu.gbr|-B.Cu.gbr|.sol', 'outline': '.gko|.gm1|-Edge_Cuts.gbr|-Edge.Cuts.gbr|.gmb', @@ -214,7 +353,9 @@ def match_gerbers_in_dir(path): for side, layers in LAYER_SPEC.items(): out[side] = {} for layer, match in layers.items(): - out[side][layer] = list(find_gerber_in_dir(path, match)) + l = list(find_gerber_in_dir(path, match)) + if l: + out[side][layer] = l return out def find_gerber_in_dir(path, extensions): @@ -245,6 +386,51 @@ def calculate_apertureless_bounding_box(cam): return ((min_x, max_x), (min_y, max_y)) +def unpack_if_necessary(source, tmpdir, dirname='input'): + """ Handle command-line input paths. If path points to a directory, return unchanged. If path points to a zip file, + unpack to a directory inside tmpdir and return that. If path points to a file that is not a zip, copy that file into + a subdir of tmpdir and return that subdir. """ + # If source is not a directory with gerber files (-> zip/single gerber), make it one + if not source.is_dir(): + tmp_indir = tmpdir / dirname + tmp_indir.mkdir() + + if source.suffix.lower() == '.zip' or is_zipfile(source): + with ZipFile(source) as f: + f.extractall(path=tmp_indir) + + else: # single input file + shutil.copy(source, tmp_indir) + + return tmp_indir + + else: + return source + +def load_side(side_matches): + """ Load all gerber files for one side returned by match_gerbers_in_dir. """ + def load(layer, path): + print('loading', layer, 'layer from:', path) + grb = gerber.load_layer(str(path)) + grb.layer_class = LAYER_CLASSES.get(layer, 'unknown') + return grb + + layers = { layer: [ (path, load(layer, path)) for path in files ] + for layer, files in side_matches.items() } + + for layer, elems in layers.items(): + if len(elems) > 1 and layer != 'drill': + raise SystemError(f'Multiple files found for layer {layer}: {", ".join(side_matches[layer]) }') + + unitses = set(layer.cam_source.units for items in layers.values() for _path, layer in items) + if len(unitses) != 1: + # FIXME: we should ideally be able to deal with this. We'll have to figure out a way to update a + # GerberCairoContext's scale in between layers. + raise SystemError('Input gerber files mix metric and imperial units. Please fix your export.') + units, = unitses + + return units, layers + # SVG export #=========== @@ -277,9 +463,11 @@ def template_svg_for_png(bounds, png_data, extra_layers=DEFAULT_EXTRA_LAYERS): ''' return textwrap.dedent(template) +# this is fixed, we cannot tell cairo-svg to use some other value. we just have to work around it. +CAIRO_SVG_HARDCODED_DPI = 72.0 MM_PER_INCH = 25.4 -def svg_pt_to_mm(pt_len, dpi=72.0): +def svg_pt_to_mm(pt_len, dpi=CAIRO_SVG_HARDCODED_DPI): if pt_len.endswith('pt'): pt_len = pt_len[:-2] @@ -289,7 +477,7 @@ def create_template_from_svg(bounds, svg_data, extra_layers=DEFAULT_EXTRA_LAYERS svg = etree.fromstring(svg_data) # add inkscape namespaces - NS = '{http://www.w3.org/2000/svg}' + SVG_NS = '{http://www.w3.org/2000/svg}' INKSCAPE_NS = 'http://www.inkscape.org/namespaces/inkscape' SODIPODI_NS = 'http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd' # glObAL stAtE YaY @@ -303,7 +491,7 @@ def create_template_from_svg(bounds, svg_data, extra_layers=DEFAULT_EXTRA_LAYERS svg.set('height', svg_pt_to_mm(svg.get('height'))) # make original group an inkscape layer - orig_g = svg.find(NS+'g') + orig_g = svg.find(SVG_NS+'g') orig_g.set('id', 'g-preview') orig_g.set(INKSCAPE_NS+'label', 'Preview') orig_g.set(SODIPODI_NS+'insensitive', 'true') # lock group @@ -311,12 +499,83 @@ def create_template_from_svg(bounds, svg_data, extra_layers=DEFAULT_EXTRA_LAYERS # add layers for layer in extra_layers: - new_g = etree.SubElement(svg, NS+'g') + new_g = etree.SubElement(svg, SVG_NS+'g') new_g.set('id', f'g-{slugify(layer)}') new_g.set(INKSCAPE_NS+'label', layer) new_g.set(INKSCAPE_NS+'groupmode', 'layer') return etree.tostring(svg) +# SVG/gerber import +#================== + +def dilate_gerber(layers, layer_name, dilation, bbox, tmpdir, outfile, units): + # render gerber to SVG + scale = 1/25.4 if units == 'metric' else 1.0 # pcb-tools gerber scale + scale *= CAIRO_SVG_HARDCODED_DPI + + bounds = get_bounds(bbox, layers) + ctx = GerberCairoContext(scale=scale) + for _path, to_render in layers.get(layer_name, ()): + ctx.render_layer(to_render, bounds=bounds, + settings=gerber.render.RenderSettings(color=(0,0,0)), # FIXME should be 1, 1, 1 + bgsettings=gerber.render.RenderSettings(color=(0,0,0), alpha=0)) + + tmpfile = tmpdir / 'dilate-tmp.svg' + ctx.dump(str(tmpfile)) + + # FIXME DEBUG + import uuid + fn = Path('debug') / f'in-{uuid.uuid4()}.svg' + shutil.copy(tmpfile, fn) + print('tmp debug dilation svg:', fn) + + # dilate & render back to gerber + svg_to_gerber(tmpfile, outfile, dilate=dilation) + +def svg_to_gerber(infile, outfile, layer=None, trace_space:'mm'=0.1, vectorizer=None, vectorizer_map=None, exclude_groups=None, dilate=None): + if 'SVG_FLATTEN' in os.environ: + candidates = [os.environ['SVG_FLATTEN']] + + else: + # By default, try three options: + candidates = [ + # somewhere in $PATH + 'svg-flatten', + # next to our current python interpreter (e.g. in virtualenv + str(Path(sys.executable).parent / 'svg-flatten'), + # next to this python source file in the development repo + str(Path(__file__).parent.parent / 'svg-flatten' / 'build' / 'svg-flatten') ] + + args = [ '--format', 'gerber', + '--precision', '6', # intermediate file, use higher than necessary precision + '--trace-space', str(trace_space) ] + if layer: + args += ['--only-groups', f'g-{slugify(layer)}'] + if vectorizer: + args += ['--vectorizer', vectorizer] + if vectorizer_map: + args += ['--vectorizer-map', vectorizer_map] + if exclude_groups: + args += ['--exclude-groups', exclude_groups] + if dilate: + args += ['--dilate', str(dilate)] + + args += [str(infile), str(outfile)] + print('full args:', " ".join(args)) + + for candidate in candidates: + try: + print('trying', candidate) + res = subprocess.run([candidate, *args], check=True) + break + except FileNotFoundError: + print('fail') + continue + else: + raise SystemError('svg-flatten executable not found') + + if __name__ == '__main__': - render_preview() + #render_preview() + paste_vectors() -- cgit