From be8371c7bc21abe12db49f9dd38dac4513a64886 Mon Sep 17 00:00:00 2001 From: jaseg Date: Wed, 6 Nov 2024 14:49:50 +0100 Subject: Improve allegro/orcad import --- gerbonara/cam.py | 3 + gerbonara/cli.py | 2 +- gerbonara/excellon.py | 46 +++++++++++---- gerbonara/layer_rules.py | 16 ++++-- gerbonara/layers.py | 126 ++++++++++++++++++++++++++++++++--------- gerbonara/rs274x.py | 7 ++- gerbonara/tests/test_layers.py | 16 +++++- 7 files changed, 167 insertions(+), 49 deletions(-) (limited to 'gerbonara') diff --git a/gerbonara/cam.py b/gerbonara/cam.py index e6bb651..14c1e8b 100644 --- a/gerbonara/cam.py +++ b/gerbonara/cam.py @@ -60,6 +60,9 @@ class FileSettings: #: If you want to export the macros with their original formulaic expressions (which is completely fine by the #: Gerber standard, btw), set this parameter to ``False`` before exporting. calculate_out_all_aperture_macros: bool = True + #: Internal field used to communicate if only decimal coordinates were found inside an Excellon file, or if it + #: contained at least some coordinates in fixed-width notation. + _file_has_fixed_width_coordinates: bool = False # input validation def __setattr__(self, name, value): diff --git a/gerbonara/cli.py b/gerbonara/cli.py index 93b989e..22aa098 100644 --- a/gerbonara/cli.py +++ b/gerbonara/cli.py @@ -431,7 +431,7 @@ def merge(inpath, outpath, offset, rotation, input_map, command_line_units, outp @click.option('--version', is_flag=True, callback=_print_version, expose_value=False, is_eager=True) @click.option('--warnings', 'format_warnings', type=click.Choice(['default', 'ignore', 'once']), default='default', help='''Enable or disable file format warnings during parsing (default: on)''') -@click.option('--units', type=Unit(), help='Output bounding box in this unit (default: millimeter)') +@click.option('--units', type=Unit(), default='metric', help='Output bounding box in this unit (default: millimeter)') @click.option('--input-number-format', help='Override number format of input file (mostly useful for Excellon files)') @click.option('--input-units', type=Unit(), help='Override units of input file') @click.option('--input-zero-suppression', type=click.Choice(['off', 'leading', 'trailing']), help='Override zero suppression setting of input file') diff --git a/gerbonara/excellon.py b/gerbonara/excellon.py index 9b75a69..73f9592 100755 --- a/gerbonara/excellon.py +++ b/gerbonara/excellon.py @@ -566,6 +566,8 @@ class ExcellonParser(object): self.filename = None self.external_tools = external_tools or {} self.found_kicad_format_comment = False + self.allegro_eof_toolchange_hack = False + self.allegro_eof_toolchange_hack_index = 1 def warn(self, msg): warnings.warn(f'{self.filename}:{self.lineno} "{self.line}": {msg}', SyntaxWarning) @@ -606,18 +608,25 @@ class ExcellonParser(object): exprs = RegexMatcher() # NOTE: These must be kept before the generic comment handler at the end of this class so they match first. - @exprs.match(r';T(?P[0-9]+) Holesize (?P[0-9]+)\. = (?P[0-9/.]+) Tolerance = \+[0-9/.]+/-[0-9/.]+ (?PPLATED|NON_PLATED|OPTIONAL) (?PMILS|MM) Quantity = [0-9]+') + @exprs.match(r';(?PT(?P[0-9]+))?\s+Holesize (?P[0-9]+)\. = (?P[0-9/.]+) Tolerance = \+[0-9/.]+/-[0-9/.]+ (?PPLATED|NON_PLATED|OPTIONAL) (?PMILS|MM) Quantity = [0-9]+') def parse_allegro_tooldef(self, match): # NOTE: We ignore the given tolerances here since they are non-standard. self.program_state = ProgramState.HEADER # TODO is this needed? we need a test file. self.generator_hints.append('allegro') - if (index := int(match['index1'])) != int(match['index2']): # index1 has leading zeros, index2 not. + index = int(match['index2']) + + if match['index1'] and index != int(match['index1']): # index1 has leading zeros, index2 not. raise SyntaxError('BUG: Allegro excellon tool def has mismatching tool indices. Please file a bug report on our issue tracker and provide this file!') if index in self.tools: self.warn('Re-definition of tool index {index}, overwriting old definition.') + if not match['index1_prefix']: + # This is a really nasty orcad file without tool change commands, that instead just puts all holes in order + # of the hole size definitions with M00's in between. + self.allegro_eof_toolchange_hack = True + # NOTE: We map "optionally" plated holes to plated holes for API simplicity. If you hit a case where that's a # problem, please raise an issue on our issue tracker, explain why you need this and provide an example file. is_plated = None if match['plated'] is None else (match['plated'] in ('PLATED', 'OPTIONAL')) @@ -630,13 +639,19 @@ class ExcellonParser(object): else: unit = MM - if unit != self.settings.unit: + if self.settings.unit is None: + self.settings.unit = unit + + elif unit != self.settings.unit: self.warn('Allegro Excellon drill file tool definitions in {unit.name}, but file parameters say the ' 'file should be in {settings.unit.name}. Please double-check that this is correct, and if it is, ' 'please raise an issue on our issue tracker.') self.tools[index] = ExcellonTool(diameter=diameter, plated=is_plated, unit=unit) + if self.allegro_eof_toolchange_hack and self.active_tool is None: + self.active_tool = self.tools[index] + # Searching Github I found that EasyEDA has two different variants of the unit specification here. @exprs.match(';Holesize (?P[0-9]+) = (?P[.0-9]+) (?PINCH|inch|METRIC|mm)') def parse_easyeda_tooldef(self, match): @@ -753,6 +768,12 @@ class ExcellonParser(object): def handle_end_of_program(self, match): if self.program_state in (None, ProgramState.HEADER): self.warn('M30 statement found before end of header.') + + if self.allegro_eof_toolchange_hack: + self.allegro_eof_toolchange_hack_index = min(max(self.tools), self.allegro_eof_toolchange_hack_index + 1) + self.active_tool = self.tools[self.allegro_eof_toolchange_hack_index] + return + self.program_state = ProgramState.FINISHED # TODO: maybe add warning if this is followed by other commands. @@ -762,14 +783,17 @@ class ExcellonParser(object): def do_move(self, coord_groups): x_s, x, y_s, y = coord_groups - if self.settings.number_format == (None, None) and '.' not in x: - # TARGET3001! exports zeros as "00" even when it uses an explicit decimal point everywhere else. - if x != '00': - raise SyntaxError('No number format set and value does not contain a decimal point. If this is an Allegro ' - 'Excellon drill file make sure either nc_param.txt or ncdrill.log ends up in the same folder as ' - 'it, because Allegro does not include this critical information in their Excellon output. If you ' - 'call this through ExcellonFile.from_string, you must manually supply from_string with a ' - 'FileSettings object from excellon.parse_allegro_ncparam.') + if '.' not in x: + self.settings._file_has_fixed_width_coordinates = True + + if self.settings.number_format == (None, None): + # TARGET3001! exports zeros as "00" even when it uses an explicit decimal point everywhere else. + if x != '00': + raise SyntaxError('No number format set and value does not contain a decimal point. If this is an Allegro ' + 'Excellon drill file make sure either nc_param.txt or ncdrill.log ends up in the same folder as ' + 'it, because Allegro does not include this critical information in their Excellon output. If you ' + 'call this through ExcellonFile.from_string, you must manually supply from_string with a ' + 'FileSettings object from excellon.parse_allegro_ncparam.') x = self.settings.parse_gerber_value(x) if x_s: diff --git a/gerbonara/layer_rules.py b/gerbonara/layer_rules.py index aae52ae..eb3e0e5 100644 --- a/gerbonara/layer_rules.py +++ b/gerbonara/layer_rules.py @@ -82,6 +82,7 @@ MATCH_RULES = { 'bottom paste': r'.*_boardoutline\.\w+', # FIXME verify this 'drill plated': r'.*\.(drl)', # diptrace has unplated drills on the outline layer 'other netlist': r'.*\.ipc', # default rule due to lack of tool-specific examples + 'header regex': [['sufficient', r'top .*|bottom .*', r'G04 DipTrace [.-0-9a-z]*\*']], }, 'target': { @@ -151,22 +152,25 @@ MATCH_RULES = { 'allegro': { # Allegro doesn't have any widespread convention, so we rely heavily on the layer name auto-guesser here. - 'drill mech': r'.*\.(drl|rou)', - 'generic gerber': r'.*\.art', + 'drill plated': r'.*\.(drl)', + 'drill nonplated': r'.*\.(rou)', + 'other unknown': r'.*(place|assembly|keep.?in|keep.?out).*\.art', + 'autoguess': r'.*\.art', 'excellon params': r'nc_param\.txt|ncdrill\.log|ncroute\.log', 'other netlist': r'.*\.ipc', # default rule due to lack of tool-specific examples + 'header regex': [['required,sufficient', r'.*\.art', r'G04 File Origin:\s+Cadence Allegro [0-9]+\.[0-9]+[-a-zA-Z0-9]*']], }, 'pads': { # Pads also does not seem to have a factory-default naming schema. Or it has one but everyone ignores it. - 'generic gerber': r'.*\.pho', - 'drill mech': r'.*\.drl', + 'autoguess': r'.*\.pho', + 'drill plated': r'.*\.drl', }, 'zuken': { - 'generic gerber': r'.*\.fph', + 'autoguess': r'.*\.fph', 'gerber params': r'.*\.fpl', - 'drill mech': r'.*\.fdr', + 'drill unknown': r'.*\.fdr', 'excellon params': r'.*\.fdl', 'other netlist': r'.*\.ipc', 'ipc-2581': r'.*\.xml', diff --git a/gerbonara/layers.py b/gerbonara/layers.py index 4b9d360..e200e62 100644 --- a/gerbonara/layers.py +++ b/gerbonara/layers.py @@ -112,31 +112,61 @@ class NamingScheme: } +def apply_rules(filenames, rules): + certain = False + gen = {} + already_matched = set() + header_regex = rules.pop('header regex', []) + header_regex_matched = [False] * len(header_regex) + + file_headers = {} + def get_header(path): + if path not in file_headers: + with open(path) as f: + file_headers[path] = f.read(16384) + return file_headers[path] + + for layer, regex in rules.items(): + for fn in filenames: + if fn in already_matched: + continue -def _match_files(filenames): - matches = {} - for generator, rules in MATCH_RULES.items(): - already_matched = set() - gen = {} - matches[generator] = gen - for layer, regex in rules.items(): - for fn in filenames: - if fn in already_matched: - continue + target = None + if (m := re.fullmatch(regex, fn.name, re.IGNORECASE)): + if layer == 'inner copper': + target = 'inner_' + ''.join(e or '' for e in m.groups()) + ' copper' + else: + target = layer - if (m := re.fullmatch(regex, fn.name, re.IGNORECASE)): - if layer == 'inner copper': - target = 'inner_' + ''.join(e or '' for e in m.groups()) + ' copper' - else: - target = layer + gen[target] = gen.get(target, []) + [fn] + already_matched.add(fn) - gen[target] = gen.get(target, []) + [fn] - already_matched.add(fn) - return matches + for i, (match_type, layer_match, header_match) in enumerate(header_regex): + if re.fullmatch(layer_match, fn.name, re.IGNORECASE) or ( + target is not None and re.fullmatch(layer_match, target, re.IGNORECASE)): + if re.search(header_match, get_header(fn)): + if 'sufficient' in match_type: + certain = True + + header_regex_matched[i] = True + + if any('required' in match_type and not match + for match, (match_type, *_) in zip(header_regex_matched, header_regex)): + return False, {} + + return certain, gen def _best_match(filenames): - matches = _match_files(filenames) + matches = {} + for generator, rules in MATCH_RULES.items(): + certain, candidate = apply_rules(filenames, rules) + + if certain: + return generator, candidate + + matches[generator] = candidate + matches = sorted(matches.items(), key=lambda pair: len(pair[1])) generator, files = matches[-1] return generator, files @@ -243,7 +273,7 @@ def _layername_autoguesser(fn): elif re.search('film', fn): use = 'copper' - elif re.search('out(line)?', fn): + elif re.search('out(line)?|board.?geometry', fn): use = 'outline' side = 'mechanical' @@ -385,7 +415,7 @@ class LayerStack: with ZipFile(file) as f: f.extractall(path=tmp_indir) - inst = kls.open_dir(tmp_indir, board_name=board_name, lazy=lazy) + inst = kls.open_dir(tmp_indir, board_name=board_name, lazy=lazy, overrides=overrides, autoguess=autoguess) inst.tmpdir = tmpdir inst.original_path = Path(original_path or file) inst.was_zipped = True @@ -445,6 +475,11 @@ class LayerStack: filemap[layer].remove(fn) filemap[layer] = filemap.get(layer, []) + [fn] + if 'autoguess' in filemap: + warnings.warn(f'This generator ({generator}) often exports ambiguous filenames. Falling back to autoguesser for some files. Use at your own peril.') + for key, values in _do_autoguess(filemap.pop('autoguess')).items(): + filemap[key] = filemap.get(key, []) + values + if sum(len(files) for files in filemap.values()) < 6 and autoguess: warnings.warn('Ambiguous gerber filenames. Trying last-resort autoguesser.') generator = None @@ -453,6 +488,8 @@ class LayerStack: raise ValueError('Cannot figure out gerber file mapping. Partial map is: ', filemap) excellon_settings, external_tools = None, None + automatch_drill_scale = False + if generator == 'geda': # geda is written by geniuses who waste no bytes of unnecessary output so it doesn't actually include the # number format in files that use imperial units. Unfortunately it also doesn't include any hints that the @@ -470,16 +507,18 @@ class LayerStack: if (external_tools := parse_allegro_logfile(file.read_text())): break del filemap['excellon params'] - # Ignore if we can't find the param file -- maybe the user has convinced Allegro to actually put this - # information into a comment, or maybe they have made Allegro just use decimal points like XNC does. + else: + # Ignore if we can't find the param file -- maybe the user has convinced Allegro to actually put this + # information into a comment, or maybe they have made Allegro just use decimal points like XNC does. + # We'll run an automatic scale matching later. + excellon_settings = FileSettings(number_format=(2, 4)) + automatch_drill_scale = True - filemap = _do_autoguess([ f for files in filemap.values() for f in files ]) if len(filemap) < 6: raise SystemError('Cannot figure out gerber file mapping') # FIXME use layer metadata from comments and ipc file if available elif generator == 'zuken': - filemap = _do_autoguess([ f for files in filemap.values() for f in files ]) if len(filemap) < 6: raise SystemError('Cannot figure out gerber file mapping') # FIXME use layer metadata from comments and ipc file if available @@ -503,7 +542,9 @@ class LayerStack: else: excellon_settings = None - ambiguous = [ f'{key} ({", ".join(x.name for x in value)})' for key, value in filemap.items() if len(value) > 1 and not 'drill' in key ] + ambiguous = [ f'{key} ({", ".join(x.name for x in value)})' + for key, value in filemap.items() + if len(value) > 1 and not 'drill' in key and not key == 'other unknown'] if ambiguous: raise SystemError(f'Ambiguous layer names: {", ".join(ambiguous)}') @@ -512,8 +553,8 @@ class LayerStack: netlist = None layers = {} # { tuple(key.split()): None for key in STANDARD_LAYERS } for key, paths in filemap.items(): - if len(paths) > 1 and not 'drill' in key: - raise ValueError(f'Multiple matching files found for {key} layer: {", ".join(value)}') + if len(paths) > 1 and not 'drill' in key and not key == 'other unknown': + raise ValueError(f'Multiple matching files found for {key} layer: {", ".join(map(str, value))}') for path in paths: id_result = identify_file(path.read_text()) @@ -574,6 +615,35 @@ class LayerStack: board_name = re.sub(r'^\W+', '', board_name) board_name = re.sub(r'\W+$', '', board_name) + if automatch_drill_scale: + top_copper = layers[('top', 'copper')].to_excellon(errors='ignore', holes_only=True) + + # precision is matching precision in mm + def map_coords(obj, precision=0.01, scale=1): + obj = obj.converted(MM) + return round(obj.x*scale/precision), round(obj.y*scale/precision) + + aper_coords = {map_coords(obj) for obj in top_copper.drills()} + + for drill_file in [drill_pth, drill_npth, *drill_layers]: + if not drill_file or not drill_pth.import_settings._file_has_fixed_width_coordinates: + continue + + scale_matches = {} + for exp in range(-6, 6): + scale = 10**exp + hole_coords = {map_coords(obj, scale=scale) for obj in drill_file.drills()} + + scale_matches[scale] = len(aper_coords - hole_coords), len(hole_coords - aper_coords) + scales_out = [(max(a, b), scale) for scale, (a, b) in scale_matches.items()] + _matches, scale = sorted(scales_out)[0] + warnings.warn(f'Performing automatic alignment of poorly exported drill layer. Scale matching results: {scale_matches}. Chosen scale: {scale}') + + # Note: This is only used with allegro files, which use decimal points and explicit units in their tool + # definitions. Thus, we only scale object coordinates, and not apertures. + for obj in drill_file.objects: + obj.scale(scale) + return kls(layers, drill_pth, drill_npth, drill_layers, board_name=board_name, original_path=original_path, was_zipped=was_zipped, generator=[*all_generator_hints, None][0]) diff --git a/gerbonara/rs274x.py b/gerbonara/rs274x.py index fd83c5b..108ccd6 100644 --- a/gerbonara/rs274x.py +++ b/gerbonara/rs274x.py @@ -152,7 +152,7 @@ class GerberFile(CamFile): self.map_apertures(lookup) - def to_excellon(self, plated=None, errors='raise'): + def to_excellon(self, plated=None, errors='raise', holes_only=False): """ Convert this excellon file into a :py:class:`~.excellon.ExcellonFile`. This will convert interpolated lines into slots, and circular aperture flashes into holes. Other features such as ``G36`` polygons or flashes with non-circular apertures will result in a :py:obj:`ValueError`. You can, of course, programmatically remove such @@ -160,7 +160,10 @@ class GerberFile(CamFile): new_objs = [] new_tools = {} for obj in self.objects: - if not (isinstance(obj, go.Line) or isinstance(obj, go.Arc) or isinstance(obj, go.Flash)) or \ + if holes_only and not isinstance(obj, go.Flash): + continue + + if not isinstance(obj, (go.Line, go.Arc, go.Flash)) or \ not isinstance(getattr(obj, 'aperture', None), apertures.CircleAperture): if errors == 'raise': raise ValueError(f'Cannot convert {obj} to excellon.') diff --git a/gerbonara/tests/test_layers.py b/gerbonara/tests/test_layers.py index a5103de..b473f91 100644 --- a/gerbonara/tests/test_layers.py +++ b/gerbonara/tests/test_layers.py @@ -302,7 +302,21 @@ REFERENCE_DIRS = { 'Drill/8seg_Driver__routed_Drill_thru_plt.fdr/8seg_Driver__routed_Drill_thru_plt.fdl': None, 'Drill/8seg_Driver__routed_Drill_thru_plt.fdr/8seg_Driver__routed_Drill_thru_plt.fdr': 'drill plated', 'Drill/8seg_Driver__routed_Drill_thru_nplt.fdr': 'drill nonplated', - } + }, + 'orcad': { + 'Assembly.art': None, + 'BOTTOM.art': 'bottom copper', + 'GND2.art': 'inner_3 copper', + 'LAYER_1.art': 'inner_2 copper', + 'LAYER_2.art': 'inner_4 copper', + 'PWR.art': 'inner_2 copper', + 'Solder_Mask_Bottom.art': 'bottom mask', + 'Solder_Mask_Top.art': 'top mask', + 'TOP.art': 'top copper', + 'arena_12-12_v6_L1-L6.drl': 'drill plated', + 'silk_screen_bottom.art': 'bottom silk', + 'silk_screen_top.art': 'top silk', + }, } @filter_syntax_warnings -- cgit