summaryrefslogtreecommitdiff
path: root/gerbonara
diff options
context:
space:
mode:
authorjaseg <git@jaseg.de>2024-07-08 16:29:35 +0200
committerjaseg <git@jaseg.de>2024-07-08 16:29:35 +0200
commitec85d6c169620fd0968693b19b6109fb0184aa36 (patch)
tree5162868780af1a00f6881c6078c1abf41b0add23 /gerbonara
parentf447b12571049da91e726d2bb20e30eff07bac46 (diff)
downloadgerbonara-ec85d6c169620fd0968693b19b6109fb0184aa36.tar.gz
gerbonara-ec85d6c169620fd0968693b19b6109fb0184aa36.tar.bz2
gerbonara-ec85d6c169620fd0968693b19b6109fb0184aa36.zip
tests: Speed up tests by a lot by bulk-caching kicad footprint renders
Diffstat (limited to 'gerbonara')
-rw-r--r--gerbonara/tests/conftest.py35
-rw-r--r--gerbonara/tests/image_support.py36
2 files changed, 61 insertions, 10 deletions
diff --git a/gerbonara/tests/conftest.py b/gerbonara/tests/conftest.py
index ea16217..7d4e996 100644
--- a/gerbonara/tests/conftest.py
+++ b/gerbonara/tests/conftest.py
@@ -1,11 +1,14 @@
import os
from pathlib import Path
+import tqdm
+import multiprocessing.pool
+import subprocess
from itertools import chain
import pytest
-from .image_support import ImageDifference, run_cargo_cmd
+from .image_support import ImageDifference, run_cargo_cmd, bulk_populate_kicad_fp_export_cache
def pytest_assertrepr_compare(op, left, right):
if isinstance(left, ImageDifference) or isinstance(right, ImageDifference):
@@ -24,41 +27,53 @@ def pytest_runtest_makereport(item, call):
fail_dir = Path('gerbonara_test_failures')
def pytest_sessionstart(session):
- if not hasattr(session.config, 'workerinput'): # on worker
+ if 'PYTEST_XDIST_WORKER' in os.environ: # only run this on the controller
return
- # on coordinator
for f in chain(fail_dir.glob('*.gbr'), fail_dir.glob('*.png')):
f.unlink()
try:
- run_cargo_cmd('resvg', '--help')
+ run_cargo_cmd('resvg', '--help', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except FileNotFoundError:
pytest.exit('resvg binary not found, aborting test.', 2)
+
+def pytest_configure(config):
+ if 'PYTEST_XDIST_WORKER' in os.environ: # only run this on the controller
+ return
+
+ if (lib_dir := os.environ.get('KICAD_FOOTPRINTS')):
+ lib_dir = Path(lib_dir).expanduser()
+ if not lib_dir.is_dir():
+ raise ValueError(f'Path "{lib_dir}" given by KICAD_FOOTPRINTS environment variable does not exist or is not a directory.')
+
+ print('Checking and bulk re-building KiCad footprint library cache')
+ with multiprocessing.pool.ThreadPool() as pool: # use thread pool here since we're only monitoring podman processes
+ lib_dirs = list(lib_dir.glob('*.pretty'))
+ res = list(tqdm.tqdm(pool.imap(lambda path: bulk_populate_kicad_fp_export_cache(path), lib_dirs), total=len(lib_dirs)))
+
+
def pytest_addoption(parser):
parser.addoption('--kicad-symbol-library', nargs='*', help='Run symbol library tests on given symbol libraries. May be given multiple times.')
parser.addoption('--kicad-footprint-files', nargs='*', help='Run footprint library tests on given footprint files. May be given multiple times.')
+
def pytest_generate_tests(metafunc):
if 'kicad_library_file' in metafunc.fixturenames:
if not (library_files := metafunc.config.getoption('symbol_library', None)):
if (lib_dir := os.environ.get('KICAD_SYMBOLS')):
lib_dir = Path(lib_dir).expanduser()
- if not lib_dir.is_dir():
- raise ValueError(f'Path "{lib_dir}" given by KICAD_SYMBOLS environment variable does not exist or is not a directory.')
library_files = list(lib_dir.glob('*.kicad_sym'))
else:
- raise ValueError('Either --kicad-symbol-library command line parameter or KICAD_SYMBOLS environment variable must be given.')
+ raise ValueError('Either --kicad-symbol-library command line parameter or KICAD_SYMBOLS environment variable must be given to run kicad symbol tests.')
metafunc.parametrize('kicad_library_file', library_files, ids=list(map(str, library_files)))
if 'kicad_mod_file' in metafunc.fixturenames:
if not (mod_files := metafunc.config.getoption('footprint_files', None)):
if (lib_dir := os.environ.get('KICAD_FOOTPRINTS')):
lib_dir = Path(lib_dir).expanduser()
- if not lib_dir.is_dir():
- raise ValueError(f'Path "{lib_dir}" given by KICAD_FOOTPRINTS environment variable does not exist or is not a directory.')
mod_files = list(lib_dir.glob('*.pretty/*.kicad_mod'))
else:
- raise ValueError('Either --kicad-footprint-files command line parameter or KICAD_FOOTPRINTS environment variable must be given.')
+ raise ValueError('Either --kicad-footprint-files command line parameter or KICAD_FOOTPRINTS environment variable must be given to run kicad footprint tests.')
metafunc.parametrize('kicad_mod_file', mod_files, ids=list(map(str, mod_files)))
diff --git a/gerbonara/tests/image_support.py b/gerbonara/tests/image_support.py
index c5ebee4..efb962a 100644
--- a/gerbonara/tests/image_support.py
+++ b/gerbonara/tests/image_support.py
@@ -23,13 +23,17 @@ from pathlib import Path
import tempfile
import textwrap
import os
+import sys
import stat
+import random
+import statistics
from functools import total_ordering
import shutil
import bs4
from contextlib import contextmanager
import hashlib
+import tqdm
import numpy as np
from PIL import Image
@@ -174,6 +178,38 @@ def kicad_fp_export(mod_file, out_svg):
print(f'Re-using cache for {mod_file.name}')
shutil.copy(cachefile, out_svg)
+
+def bulk_populate_kicad_fp_export_cache(pretty_dir):
+ def cachefile(mod_file):
+ params = f'(noparams)'.encode()
+ digest = hashlib.blake2b(mod_file.read_bytes() + params).hexdigest()
+ return cachedir / f'{digest}.svg'
+
+ mod_files = list(pretty_dir.glob('*.kicad_mod'))
+ hit_rate = statistics.mean([int(cachefile(fn).is_file())
+ for fn in random.sample(mod_files, min(len(mod_files), 50))])
+
+ if hit_rate < 0.9:
+ #tqdm.tqdm.write(f'Modfile cache is out of date (hit rate {hit_rate*100:.0f}%), re-building entire cache in bulk')
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ os.chmod(tmpdir, 0o1777)
+ cmd = ['podman', 'run',
+ '--rm', # Clean up volumes after exit
+ '--userns=keep-id', # To allow container to read from bind mount
+ '--mount', f'type=bind,src={pretty_dir},dst=/{pretty_dir.name}',
+ '--mount', f'type=bind,src={tmpdir},dst=/out',
+ 'registry.hub.docker.com/kicad/kicad:nightly',
+ 'kicad-cli', 'fp', 'export', 'svg', '--output', '/out', f'/{pretty_dir.name}']
+ subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL)
+
+ for fn in mod_files:
+ out_file = Path(tmpdir) / fn.with_suffix('.svg').name
+ if not out_file.is_file():
+ tqdm.tqdm.write(f'Output file {out_file} is missing while bulk re-building cache for {pretty_dir}.')
+ else:
+ shutil.copy(out_file, cachefile(fn))
+
@contextmanager
def svg_soup(filename):
with open(filename, 'r') as f: