logo

qmk_firmware

custom branch of QMK firmware git clone https://anongit.hacktivis.me/git/qmk_firmware.git

mass_compile.py (6709B)

    

  1. """Compile all keyboards.
  3. This will compile everything in parallel, for testing purposes.
  4. """
  5. import os
  6. from typing import List
  7. from pathlib import Path
  8. from subprocess import DEVNULL
  9. from milc import cli
  10. import shlex
  12. from qmk.constants import QMK_FIRMWARE
  13. from qmk.commands import find_make, get_make_parallel_args, build_environment
  14. from qmk.search import search_keymap_targets, search_make_targets
  15. from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
  16. from qmk.util import maybe_exit_config
  19. def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
  20.     if len(targets) == 0:
  21.         return
  23.     os.environ.setdefault('SKIP_SCHEMA_VALIDATION', '1')
  25.     make_cmd = find_make()
  26.     builddir = Path(QMK_FIRMWARE) / '.build'
  27.     makefile = builddir / 'parallel_kb_builds.mk'
  29.     if dry_run:
  30.         cli.log.info('Compilation targets:')
  31.         for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
  32.             extra_args = ' '.join([f"-e {shlex.quote(f'{k}={v}')}" for k, v in target.extra_args.items()])
  33.             cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap} {extra_args}{{fg_reset}}")
  34.     else:
  35.         if clean:
  36.             cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
  38.         builddir.mkdir(parents=True, exist_ok=True)
  39.         with open(makefile, "w") as f:
  40.             for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
  41.                 keyboard_name = target.keyboard
  42.                 keymap_name = target.keymap
  43.                 keyboard_safe = keyboard_name.replace('/', '_')
  44.                 target_filename = target.target_name(**env)
  45.                 target.configure(parallel=1)  # We ignore parallelism on a per-build basis as we defer to the parent make invocation
  46.                 target.prepare_build(**env)  # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
  47.                 command = target.compile_command(**env)
  48.                 command[0] = '+@$(MAKE)'  # Override the make so that we can use jobserver to handle parallelism
  49.                 extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()])
  50.                 build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
  51.                 failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
  52.                 target_suffix = ''
  53.                 if len(extra_args) > 0:
  54.                     build_log += f".{extra_args}"
  55.                     failed_log += f".{extra_args}"
  56.                     target_suffix = f"_{extra_args}"
  57.                 # yapf: disable
  58.                 f.write(
  59.                     f"""\
  60. .PHONY: {target_filename}{target_suffix}_binary
  61. all: {target_filename}{target_suffix}_binary
  62. {target_filename}{target_suffix}_binary:
  63. 	@rm -f "{build_log}" || true
  64. 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
  65. 	{' '.join(command)} \\
  66. 		>>"{build_log}" 2>&1 \\
  67. 		|| cp "{build_log}" "{failed_log}"
  68. 	@{{ grep '\\[ERRORS\\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \\e[1;31m[ERRORS]\\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
  69. 		|| {{ grep '\\[WARNINGS\\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \\e[1;33m[WARNINGS]\\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
  70. 		|| printf "Build %-64s \\e[1;32m[OK]\\e[0m\\n" "{keyboard_name}:{keymap_name}"
  71. 	@rm -f "{build_log}" || true
  72. """# noqa
  73.                 )
  74.                 # yapf: enable
  76.                 if no_temp:
  77.                     # yapf: disable
  78.                     f.write(
  79.                         f"""\
  80. 	@rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.elf" 2>/dev/null || true
  81. 	@rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.map" 2>/dev/null || true
  82. 	@rm -rf "{QMK_FIRMWARE}/.build/obj_{target_filename}" || true
  83. """# noqa
  84.                     )
  85.                     # yapf: enable
  86.                 f.write('\n')
  88.         cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
  90.         # Check for failures
  91.         failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
  92.         if len(failures) > 0:
  93.             return False
  96. @cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
  97. @cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
  98. @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
  99. @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
  100. @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
  101. @cli.argument(
  102.     '-f',
  103.     '--filter',
  104.     arg_only=True,
  105.     action='append',
  106.     default=[],
  107.     help=  # noqa: `format-python` and `pytest` don't agree here.
  108.     "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'."  # noqa: `format-python` and `pytest` don't agree here.
  109. )
  110. @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
  111. @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
  112. @cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True)
  113. def mass_compile(cli):
  114.     """Compile QMK Firmware against all keyboards.
  115.     """
  116.     maybe_exit_config(should_exit=False, should_reraise=True)
  118.     if len(cli.args.builds) > 0:
  119.         json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
  120.         make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
  121.         targets = search_make_targets(make_like_targets)
  122.         targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
  123.     else:
  124.         targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
  126.     return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env))