From d9785ec31339d7f80279fd3d1005f76689ed2f6a Mon Sep 17 00:00:00 2001
From: Zach White <skullydazed@gmail.com>
Date: Sat, 16 Jan 2021 15:13:04 -0800
Subject: [PATCH] Improve the compile and flash subcommands (#11334)
* add support for --clean to compile and flash
* compile standalone JSON keymaps without polluting the tree
* Add support for passing environment vars to make
* make flake8 happy
* document changes to qmk compile and flash
* add -e support to json export compiling
* Fix python 3.6
* honor $MAKE
* add support for parallel builds
---
docs/cli_commands.md | 8 +-
lib/python/qmk/cli/chibios/confmigrate.py | 8 +-
lib/python/qmk/cli/compile.py | 27 +++++--
lib/python/qmk/cli/flash.py | 26 +++++--
lib/python/qmk/commands.py | 93 ++++++++++++++++++++---
lib/python/qmk/constants.py | 5 ++
6 files changed, 137 insertions(+), 30 deletions(-)
diff --git a/docs/cli_commands.md b/docs/cli_commands.md
index 61ce1aa2a4..5ab49abd27 100644
--- a/docs/cli_commands.md
+++ b/docs/cli_commands.md
@@ -11,13 +11,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K
**Usage for Configurator Exports**:
```
-qmk compile <configuratorExport.json>
+qmk compile [-c] <configuratorExport.json>
```
**Usage for Keymaps**:
```
-qmk compile -kb <keyboard_name> -km <keymap_name>
+qmk compile [-c] [-e <var>=<value>] -kb <keyboard_name> -km <keymap_name>
```
**Usage in Keyboard Directory**:
@@ -82,13 +82,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K
**Usage for Configurator Exports**:
```
-qmk flash <configuratorExport.json> -bl <bootloader>
+qmk flash [-bl <bootloader>] [-c] [-e <var>=<value>] <configuratorExport.json>
```
**Usage for Keymaps**:
```
-qmk flash -kb <keyboard_name> -km <keymap_name> -bl <bootloader>
+qmk flash -kb <keyboard_name> -km <keymap_name> [-bl <bootloader>] [-c] [-e <var>=<value>]
```
**Listing the Bootloaders**
diff --git a/lib/python/qmk/cli/chibios/confmigrate.py b/lib/python/qmk/cli/chibios/confmigrate.py
index eae294a0c6..b9cfda9614 100644
--- a/lib/python/qmk/cli/chibios/confmigrate.py
+++ b/lib/python/qmk/cli/chibios/confmigrate.py
@@ -13,7 +13,7 @@ def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
-fileHeader = """\
+file_header = """\
/* Copyright 2020 QMK
*
* This program is free software: you can redistribute it and/or modify
@@ -77,7 +77,7 @@ def check_diffs(input_defs, reference_defs):
def migrate_chconf_h(to_override, outfile):
- print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
+ print(file_header.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
for override in to_override:
print("#define %s %s" % (override[0], override[1]), file=outfile)
@@ -87,7 +87,7 @@ def migrate_chconf_h(to_override, outfile):
def migrate_halconf_h(to_override, outfile):
- print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
+ print(file_header.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
for override in to_override:
print("#define %s %s" % (override[0], override[1]), file=outfile)
@@ -97,7 +97,7 @@ def migrate_halconf_h(to_override, outfile):
def migrate_mcuconf_h(to_override, outfile):
- print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
+ print(file_header.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
print("#include_next <mcuconf.h>\n", file=outfile)
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index daee597d8e..322ce6a257 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -2,7 +2,6 @@
You can compile a keymap already in the repo or using a QMK Configurator export.
"""
-import subprocess
from argparse import FileType
from milc import cli
@@ -15,6 +14,9 @@ from qmk.commands import compile_configurator_json, create_make_command, parse_c
@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
+@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
+@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.subcommand('Compile a QMK Firmware.')
@automagic_keyboard
@automagic_keymap
@@ -25,18 +27,32 @@ def compile(cli):
If a keyboard and keymap are provided this command will build a firmware based on that.
"""
+ if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+ command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
+ # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
+ cli.run(command, capture_output=False, text=False)
+
+ # Build the environment vars
+ envs = {}
+ for env in cli.args.env:
+ if '=' in env:
+ key, value = env.split('=', 1)
+ envs[key] = value
+ else:
+ cli.log.warning('Invalid environment variable: %s', env)
+
+ # Determine the compile command
command = None
if cli.args.filename:
# If a configurator JSON was provided generate a keymap and compile it
- # FIXME(skullydazed): add code to check and warn if the keymap already exists when compiling a json keymap.
user_keymap = parse_configurator_json(cli.args.filename)
- command = compile_configurator_json(user_keymap)
+ command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
else:
if cli.config.compile.keyboard and cli.config.compile.keymap:
# Generate the make command for a specific keyboard/keymap.
- command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap)
+ command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
elif not cli.config.compile.keyboard:
cli.log.error('Could not determine keyboard!')
@@ -48,7 +64,8 @@ def compile(cli):
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not cli.args.dry_run:
cli.echo('\n')
- compile = subprocess.run(command)
+ # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
+ compile = cli.run(command, capture_output=False, text=False)
return compile.returncode
else:
diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index d720d42e71..b3827e8003 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -3,7 +3,6 @@
You can compile a keymap already in the repo or using a QMK Configurator export.
A bootloader must be specified.
"""
-import subprocess
from argparse import FileType
from milc import cli
@@ -37,6 +36,9 @@ def print_bootloader_help():
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
+@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
+@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.subcommand('QMK Flash.')
@automagic_keyboard
@automagic_keymap
@@ -50,6 +52,20 @@ def flash(cli):
If bootloader is omitted the make system will use the configured bootloader for that keyboard.
"""
+ if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+ command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
+ cli.run(command, capture_output=False)
+
+ # Build the environment vars
+ envs = {}
+ for env in cli.args.env:
+ if '=' in env:
+ key, value = env.split('=', 1)
+ envs[key] = value
+ else:
+ cli.log.warning('Invalid environment variable: %s', env)
+
+ # Determine the compile command
command = ''
if cli.args.bootloaders:
@@ -60,16 +76,16 @@ def flash(cli):
if cli.args.filename:
# Handle compiling a configurator JSON
- user_keymap = parse_configurator_json(cli.args.filename)
+ user_keymap = parse_configurator_json(cli.args.filename, parallel=cli.config.flash.parallel)
keymap_path = qmk.path.keymap(user_keymap['keyboard'])
- command = compile_configurator_json(user_keymap, cli.args.bootloader)
+ command = compile_configurator_json(user_keymap, cli.args.bootloader, **envs)
cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
else:
if cli.config.flash.keyboard and cli.config.flash.keymap:
# Generate the make command for a specific keyboard/keymap.
- command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader)
+ command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
elif not cli.config.flash.keyboard:
cli.log.error('Could not determine keyboard!')
@@ -81,7 +97,7 @@ def flash(cli):
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not cli.args.dry_run:
cli.echo('\n')
- compile = subprocess.run(command)
+ compile = cli.run(command, capture_output=False, text=True)
return compile.returncode
else:
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 4ec7d2ea53..ba225bf558 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -6,11 +6,26 @@ import platform
import subprocess
import shlex
import shutil
+from pathlib import Path
+
+from milc import cli
import qmk.keymap
+from qmk.constants import KEYBOARD_OUTPUT_PREFIX
-def create_make_command(keyboard, keymap, target=None):
+def _find_make():
+ """Returns the correct make command for this environment.
+ """
+ make_cmd = os.environ.get('MAKE')
+
+ if not make_cmd:
+ make_cmd = 'gmake' if shutil.which('gmake') else 'make'
+
+ return make_cmd
+
+
+def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
"""Create a make compile command
Args:
@@ -24,41 +39,95 @@ def create_make_command(keyboard, keymap, target=None):
target
Usually a bootloader.
+ parallel
+ The number of make jobs to run in parallel
+
+ **env_vars
+ Environment variables to be passed to make.
+
Returns:
A command that can be run to make the specified keyboard and keymap
"""
+ env = []
make_args = [keyboard, keymap]
- make_cmd = 'gmake' if shutil.which('gmake') else 'make'
+ make_cmd = _find_make()
if target:
make_args.append(target)
- return [make_cmd, ':'.join(make_args)]
+ for key, value in env_vars.items():
+ env.append(f'{key}={value}')
+
+ return [make_cmd, '-j', str(parallel), *env, ':'.join(make_args)]
-def compile_configurator_json(user_keymap, bootloader=None):
- """Convert a configurator export JSON file into a C file
+def compile_configurator_json(user_keymap, parallel=1, **env_vars):
+ """Convert a configurator export JSON file into a C file and then compile it.
Args:
- configurator_filename
- The configurator JSON export file
+ user_keymap
+ A deserialized keymap export
bootloader
A bootloader to flash
+ parallel
+ The number of make jobs to run in parallel
+
Returns:
A command to run to compile and flash the C file.
"""
- # Write the keymap C file
- qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+ # Write the keymap.c file
+ keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
+ target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
+ keyboard_output = Path(f'{KEYBOARD_OUTPUT_PREFIX}{keyboard_filesafe}')
+ keymap_output = Path(f'{keyboard_output}_{user_keymap["keymap"]}')
+ c_text = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+ keymap_dir = keymap_output / 'src'
+ keymap_c = keymap_dir / 'keymap.c'
+
+ keymap_dir.mkdir(exist_ok=True, parents=True)
+ keymap_c.write_text(c_text)
# Return a command that can be run to make the keymap and flash if given
- if bootloader is None:
- return create_make_command(user_keymap['keyboard'], user_keymap['keymap'])
- return create_make_command(user_keymap['keyboard'], user_keymap['keymap'], bootloader)
+ verbose = 'true' if cli.config.general.verbose else 'false'
+ color = 'true' if cli.config.general.color else 'false'
+ make_command = [
+ _find_make(),
+ '-j',
+ str(parallel),
+ '-r',
+ '-R',
+ '-f',
+ 'build_keyboard.mk',
+ ]
+
+ for key, value in env_vars.items():
+ make_command.append(f'{key}={value}')
+
+ make_command.extend([
+ f'KEYBOARD={user_keymap["keyboard"]}',
+ f'KEYMAP={user_keymap["keymap"]}',
+ f'KEYBOARD_FILESAFE={keyboard_filesafe}',
+ f'TARGET={target}',
+ f'KEYBOARD_OUTPUT={keyboard_output}',
+ f'KEYMAP_OUTPUT={keymap_output}',
+ f'MAIN_KEYMAP_PATH_1={keymap_output}',
+ f'MAIN_KEYMAP_PATH_2={keymap_output}',
+ f'MAIN_KEYMAP_PATH_3={keymap_output}',
+ f'MAIN_KEYMAP_PATH_4={keymap_output}',
+ f'MAIN_KEYMAP_PATH_5={keymap_output}',
+ f'KEYMAP_C={keymap_c}',
+ f'KEYMAP_PATH={keymap_dir}',
+ f'VERBOSE={verbose}',
+ f'COLOR={color}',
+ 'SILENT=false',
+ ])
+
+ return make_command
def parse_configurator_json(configurator_file):
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 94ab68e5e4..2ddaa568a2 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -1,5 +1,6 @@
"""Information that should be available to the python library.
"""
+from os import environ
from pathlib import Path
# The root of the qmk_firmware tree.
@@ -17,3 +18,7 @@ VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'
DATE_FORMAT = '%Y-%m-%d'
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
TIME_FORMAT = '%H:%M:%S'
+
+# Constants that should match their counterparts in make
+BUILD_DIR = environ.get('BUILD_DIR', '.build')
+KEYBOARD_OUTPUT_PREFIX = f'{BUILD_DIR}/obj_'