Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Userspace: add support for adding environment variables during build #22887

Open
wants to merge 41 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
d6bfe13
Attempt to get env parsing sorted.
tzarc Jan 8, 2024
dec71d7
Add support for extra args to BuildTarget.
tzarc Jan 8, 2024
dc1e8ca
Initial propagation of env vars.
tzarc Jan 11, 2024
85d0be4
Allow for same board, different env vars.
tzarc Jan 11, 2024
9dcf283
Fixup userspace-list expansion.
tzarc Jan 11, 2024
cb9bec3
Consistency.
tzarc Jan 11, 2024
1cf0d08
Move versioned build target schema definitions to userspace repo sche…
tzarc Jan 11, 2024
3585543
Merge remote-tracking branch 'upstream/master' into userspace-env
tzarc Jan 11, 2024
3a79b4c
Save/load env vars as object instead of array.
tzarc Jan 12, 2024
7038050
`qmk userspace-list` updates.
tzarc Jan 12, 2024
2de1056
More dict conversions.
tzarc Jan 12, 2024
1bef97e
`qmk pytest`
tzarc Jan 12, 2024
3192640
Schema cleanup.
tzarc Jan 12, 2024
ccbc183
Target filename manipulation... next step INTERMEDIATE_OUTPUT.
tzarc Jan 12, 2024
de3af67
First stab at building intermediate directory based on variables that…
tzarc Jan 12, 2024
07ec161
Merge remote-tracking branch 'upstream/master' into userspace-env
tzarc Jan 19, 2024
e5cedf3
Rely on makefiles to generate INTERMEDIATE_OUTPUT for the build.
tzarc Jan 19, 2024
bfa1d80
Let's have a stab at forcing community layouts.
tzarc Jan 19, 2024
cad628f
Fix.
tzarc Jan 19, 2024
4828ee4
Fix.
tzarc Jan 19, 2024
616f4c9
Fix.
tzarc Jan 19, 2024
5ce1f4e
More force_layout hot potato.
tzarc Jan 19, 2024
b4d7b79
Naming is hard.
tzarc Jan 19, 2024
6f9babf
Merge remote-tracking branch 'upstream/master' into userspace-env
tzarc Mar 9, 2024
c995d35
License years.
tzarc Mar 9, 2024
8d1f9cf
Merge remote-tracking branch 'upstream/master' into userspace-env
tzarc Apr 21, 2024
0bbd470
More conversion stuff.
tzarc Apr 22, 2024
9cb0800
Merge remote-tracking branch 'upstream/master' into userspace-env
tzarc Apr 23, 2024
a695c73
KeyboardKeymapDesc
tzarc Apr 23, 2024
91466e8
Apply original keyboard/keymap
tzarc Apr 23, 2024
bb60311
Apply original keyboard/keymap
tzarc Apr 23, 2024
05b1852
Apply original keyboard/keymap
tzarc Apr 23, 2024
d51f58b
Simplification pass.
tzarc Apr 23, 2024
0abbbd1
Fixup
tzarc Apr 23, 2024
3c9ddd3
Simplification pass.
tzarc Apr 23, 2024
c650857
Simplification pass.
tzarc Apr 23, 2024
112b4a2
Fixup
tzarc Apr 23, 2024
c4b70df
simplification
tzarc Apr 26, 2024
dc8b4b1
simplification
tzarc Apr 26, 2024
09d1ce7
`qmk pytest`, dry-run
tzarc Apr 26, 2024
53bc15e
shell quotes
tzarc Apr 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
simplification
  • Loading branch information
tzarc committed Apr 26, 2024
commit dc8b4b1006cedaab591355e8cf1e66710b0f35a6
3 changes: 3 additions & 0 deletions lib/python/qmk/build_targets.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ def __repr__(self):
return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={self._extra_args})'
return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'

def __lt__(self, __value: object) -> bool:
return self.__repr__() < __value.__repr__()

def __eq__(self, __value: object) -> bool:
if not isinstance(__value, BuildTarget):
return False
Expand Down
8 changes: 5 additions & 3 deletions lib/python/qmk/cli/mass_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,17 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool,
extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()])
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
target_suffix = ''
if len(extra_args) > 0:
build_log += f".{extra_args}"
failed_log += f".{extra_args}"
target_suffix = f"_{extra_args}"
# yapf: disable
f.write(
f"""\
.PHONY: {target_filename}_binary
all: {target_filename}_binary
{target_filename}_binary:
.PHONY: {target_filename}{target_suffix}_binary
all: {target_filename}{target_suffix}_binary
{target_filename}{target_suffix}_binary:
@rm -f "{build_log}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
{' '.join(command)} \\
Expand Down
2 changes: 1 addition & 1 deletion lib/python/qmk/cli/userspace/compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def userspace_compile(cli):
if isinstance(e, Path):
build_targets.append(JsonKeymapBuildTarget(e))
elif isinstance(e, dict):
f = None if 'env' not in e else partial(_extra_arg_setter, extra_args=e['env'])
f = e['env'] if 'env' in e else None
keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f))
if len(keyboard_keymap_targets) > 0:
build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
Expand Down
2 changes: 1 addition & 1 deletion lib/python/qmk/cli/userspace/list.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def userspace_list(cli):
if isinstance(e, Path):
build_targets.append(e)
elif isinstance(e, dict) or isinstance(e, Dotty):
f = None if 'env' not in e else partial(_extra_arg_setter, extra_args=e['env'])
f = e['env'] if 'env' in e else None
keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f))
if len(keyboard_keymap_targets) > 0:
build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
Expand Down
67 changes: 27 additions & 40 deletions lib/python/qmk/search.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
"""Functions for searching through QMK keyboards and keymaps.
"""
from dataclasses import dataclass
import contextlib
import functools
import fnmatch
import json
import logging
import re
from typing import Callable, List, Optional, Tuple, Union
from typing import Callable, Dict, List, Optional, Tuple, Union
from dotty_dict import dotty, Dotty
from milc import cli

Expand All @@ -16,28 +18,18 @@
from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget


@functools.total_ordering
@dataclass
class KeyboardKeymapDesc:
def __init__(self, keyboard: str, keymap: str, data: dict = None, construction_callback: Callable[[BuildTarget], None] = None):
self.keyboard = keyboard
self.keymap = keymap
self.data = data.to_dict() if isinstance(data, Dotty) else data
self.construction_callback = construction_callback

def __str__(self):
return f'{self.keyboard}:{self.keymap}'

def __repr__(self):
return f'KeyboardKeymapDesc(keyboard={self.keyboard}, keymap={self.keymap})'

def __eq__(self, other: object):
return self.__repr__() == other.__repr__()
keyboard: str
keymap: str
data: dict = None
extra_args: dict = None

def __lt__(self, other):
return self.__repr__() < other.__repr__()
def __hash__(self) -> int:
return self.keyboard.__hash__() ^ self.keymap.__hash__() ^ json.dumps(self.extra_args, sort_keys=True).__hash__()

def __hash__(self):
return self.__repr__().__hash__()
def __lt__(self, other) -> bool:
return (self.keyboard, self.keymap, json.dumps(self.extra_args, sort_keys=True)) < (other.keyboard, other.keymap, json.dumps(other.extra_args, sort_keys=True))

def load_data(self):
data = keymap_json(self.keyboard, self.keymap)
Expand All @@ -47,14 +39,9 @@ def load_data(self):
def dotty(self) -> Dotty:
return dotty(self.data) if self.data is not None else None

@property
def json(self) -> dict:
return self.data

def to_build_target(self) -> KeyboardKeymapBuildTarget:
target = KeyboardKeymapBuildTarget(keyboard=self.keyboard, keymap=self.keymap, json=self.json)
if self.construction_callback is not None:
self.construction_callback(target)
target = KeyboardKeymapBuildTarget(keyboard=self.keyboard, keymap=self.keymap, json=self.data)
target.extra_args = self.extra_args
return target


Expand Down Expand Up @@ -168,23 +155,23 @@ def _load_keymap_info(target: KeyboardKeymapDesc) -> KeyboardKeymapDesc:
return target


def expand_make_targets(targets: List[Union[str, Tuple[str, Callable[[BuildTarget], None]]]]) -> List[KeyboardKeymapDesc]:
def expand_make_targets(targets: List[Union[str, Tuple[str, Dict[str, str]]]]) -> List[KeyboardKeymapDesc]:
"""Expand a list of make targets into a list of KeyboardKeymapDesc.

Caters for 'all' in either keyboard or keymap, or both.
"""
split_targets = []
for target in targets:
construction_callback = None
extra_args = None
if isinstance(target, tuple):
split_target = target[0].split(':')
construction_callback = target[1]
extra_args = target[1]
else:
split_target = target.split(':')
if len(split_target) != 2:
cli.log.error(f"Invalid build target: {target}")
return []
split_targets.append(KeyboardKeymapDesc(split_target[0], split_target[1], construction_callback=construction_callback))
split_targets.append(KeyboardKeymapDesc(split_target[0], split_target[1], extra_args=extra_args))
return expand_keymap_targets(split_targets)


Expand All @@ -203,18 +190,18 @@ def _expand_keymap_target(target: KeyboardKeymapDesc, all_keyboards: List[str] =
for kb in parallel_map(_all_keymaps, all_keyboards):
targets.extend(kb)
for t in targets:
t.construction_callback = target.construction_callback
t.extra_args = target.extra_args
return targets
else:
cli.log.info(f'Retrieving list of keyboards with keymap "{target.keymap}"...')
keyboard_filter = functools.partial(_keymap_exists, keymap=target.keymap)
return [KeyboardKeymapDesc(kb, target.keymap, construction_callback=target.construction_callback) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
return [KeyboardKeymapDesc(kb, target.keymap, extra_args=target.extra_args) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
else:
if target.keymap == 'all':
cli.log.info(f'Retrieving list of keymaps for keyboard "{target.keyboard}"...')
targets = _all_keymaps(target.keyboard)
for t in targets:
t.construction_callback = target.construction_callback
t.extra_args = target.extra_args
return targets
else:
return [target]
Expand Down Expand Up @@ -276,7 +263,7 @@ def _make_filter(k, v):
expr = fnmatch.translate(v)
rule = re.compile(f'^{expr}$', re.IGNORECASE)

def f(e):
def f(e: KeyboardKeymapDesc):
lhs = e.dotty.get(k)
lhs = str(False if lhs is None else lhs)
return rule.search(lhs) is not None
Expand All @@ -294,24 +281,24 @@ def f(e):
return targets


def search_keymap_targets(targets: List[Union[Tuple[str, str], Tuple[str, str, Callable[[BuildTarget], None]]]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
def search_keymap_targets(targets: List[Union[Tuple[str, str], Tuple[str, str, Dict[str, str]]]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
def _make_desc(e):
if len(e) == 3:
return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1], construction_callback=e[2])
return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1], extra_args=e[2])
else:
return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1])

targets = map(_make_desc, targets)
targets = _filter_keymap_targets(expand_keymap_targets(targets), filters)
targets = list(set(parallel_map(_construct_build_target, list(targets))))
return targets
return sorted(targets)


def search_make_targets(targets: List[Union[str, Tuple[str, Callable[[BuildTarget], None]]]], filters: List[str] = []) -> List[BuildTarget]:
def search_make_targets(targets: List[Union[str, Tuple[str, Dict[str, str]]]], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
targets = _filter_keymap_targets(expand_make_targets(targets), filters)
targets = list(set(parallel_map(_construct_build_target, list(targets))))
return targets
return sorted(targets)
Loading