Create a system to map between info.json and config.h/rules.mk (#11548)

* generate rules.mk from a json mapping

* generate rules.mk from a json mapping

* support for config.h from json maps

* improve the mapping system

* document the mapping system

* move data/maps to data/mappings

* fix flake8 errors

* fixup LED_MATRIX_DRIVER

* remove product and description from the vision_division keymap level

* reduce the complexity of generate-rules-mk

* add tests for the generate commands

* fix qmk doctor when submodules are not clean

bin/qmk

@@ -27,7 +27,8 @@ def _check_modules(requirements):
                 line = line.split('#')[0]
 
             module = dict()
-            module['name'] = module['import'] = line.split('=')[0] if '=' in line else line
+            module['name'] = line.split('=')[0] if '=' in line else line
+            module['import'] = module['name'].replace('-', '_')
 
             # Not every module is importable by its own name.
             if module['name'] == "pep8-naming":

data/mappings/info_config.json

@@ -0,0 +1,42 @@
+# This file maps keys between `config.h` and `info.json`. It is used by QMK 
+# to correctly and consistently map back and forth between the two systems.
+{
+    # Format:
+    # <config.h key>: {"info_key": <info.json key>, ["value_type": <value_type>], ["to_json": <true/false>], ["to_c": <true/false>]}
+    # value_type: one of "array", "array.int", "int", "hex", "list", "mapping"
+    # to_json: Default `true`. Set to `false` to exclude this mapping from info.json
+    # to_c: Default `true`. Set to `false` to exclude this mapping from config.h
+    # warn_duplicate: Default `true`. Set to `false` to turn off warning when a value exists in both places
+    "DEBOUNCE": {"info_key": "debounce", "value_type": "int"}
+    "DEVICE_VER": {"info_key": "usb.device_ver", "value_type": "hex"},
+    "DESCRIPTION": {"info_key": "keyboard_folder", "to_json": false},
+    "DIODE_DIRECTION": {"info_key": "diode_direction"},
+    "LAYOUTS": {"info_key": "layout_aliases", "value_type": "mapping"},
+    "LED_CAPS_LOCK_PIN": {"info_key": "indicators.caps_lock"},
+    "LED_NUM_LOCK_PIN": {"info_key": "indicators.num_lock"},
+    "LED_SCROLL_LOCK_PIN": {"info_key": "indicators.scroll_lock"},
+    "MANUFACTURER": {"info_key": "manufacturer"},
+    "RGB_DI_PIN": {"info_key": "rgblight.pin"},
+    "RGBLED_NUM": {"info_key": "rgblight.led_count", "value_type": "int"},
+    "RGBLED_SPLIT": {"info_key": "rgblight.split_count", "value_type": "array.int"},
+    "RGBLIGHT_ANIMATIONS": {"info_key": "rgblight.animations.all", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_ALTERNATING": {"info_key": "rgblight.animations.alternating", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_BREATHING": {"info_key": "rgblight.animations.breathing", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_CHRISTMAS": {"info_key": "rgblight.animations.christmas", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_KNIGHT": {"info_key": "rgblight.animations.knight", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_RAINBOW_MOOD": {"info_key": "rgblight.animations.rainbow_mood", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_RAINBOW_SWIRL": {"info_key": "rgblight.animations.rainbow_swirl", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_RGB_TEST": {"info_key": "rgblight.animations.rgb_test", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_SNAKE": {"info_key": "rgblight.animations.snake", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_STATIC_GRADIENT": {"info_key": "rgblight.animations.static_gradient", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_TWINKLE": {"info_key": "rgblight.animations.twinkle"},
+    "RGBLIGHT_LIMIT_VAL": {"info_key": "rgblight.max_brightness", "value_type": "int"},
+    "RGBLIGHT_HUE_STEP": {"info_key": "rgblight.hue_steps", "value_type": "int"},
+    "RGBLIGHT_SAT_STEP": {"info_key": "rgblight.saturation_steps", "value_type": "int"},
+    "RGBLIGHT_VAL_STEP": {"info_key": "rgblight.brightness_steps", "value_type": "int"},
+    "RGBLIGHT_SLEEP": {"info_key": "rgblight.sleep", "value_type": "bool"},
+    "RGBLIGHT_SPLIT": {"info_key": "rgblight.split", "value_type": "bool"},
+    "PRODUCT": {"info_key": "keyboard_folder", "to_json": false},
+    "PRODUCT_ID": {"info_key": "usb.pid", "value_type": "hex"},
+    "VENDOR_ID": {"info_key": "usb.vid", "value_type": "hex"}
+}

data/mappings/info_rules.json

@@ -0,0 +1,15 @@
+# This file maps keys between `rules.mk` and `info.json`. It is used by QMK 
+# to correctly and consistently map back and forth between the two systems.
+{
+    # Format:
+    # <rules.mk key>: {"info_key": <info.json key>, ["value_type": <value_type>], ["to_json": <true/false>], ["to_c": <true/false>]}
+    # value_type: one of "array", "array.int", "int", "list", "hex", "mapping"
+    # to_json: Default `true`. Set to `false` to exclude this mapping from info.json
+    # to_c: Default `true`. Set to `false` to exclude this mapping from rules.mk
+    # warn_duplicate: Default `true`. Set to `false` to turn off warning when a value exists in both places
+    "BOARD": {"info_key": "board"},
+    "BOOTLOADER": {"info_key": "bootloader", "warn_duplicate": false},
+    "LAYOUTS": {"info_key": "community_layouts", "value_type": "list"},
+    "LED_MATRIX_DRIVER": {"info_key": "led_matrix.driver"},
+    "MCU": {"info_key": "processor", "warn_duplicate": false},
+}

docs/data_driven_config.md

@@ -12,17 +12,18 @@ Now we have support for generating `rules.mk` and `config.h` values from `info.j
 
 ## Overview
 
-On the C side of things nothing really changes. When you need to create a new rule or define you follow the same process:
+On the C side of things nothing changes. When you need to create a new rule or define you follow the same process:
 
 1. Add it to `docs/config_options.md`
 1. Set a default in the appropriate core file
-1. Add your `ifdef` and/or `#ifdef` statements as needed
+1. Add your ifdef statements as needed
 
 You will then need to add support for your new configuration to `info.json`. The basic process is:
 
 1. Add it to the schema in `data/schemas/keyboards.jsonschema`
-1. Add code to extract it from `config.h`/`rules.mk` to `lib/python/qmk/info.py`
-1. Add code to generate it to one of:
+1. Add a mapping in `data/maps`
+1. (optional and discoraged) Add code to extract/generate it to:
+  * `lib/python/qmk/info.py`
   * `lib/python/qmk/cli/generate/config_h.py`
   * `lib/python/qmk/cli/generate/rules_mk.py`
 
@@ -32,12 +33,43 @@ This section describes adding support for a `config.h`/`rules.mk` value to info.
 
 ### Add it to the schema
 
-QMK maintains schema files in `data/schemas`. The values that go into keyboard-specific `info.json` files are kept in `keyboard.jsonschema`. Any value you want to make available to end users to edit must go in here.
+QMK maintains [jsonschema](https://json-schema.org/) files in `data/schemas`. The values that go into keyboard-specific `info.json` files are kept in `keyboard.jsonschema`. Any value you want to make available to end users to edit must go in here.
 
-In some cases you can simply add a new top-level key. Some examples to follow are `keyboard_name`, `maintainer`, `processor`, and `url`. This is appropriate when your option is self-contained and not directly related to other options. In other cases you should group like options together in an `object`. This is particularly true when adding support for a feature. Some examples to follow for this are `indicators`, `matrix_pins`, and `rgblight`. If you are not sure how to integrate your new option(s) [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and start a conversation there.
+In some cases you can simply add a new top-level key. Some examples to follow are `keyboard_name`, `maintainer`, `processor`, and `url`. This is appropriate when your option is self-contained and not directly related to other options.
+
+In other cases you should group like options together in an `object`. This is particularly true when adding support for a feature. Some examples to follow for this are `indicators`, `matrix_pins`, and `rgblight`. If you are not sure how to integrate your new option(s) [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and start a conversation there.
+
+### Add a mapping
+
+In most cases you can add a simple mapping. These are maintained as JSON files in `data/mappings/info_config.json` and `data/mappings/info_rules.json`, and control mapping for `config.h` and `rules.mk`, respectively. Each mapping is keyed by the `config.h` or `rules.mk` variable, and the value is a hash with the following keys:
+
+* `info_key`: (required) The location within `info.json` for this value. See below.
+* `value_type`: (optional) Default `str`. The format for this variable's value. See below.
+* `to_json`: (optional) Default `true`. Set to `false` to exclude this mapping from info.json
+* `to_c`: (optional) Default `true`. Set to `false` to exclude this mapping from config.h
+* `warn_duplicate`: (optional) Default `true`. Set to `false` to turn off warning when a value exists in both places
+
+#### Info Key
+
+We use JSON dot notation to address variables within info.json. For example, to access `info_json["rgblight"]["split_count"]` I would specify `rgblight.split_count`. This allows you to address deeply nested keys with a simple string.
+
+Under the hood we use [Dotty Dict](https://dotty-dict.readthedocs.io/en/latest/), you can refer to that documentation for how these strings are converted to object access.
+
+#### Value Types
+
+By default we treat all values as simple strings. If your value is more complex you can use one of these types to intelligently parse the data:
+
+* `array`: A comma separated array of strings
+* `array.int`: A comma separated array of integers
+* `int`: An integer
+* `hex`: A number formatted as hex
+* `list`: A space separate array of strings
+* `mapping`: A hash of key/value pairs
 
 ### Add code to extract it
 
+Most use cases can be solved by the mapping files described above. If yours can't you can instead write code to extract your config values.
+
 Whenever QMK generates a complete `info.json` it extracts information from `config.h` and `rules.mk`. You will need to add code for your new config value to `lib/python/qmk/info.py`. Typically this means adding a new `_extract_<feature>()` function and then calling your function in either `_extract_config_h()` or `_extract_rules_mk()`.
 
 If you are not sure how to edit this file or are not comfortable with Python [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and someone can help you with this part.

keyboards/handwired/pytest/basic/info.json

@@ -0,0 +1,10 @@
+{
+  "maintainer": "qmk",
+  "layouts": {
+    "LAYOUT_custom": {
+      "layout": [
+        { "label": "KC_Q", "matrix": [0, 0], "w": 1, "x": 0, "y": 0 }
+      ]
+    }
+  }
+}

keyboards/vision_division/keymaps/default/config.h

@@ -8,7 +8,6 @@
 #define VENDOR_ID       0xFEED
 #define DEVICE_VER      0x0001
 #define MANUFACTURER    IBNobody
-#define PRODUCT         Vision Division
 
 #define MATRIX_ROWS     6
 #define MATRIX_ROW_PINS { C2, C3, F4, F5, F6, F7 }

lib/python/qmk/cli/doctor.py

@@ -107,9 +107,9 @@ def doctor(cli):
             submodules.update()
             sub_ok = check_submodules()
 
-        if CheckStatus.ERROR in sub_ok:
+        if sub_ok == CheckStatus.ERROR:
             status = CheckStatus.ERROR
-        elif CheckStatus.WARNING in sub_ok and status == CheckStatus.OK:
+        elif sub_ok == CheckStatus.WARNING and status == CheckStatus.OK:
             status = CheckStatus.WARNING
 
     # Report a summary of our findings to the user