Merge remote-tracking branch 'qmk 0.17.0' into firmware21

lib/python/qmk/c_parse.py

@@ -1,5 +1,9 @@
 """Functions for working with config.h files.
 """
+from pygments.lexers.c_cpp import CLexer
+from pygments.token import Token
+from pygments import lex
+from itertools import islice
 from pathlib import Path
 import re
 
@@ -10,6 +14,14 @@ from qmk.comment_remover import comment_remover
 default_key_entry = {'x': -1, 'y': 0, 'w': 1}
 single_comment_regex = re.compile(r'\s+/[/*].*$')
 multi_comment_regex = re.compile(r'/\*(.|\n)*?\*/', re.MULTILINE)
+layout_macro_define_regex = re.compile(r'^#\s*define')
+
+
+def _get_chunks(it, size):
+    """Break down a collection into smaller parts
+    """
+    it = iter(it)
+    return iter(lambda: tuple(islice(it, size)), ())
 
 
 def strip_line_comment(string):
@@ -51,7 +63,7 @@ def find_layouts(file):
     file_contents = file_contents.replace('\\\n', '')
 
     for line in file_contents.split('\n'):
-        if line.startswith('#define') and '(' in line and 'LAYOUT' in line:
+        if layout_macro_define_regex.match(line.lstrip()) and '(' in line and 'LAYOUT' in line:
             # We've found a LAYOUT macro
             macro_name, layout, matrix = _parse_layout_macro(line.strip())
 
@@ -169,3 +181,110 @@ def _parse_matrix_locations(matrix, file, macro_name):
                 matrix_locations[identifier] = [row_num, col_num]
 
     return matrix_locations
+
+
+def _coerce_led_token(_type, value):
+    """ Convert token to valid info.json content
+    """
+    value_map = {
+        'NO_LED': None,
+        'LED_FLAG_ALL': 0xFF,
+        'LED_FLAG_NONE': 0x00,
+        'LED_FLAG_MODIFIER': 0x01,
+        'LED_FLAG_UNDERGLOW': 0x02,
+        'LED_FLAG_KEYLIGHT': 0x04,
+        'LED_FLAG_INDICATOR': 0x08,
+    }
+    if _type is Token.Literal.Number.Integer:
+        return int(value)
+    if _type is Token.Literal.Number.Float:
+        return float(value)
+    if _type is Token.Literal.Number.Hex:
+        return int(value, 0)
+    if _type is Token.Name and value in value_map.keys():
+        return value_map[value]
+
+
+def _parse_led_config(file, matrix_cols, matrix_rows):
+    """Return any 'raw' led/rgb matrix config
+    """
+    file_contents = file.read_text(encoding='utf-8')
+    file_contents = comment_remover(file_contents)
+    file_contents = file_contents.replace('\\\n', '')
+
+    matrix_raw = []
+    position_raw = []
+    flags = []
+
+    found_led_config = False
+    bracket_count = 0
+    section = 0
+    for _type, value in lex(file_contents, CLexer()):
+        # Assume g_led_config..stuff..;
+        if value == 'g_led_config':
+            found_led_config = True
+        elif value == ';':
+            found_led_config = False
+        elif found_led_config:
+            # Assume bracket count hints to section of config we are within
+            if value == '{':
+                bracket_count += 1
+                if bracket_count == 2:
+                    section += 1
+            elif value == '}':
+                bracket_count -= 1
+            else:
+                # Assume any non whitespace value here is important enough to stash
+                if _type in [Token.Literal.Number.Integer, Token.Literal.Number.Float, Token.Literal.Number.Hex, Token.Name]:
+                    if section == 1 and bracket_count == 3:
+                        matrix_raw.append(_coerce_led_token(_type, value))
+                    if section == 2 and bracket_count == 3:
+                        position_raw.append(_coerce_led_token(_type, value))
+                    if section == 3 and bracket_count == 2:
+                        flags.append(_coerce_led_token(_type, value))
+
+    # Slightly better intrim format
+    matrix = list(_get_chunks(matrix_raw, matrix_cols))
+    position = list(_get_chunks(position_raw, 2))
+    matrix_indexes = list(filter(lambda x: x is not None, matrix_raw))
+
+    # If we have not found anything - bail
+    if not section:
+        return None
+
+    # TODO: Improve crude parsing/validation
+    if len(matrix) != matrix_rows and len(matrix) != (matrix_rows / 2):
+        raise ValueError("Unable to parse g_led_config matrix data")
+    if len(position) != len(flags):
+        raise ValueError("Unable to parse g_led_config position data")
+    if len(matrix_indexes) and (max(matrix_indexes) >= len(flags)):
+        raise ValueError("OOB within g_led_config matrix data")
+
+    return (matrix, position, flags)
+
+
+def find_led_config(file, matrix_cols, matrix_rows):
+    """Search file for led/rgb matrix config
+    """
+    found = _parse_led_config(file, matrix_cols, matrix_rows)
+    if not found:
+        return None
+
+    # Expand collected content
+    (matrix, position, flags) = found
+
+    # Align to output format
+    led_config = []
+    for index, item in enumerate(position, start=0):
+        led_config.append({
+            'x': item[0],
+            'y': item[1],
+            'flags': flags[index],
+        })
+    for r in range(len(matrix)):
+        for c in range(len(matrix[r])):
+            index = matrix[r][c]
+            if index is not None:
+                led_config[index]['matrix'] = [r, c]
+
+    return led_config

lib/python/qmk/cli/__init__.py

@@ -16,7 +16,8 @@ import_names = {
     # A mapping of package name to importable name
     'pep8-naming': 'pep8ext_naming',
     'pyusb': 'usb.core',
-    'qmk-dotty-dict': 'dotty_dict'
+    'qmk-dotty-dict': 'dotty_dict',
+    'pillow': 'PIL'
 }
 
 safe_commands = [
@@ -51,6 +52,7 @@ subcommands = [
     'qmk.cli.generate.dfu_header',
     'qmk.cli.generate.docs',
     'qmk.cli.generate.info_json',
+    'qmk.cli.generate.keyboard_c',
     'qmk.cli.generate.keyboard_h',
     'qmk.cli.generate.layouts',
     'qmk.cli.generate.rgb_breathe_table',
@@ -67,6 +69,7 @@ subcommands = [
     'qmk.cli.multibuild',
     'qmk.cli.new.keyboard',
     'qmk.cli.new.keymap',
+    'qmk.cli.painter',
     'qmk.cli.pyformat',
     'qmk.cli.pytest',
     'qmk.cli.via2json',

lib/python/qmk/cli/format/c.py

@@ -9,7 +9,7 @@ from milc import cli
 from qmk.path import normpath
 from qmk.c_parse import c_source_files
 
-c_file_suffixes = ('c', 'h', 'cpp')
+c_file_suffixes = ('c', 'h', 'cpp', 'hpp')
 core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
 ignored = ('tmk_core/protocol/usb_hid', 'platforms/chibios/boards')
 

lib/python/qmk/cli/generate/api.py

@@ -1,7 +1,7 @@
 """This script automates the generation of the QMK API data.
 """
 from pathlib import Path
-from shutil import copyfile
+import shutil
 import json
 
 from milc import cli
@@ -12,28 +12,42 @@ from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
 
+TEMPLATE_PATH = Path('data/templates/api/')
+BUILD_API_PATH = Path('.build/api_data/')
+
 
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.")
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.")
 @cli.subcommand('Creates a new keymap for the keyboard of your choosing', hidden=False if cli.config.user.developer else True)
 def generate_api(cli):
     """Generates the QMK API data.
     """
-    api_data_dir = Path('api_data')
-    v1_dir = api_data_dir / 'v1'
+    if BUILD_API_PATH.exists():
+        shutil.rmtree(BUILD_API_PATH)
+
+    shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
+
+    v1_dir = BUILD_API_PATH / 'v1'
     keyboard_all_file = v1_dir / 'keyboards.json'  # A massive JSON containing everything
     keyboard_list_file = v1_dir / 'keyboard_list.json'  # A simple list of keyboard targets
     keyboard_aliases_file = v1_dir / 'keyboard_aliases.json'  # A list of historical keyboard names and their new name
     keyboard_metadata_file = v1_dir / 'keyboard_metadata.json'  # All the data configurator/via needs for initialization
     usb_file = v1_dir / 'usb.json'  # A mapping of USB VID/PID -> keyboard target
 
-    if not api_data_dir.exists():
-        api_data_dir.mkdir()
+    # Filter down when required
+    keyboard_list = list_keyboards()
+    if cli.args.filter:
+        kb_list = []
+        for keyboard_name in keyboard_list:
+            if any(i in keyboard_name for i in cli.args.filter):
+                kb_list.append(keyboard_name)
+        keyboard_list = kb_list
 
     kb_all = {}
     usb_list = {}
 
     # Generate and write keyboard specific JSON files
-    for keyboard_name in list_keyboards():
+    for keyboard_name in keyboard_list:
         kb_all[keyboard_name] = info_json(keyboard_name)
         keyboard_dir = v1_dir / 'keyboards' / keyboard_name
         keyboard_info = keyboard_dir / 'info.json'
@@ -47,7 +61,7 @@ def generate_api(cli):
             cli.log.debug('Wrote file %s', keyboard_info)
 
             if keyboard_readme_src:
-                copyfile(keyboard_readme_src, keyboard_readme)
+                shutil.copyfile(keyboard_readme_src, keyboard_readme)
                 cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme)
 
         if 'usb' in kb_all[keyboard_name]:

lib/python/qmk/cli/generate/config_h.py

@@ -5,10 +5,9 @@ from pathlib import Path
 from dotty_dict import dotty
 from milc import cli
 
-from qmk.info import info_json
-from qmk.json_schema import json_load, validate
+from qmk.info import info_json, keymap_json_config
+from qmk.json_schema import json_load
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import locate_keymap
 from qmk.commands import dump_lines
 from qmk.path import normpath
 from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
@@ -84,7 +83,7 @@ def generate_config_items(kb_info_json, config_h_lines):
 
     for config_key, info_dict in info_config_map.items():
         info_key = info_dict['info_key']
-        key_type = info_dict.get('value_type', 'str')
+        key_type = info_dict.get('value_type', 'raw')
         to_config = info_dict.get('to_config', True)
 
         if not to_config:
@@ -95,7 +94,12 @@ def generate_config_items(kb_info_json, config_h_lines):
         except KeyError:
             continue
 
-        if key_type.startswith('array'):
+        if key_type.startswith('array.array'):
+            config_h_lines.append('')
+            config_h_lines.append(f'#ifndef {config_key}')
+            config_h_lines.append(f'#   define {config_key} {{ {", ".join(["{" + ",".join(list(map(str, x))) + "}" for x in config_value])} }}')
+            config_h_lines.append(f'#endif // {config_key}')
+        elif key_type.startswith('array'):
             config_h_lines.append('')
             config_h_lines.append(f'#ifndef {config_key}')
             config_h_lines.append(f'#   define {config_key} {{ {", ".join(map(str, config_value))} }}')
@@ -112,6 +116,11 @@ def generate_config_items(kb_info_json, config_h_lines):
                 config_h_lines.append(f'#ifndef {key}')
                 config_h_lines.append(f'#   define {key} {value}')
                 config_h_lines.append(f'#endif // {key}')
+        elif key_type == 'str':
+            config_h_lines.append('')
+            config_h_lines.append(f'#ifndef {config_key}')
+            config_h_lines.append(f'#   define {config_key} "{config_value}"')
+            config_h_lines.append(f'#endif // {config_key}')
         elif key_type == 'bcd_version':
             (major, minor, revision) = config_value.split('.')
             config_h_lines.append('')
@@ -175,10 +184,7 @@ def generate_config_h(cli):
     """
     # Determine our keyboard/keymap
     if cli.args.keymap:
-        km = locate_keymap(cli.args.keyboard, cli.args.keymap)
-        km_json = json_load(km)
-        validate(km_json, 'qmk.keymap.v1')
-        kb_info_json = dotty(km_json.get('config', {}))
+        kb_info_json = dotty(keymap_json_config(cli.args.keyboard, cli.args.keymap))
     else:
         kb_info_json = dotty(info_json(cli.args.keyboard))
 

lib/python/qmk/cli/generate/docs.py

@@ -10,6 +10,7 @@ DOCS_PATH = Path('docs/')
 BUILD_PATH = Path('.build/')
 BUILD_DOCS_PATH = BUILD_PATH / 'docs'
 DOXYGEN_PATH = BUILD_PATH / 'doxygen'
+MOXYGEN_PATH = BUILD_DOCS_PATH / 'internals'
 
 
 @cli.subcommand('Build QMK documentation.', hidden=False if cli.config.user.developer else True)
@@ -34,10 +35,10 @@ def generate_docs(cli):
         'stdin': DEVNULL,
     }
 
-    cli.log.info('Generating internal docs...')
+    cli.log.info('Generating docs...')
 
     # Generate internal docs
     cli.run(['doxygen', 'Doxyfile'], **args)
-    cli.run(['moxygen', '-q', '-g', '-o', BUILD_DOCS_PATH / 'internals_%s.md', DOXYGEN_PATH / 'xml'], **args)
+    cli.run(['moxygen', '-q', '-g', '-o', MOXYGEN_PATH / '%s.md', DOXYGEN_PATH / 'xml'], **args)
 
-    cli.log.info('Successfully generated internal docs to %s.', BUILD_DOCS_PATH)
+    cli.log.info('Successfully generated docs to %s.', BUILD_DOCS_PATH)

lib/python/qmk/cli/generate/keyboard_c.py

@@ -0,0 +1,75 @@
+"""Used by the make system to generate keyboard.c from info.json.
+"""
+from milc import cli
+
+from qmk.info import info_json
+from qmk.commands import dump_lines
+from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.path import normpath
+from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+
+
+def _gen_led_config(info_data):
+    """Convert info.json content to g_led_config
+    """
+    cols = info_data['matrix_size']['cols']
+    rows = info_data['matrix_size']['rows']
+
+    config_type = None
+    if 'layout' in info_data.get('rgb_matrix', {}):
+        config_type = 'rgb_matrix'
+    elif 'layout' in info_data.get('led_matrix', {}):
+        config_type = 'led_matrix'
+
+    lines = []
+    if not config_type:
+        return lines
+
+    matrix = [['NO_LED'] * cols for i in range(rows)]
+    pos = []
+    flags = []
+
+    led_config = info_data[config_type]['layout']
+    for index, item in enumerate(led_config, start=0):
+        if 'matrix' in item:
+            (x, y) = item['matrix']
+            matrix[x][y] = str(index)
+        pos.append(f'{{ {item.get("x", 0)},{item.get("y", 0)} }}')
+        flags.append(str(item.get('flags', 0)))
+
+    if config_type == 'rgb_matrix':
+        lines.append('#ifdef RGB_MATRIX_ENABLE')
+        lines.append('#include "rgb_matrix.h"')
+    elif config_type == 'led_matrix':
+        lines.append('#ifdef LED_MATRIX_ENABLE')
+        lines.append('#include "led_matrix.h"')
+
+    lines.append('__attribute__ ((weak)) led_config_t g_led_config = {')
+    lines.append('  {')
+    for line in matrix:
+        lines.append(f'    {{ {",".join(line)} }},')
+    lines.append('  },')
+    lines.append(f'  {{ {",".join(pos)} }},')
+    lines.append(f'  {{ {",".join(flags)} }},')
+    lines.append('};')
+    lines.append('#endif')
+
+    return lines
+
+
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.c for.')
+@cli.subcommand('Used by the make system to generate keyboard.c from info.json', hidden=True)
+def generate_keyboard_c(cli):
+    """Generates the keyboard.h file.
+    """
+    kb_info_json = info_json(cli.args.keyboard)
+
+    # Build the layouts.h file.
+    keyboard_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#include QMK_KEYBOARD_H', '']
+
+    keyboard_h_lines.extend(_gen_led_config(kb_info_json))
+
+    # Show the results
+    dump_lines(cli.args.output, keyboard_h_lines, cli.args.quiet)

lib/python/qmk/cli/generate/rgb_breathe_table.py

@@ -34,7 +34,7 @@ def generate_rgb_breathe_table(cli):
     """
     breathe_values = [0] * 256
     for pos in range(0, 256):
-        breathe_values[pos] = (int)((math.exp(math.sin((pos/255) * math.pi)) - cli.args.center / math.e) * (cli.args.max / (math.e - 1 / math.e)))  # noqa: yapf insists there be no whitespace around /
+        breathe_values[pos] = (int)((math.exp(math.sin((pos / 255) * math.pi)) - cli.args.center / math.e) * (cli.args.max / (math.e - 1 / math.e)))
 
     values_template = ''
     for s in range(0, 3):
@@ -46,7 +46,7 @@ def generate_rgb_breathe_table(cli):
             values_template += '    ' if pos % 8 == 0 else ''
             values_template += '0x{:02X}'.format(breathe_values[pos])
             values_template += ',' if (pos + step) < 256 else ''
-            values_template += '\n' if (pos+step) % 8 == 0 else ' '  # noqa: yapf insists there be no whitespace around +
+            values_template += '\n' if (pos + step) % 8 == 0 else ' '
 
         values_template += '#endif'
         values_template += '\n\n' if s < 2 else ''

lib/python/qmk/cli/generate/rules_mk.py

@@ -5,10 +5,9 @@ from pathlib import Path
 from dotty_dict import dotty
 from milc import cli
 
-from qmk.info import info_json
-from qmk.json_schema import json_load, validate
+from qmk.info import info_json, keymap_json_config
+from qmk.json_schema import json_load
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import locate_keymap
 from qmk.commands import dump_lines
 from qmk.path import normpath
 from qmk.constants import GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE
@@ -21,7 +20,7 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict):
         return None
 
     info_key = info_dict['info_key']
-    key_type = info_dict.get('value_type', 'str')
+    key_type = info_dict.get('value_type', 'raw')
 
     try:
         rules_value = kb_info_json[info_key]
@@ -34,6 +33,8 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict):
         return f'{rules_key} ?= {"yes" if rules_value else "no"}'
     elif key_type == 'mapping':
         return '\n'.join([f'{key} ?= {value}' for key, value in rules_value.items()])
+    elif key_type == 'str':
+        return f'{rules_key} ?= "{rules_value}"'
 
     return f'{rules_key} ?= {rules_value}'
 
@@ -49,10 +50,7 @@ def generate_rules_mk(cli):
     """
     # Determine our keyboard/keymap
     if cli.args.keymap:
-        km = locate_keymap(cli.args.keyboard, cli.args.keymap)
-        km_json = json_load(km)
-        validate(km_json, 'qmk.keymap.v1')
-        kb_info_json = dotty(km_json.get('config', {}))
+        kb_info_json = dotty(keymap_json_config(cli.args.keyboard, cli.args.keymap))
     else:
         kb_info_json = dotty(info_json(cli.args.keyboard))
 

lib/python/qmk/cli/info.py

@@ -11,8 +11,8 @@ from qmk.json_encoders import InfoJSONEncoder
 from qmk.constants import COL_LETTERS, ROW_LETTERS
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder, render_layouts, render_layout, rules_mk
+from qmk.info import info_json, keymap_json
 from qmk.keymap import locate_keymap
-from qmk.info import info_json
 from qmk.path import is_keyboard
 
 UNICODE_SUPPORT = sys.stdout.encoding.lower().startswith('utf')
@@ -135,7 +135,7 @@ def print_parsed_rules_mk(keyboard_name):
 
 
 @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to show info for.')
-@cli.argument('-km', '--keymap', help='Show the layers for a JSON keymap too.')
+@cli.argument('-km', '--keymap', help='Keymap to show info for (Optional).')
 @cli.argument('-l', '--layouts', action='store_true', help='Render the layouts.')
 @cli.argument('-m', '--matrix', action='store_true', help='Render the layouts with matrix information.')
 @cli.argument('-f', '--format', default='friendly', arg_only=True, help='Format to display the data in (friendly, text, json) (Default: friendly).')
@@ -161,8 +161,15 @@ def info(cli):
         print_parsed_rules_mk(cli.config.info.keyboard)
         return False
 
+    # default keymap stored in config file should be ignored
+    if cli.config_source.info.keymap == 'config_file':
+        cli.config_source.info.keymap = None
+
     # Build the info.json file
-    kb_info_json = info_json(cli.config.info.keyboard)
+    if cli.config.info.keymap:
+        kb_info_json = keymap_json(cli.config.info.keyboard, cli.config.info.keymap)
+    else:
+        kb_info_json = info_json(cli.config.info.keyboard)
 
     # Output in the requested format
     if cli.args.format == 'json':
@@ -178,11 +185,12 @@ def info(cli):
         cli.log.error('Unknown format: %s', cli.args.format)
         return False
 
+    # Output requested extras
     if cli.config.info.layouts:
         show_layouts(kb_info_json, title_caps)
 
     if cli.config.info.matrix:
         show_matrix(kb_info_json, title_caps)
 
-    if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file':
+    if cli.config.info.keymap:
         show_keymap(kb_info_json, title_caps)

lib/python/qmk/cli/lint.py

@@ -116,6 +116,13 @@ def lint(cli):
             if not keymap_check(kb, cli.config.lint.keymap):
                 ok = False
 
+        # Check if all non-data driven macros exist in <keyboard.h>
+        for layout, data in keyboard_info['layouts'].items():
+            # Matrix data should be a list with exactly two integers: [0, 1]
+            if not data['c_macro'] and not all('matrix' in key_data.keys() or len(key_data) == 2 or all(isinstance(n, int) for n in key_data) for key_data in data['layout']):
+                cli.log.error(f'{kb}: "{layout}" has no "matrix" definition in either "info.json" or "<keyboard>.h"!')
+                ok = False
+
         # Report status
         if not ok:
             failed.append(kb)

lib/python/qmk/cli/new/keyboard.py

@@ -14,7 +14,7 @@ from qmk.git import git_get_username
 from qmk.json_schema import load_jsonschema
 from qmk.path import keyboard
 from qmk.json_encoders import InfoJSONEncoder
-from qmk.json_schema import deep_update
+from qmk.json_schema import deep_update, json_load
 from qmk.constants import MCU2BOOTLOADER
 
 COMMUNITY = Path('layouts/community/')
@@ -23,13 +23,14 @@ TEMPLATE = Path('data/templates/keyboard/')
 # defaults
 schema = dotty(load_jsonschema('keyboard'))
 mcu_types = sorted(schema["properties.processor.enum"], key=str.casefold)
+dev_boards = sorted(schema["properties.development_board.enum"], key=str.casefold)
 available_layouts = sorted([x.name for x in COMMUNITY.iterdir() if x.is_dir()])
 
 
 def mcu_type(mcu):
     """Callable for argparse validation.
     """
-    if mcu not in mcu_types:
+    if mcu not in (dev_boards + mcu_types):
         raise ValueError
     return mcu
 
@@ -176,14 +177,14 @@ https://docs.qmk.fm/#/compatible_microcontrollers
 
 MCU? """
     # remove any options strictly used for compatibility
-    filtered_mcu = [x for x in mcu_types if not any(xs in x for xs in ['cortex', 'unknown'])]
+    filtered_mcu = [x for x in (dev_boards + mcu_types) if not any(xs in x for xs in ['cortex', 'unknown'])]
 
     return choice(prompt, filtered_mcu, default=filtered_mcu.index("atmega32u4"))
 
 
 @cli.argument('-kb', '--keyboard', help='Specify the name for the new keyboard directory', arg_only=True, type=keyboard_name)
 @cli.argument('-l', '--layout', help='Community layout to bootstrap with', arg_only=True, type=layout_type)
-@cli.argument('-t', '--type', help='Specify the keyboard MCU type', arg_only=True, type=mcu_type)
+@cli.argument('-t', '--type', help='Specify the keyboard MCU type (or "development_board" preset)', arg_only=True, type=mcu_type)
 @cli.argument('-u', '--username', help='Specify your username (default from Git config)', dest='name')
 @cli.argument('-n', '--realname', help='Specify your real name if you want to use that. Defaults to username', arg_only=True)
 @cli.subcommand('Creates a new keyboard directory')
@@ -198,7 +199,6 @@ def new_keyboard(cli):
     real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
     default_layout = cli.args.layout if cli.args.layout else prompt_layout()
     mcu = cli.args.type if cli.args.type else prompt_mcu()
-    bootloader = select_default_bootloader(mcu)
 
     if not validate_keyboard_name(kb_name):
         cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.')
@@ -208,6 +208,16 @@ def new_keyboard(cli):
         cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} already exists! Please choose a different name.')
         return 1
 
+    # Preprocess any development_board presets
+    if mcu in dev_boards:
+        defaults_map = json_load(Path('data/mappings/defaults.json'))
+        board = defaults_map['development_board'][mcu]
+
+        mcu = board['processor']
+        bootloader = board['bootloader']
+    else:
+        bootloader = select_default_bootloader(mcu)
+
     tokens = {  # Comment here is to force multiline formatting
         'YEAR': str(date.today().year),
         'KEYBOARD': kb_name,

lib/python/qmk/cli/painter/__init__.py

@@ -0,0 +1,2 @@
+from . import convert_graphics
+from . import make_font

lib/python/qmk/cli/painter/convert_graphics.py

@@ -0,0 +1,86 @@
+"""This script tests QGF functionality.
+"""
+import re
+import datetime
+from io import BytesIO
+from qmk.path import normpath
+from qmk.painter import render_header, render_source, render_license, render_bytes, valid_formats
+from milc import cli
+from PIL import Image
+
+
+@cli.argument('-v', '--verbose', arg_only=True, action='store_true', help='Turns on verbose output.')
+@cli.argument('-i', '--input', required=True, help='Specify input graphic file.')
+@cli.argument('-o', '--output', default='', help='Specify output directory. Defaults to same directory as input.')
+@cli.argument('-f', '--format', required=True, help='Output format, valid types: %s' % (', '.join(valid_formats.keys())))
+@cli.argument('-r', '--no-rle', arg_only=True, action='store_true', help='Disables the use of RLE when encoding images.')
+@cli.argument('-d', '--no-deltas', arg_only=True, action='store_true', help='Disables the use of delta frames when encoding animations.')
+@cli.subcommand('Converts an input image to something QMK understands')
+def painter_convert_graphics(cli):
+    """Converts an image file to a format that Quantum Painter understands.
+
+    This command uses the `qmk.painter` module to generate a Quantum Painter image defintion from an image. The generated definitions are written to a files next to the input -- `INPUT.c` and `INPUT.h`.
+    """
+    # Work out the input file
+    if cli.args.input != '-':
+        cli.args.input = normpath(cli.args.input)
+
+        # Error checking
+        if not cli.args.input.exists():
+            cli.log.error('Input image file does not exist!')
+            cli.print_usage()
+            return False
+
+    # Work out the output directory
+    if len(cli.args.output) == 0:
+        cli.args.output = cli.args.input.parent
+    cli.args.output = normpath(cli.args.output)
+
+    # Ensure we have a valid format
+    if cli.args.format not in valid_formats.keys():
+        cli.log.error('Output format %s is invalid. Allowed values: %s' % (cli.args.format, ', '.join(valid_formats.keys())))
+        cli.print_usage()
+        return False
+
+    # Work out the encoding parameters
+    format = valid_formats[cli.args.format]
+
+    # Load the input image
+    input_img = Image.open(cli.args.input)
+
+    # Convert the image to QGF using PIL
+    out_data = BytesIO()
+    input_img.save(out_data, "QGF", use_deltas=(not cli.args.no_deltas), use_rle=(not cli.args.no_rle), qmk_format=format, verbose=cli.args.verbose)
+    out_bytes = out_data.getvalue()
+
+    # Work out the text substitutions for rendering the output data
+    subs = {
+        'generated_type': 'image',
+        'var_prefix': 'gfx',
+        'generator_command': f'qmk painter-convert-graphics -i {cli.args.input.name} -f {cli.args.format}',
+        'year': datetime.date.today().strftime("%Y"),
+        'input_file': cli.args.input.name,
+        'sane_name': re.sub(r"[^a-zA-Z0-9]", "_", cli.args.input.stem),
+        'byte_count': len(out_bytes),
+        'bytes_lines': render_bytes(out_bytes),
+        'format': cli.args.format,
+    }
+
+    # Render the license
+    subs.update({'license': render_license(subs)})
+
+    # Render and write the header file
+    header_text = render_header(subs)
+    header_file = cli.args.output / (cli.args.input.stem + ".qgf.h")
+    with open(header_file, 'w') as header:
+        print(f"Writing {header_file}...")
+        header.write(header_text)
+        header.close()
+
+    # Render and write the source file
+    source_text = render_source(subs)
+    source_file = cli.args.output / (cli.args.input.stem + ".qgf.c")
+    with open(source_file, 'w') as source:
+        print(f"Writing {source_file}...")
+        source.write(source_text)
+        source.close()

lib/python/qmk/cli/painter/make_font.py

@@ -0,0 +1,87 @@
+"""This script automates the conversion of font files into a format QMK firmware understands.
+"""
+
+import re
+import datetime
+from io import BytesIO
+from qmk.path import normpath
+from qmk.painter_qff import QFFFont
+from qmk.painter import render_header, render_source, render_license, render_bytes, valid_formats
+from milc import cli
+
+
+@cli.argument('-f', '--font', required=True, help='Specify input font file.')
+@cli.argument('-o', '--output', required=True, help='Specify output image path.')
+@cli.argument('-s', '--size', default=12, help='Specify font size. Default 12.')
+@cli.argument('-n', '--no-ascii', arg_only=True, action='store_true', help='Disables output of the full ASCII character set (0x20..0x7E), exporting only the glyphs specified.')
+@cli.argument('-u', '--unicode-glyphs', default='', help='Also generate the specified unicode glyphs.')
+@cli.argument('-a', '--no-aa', arg_only=True, action='store_true', help='Disable anti-aliasing on fonts.')
+@cli.subcommand('Converts an input font to something QMK understands')
+def painter_make_font_image(cli):
+    # Create the font object
+    font = QFFFont(cli)
+    # Read from the input file
+    cli.args.font = normpath(cli.args.font)
+    font.generate_image(cli.args.font, cli.args.size, include_ascii_glyphs=(not cli.args.no_ascii), unicode_glyphs=cli.args.unicode_glyphs, use_aa=(False if cli.args.no_aa else True))
+    # Render out the data
+    font.save_to_image(normpath(cli.args.output))
+
+
+@cli.argument('-i', '--input', help='Specify input graphic file.')
+@cli.argument('-o', '--output', default='', help='Specify output directory. Defaults to same directory as input.')
+@cli.argument('-n', '--no-ascii', arg_only=True, action='store_true', help='Disables output of the full ASCII character set (0x20..0x7E), exporting only the glyphs specified.')
+@cli.argument('-u', '--unicode-glyphs', default='', help='Also generate the specified unicode glyphs.')
+@cli.argument('-f', '--format', required=True, help='Output format, valid types: %s' % (', '.join(valid_formats.keys())))
+@cli.argument('-r', '--no-rle', arg_only=True, action='store_true', help='Disable the use of RLE to minimise converted image size.')
+@cli.subcommand('Converts an input font image to something QMK firmware understands')
+def painter_convert_font_image(cli):
+    # Work out the format
+    format = valid_formats[cli.args.format]
+
+    # Create the font object
+    font = QFFFont(cli.log)
+
+    # Read from the input file
+    cli.args.input = normpath(cli.args.input)
+    font.read_from_image(cli.args.input, include_ascii_glyphs=(not cli.args.no_ascii), unicode_glyphs=cli.args.unicode_glyphs)
+
+    # Work out the output directory
+    if len(cli.args.output) == 0:
+        cli.args.output = cli.args.input.parent
+    cli.args.output = normpath(cli.args.output)
+
+    # Render out the data
+    out_data = BytesIO()
+    font.save_to_qff(format, (False if cli.args.no_rle else True), out_data)
+
+    # Work out the text substitutions for rendering the output data
+    subs = {
+        'generated_type': 'font',
+        'var_prefix': 'font',
+        'generator_command': f'qmk painter-convert-font-image -i {cli.args.input.name} -f {cli.args.format}',
+        'year': datetime.date.today().strftime("%Y"),
+        'input_file': cli.args.input.name,
+        'sane_name': re.sub(r"[^a-zA-Z0-9]", "_", cli.args.input.stem),
+        'byte_count': out_data.getbuffer().nbytes,
+        'bytes_lines': render_bytes(out_data.getbuffer().tobytes()),
+        'format': cli.args.format,
+    }
+
+    # Render the license
+    subs.update({'license': render_license(subs)})
+
+    # Render and write the header file
+    header_text = render_header(subs)
+    header_file = cli.args.output / (cli.args.input.stem + ".qff.h")
+    with open(header_file, 'w') as header:
+        print(f"Writing {header_file}...")
+        header.write(header_text)
+        header.close()
+
+    # Render and write the source file
+    source_text = render_source(subs)
+    source_file = cli.args.output / (cli.args.input.stem + ".qff.c")
+    with open(source_file, 'w') as source:
+        print(f"Writing {source_file}...")
+        source.write(source_text)
+        source.close()

lib/python/qmk/cli/pytest.py

@@ -12,8 +12,7 @@ from milc import cli
 def pytest(cli):
     """Run several linting/testing commands.
     """
-    nose2 = cli.run(['nose2', '-v', '-t'
-                     'lib/python', *cli.args.test], capture_output=False, stdin=DEVNULL)
+    nose2 = cli.run(['nose2', '-v', '-t', 'lib/python', *cli.args.test], capture_output=False, stdin=DEVNULL)
     flake8 = cli.run(['flake8', 'lib/python'], capture_output=False, stdin=DEVNULL)
 
     return flake8.returncode | nose2.returncode

lib/python/qmk/commands.py

@@ -228,7 +228,7 @@ def dump_lines(output_file, lines, quiet=True):
         output_file.parent.mkdir(parents=True, exist_ok=True)
         if output_file.exists():
             output_file.replace(output_file.parent / (output_file.name + '.bak'))
-        output_file.write_text(generated)
+        output_file.write_text(generated, encoding='utf-8')
 
         if not quiet:
             cli.log.info(f'Wrote {output_file.name} to {output_file}.')

lib/python/qmk/constants.py

@@ -14,7 +14,7 @@ QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
 MAX_KEYBOARD_SUBFOLDERS = 5
 
 # Supported processor types
-CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66FX1M0', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F405', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L432', 'STM32L433', 'STM32L442', 'STM32L443', 'GD32VF103', 'WB32F3G71'
+CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66FX1M0', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F405', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L432', 'STM32L433', 'STM32L442', 'STM32L443', 'GD32VF103', 'WB32F3G71', 'WB32FQ95'
 LUFA_PROCESSORS = 'at90usb162', 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None
 VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'
 
@@ -43,6 +43,7 @@ MCU2BOOTLOADER = {
     "STM32L443": "stm32-dfu",
     "GD32VF103": "gd32v-dfu",
     "WB32F3G71": "wb32-dfu",
+    "WB32FQ95": "wb32-dfu",
     "atmega16u2": "atmel-dfu",
     "atmega32u2": "atmel-dfu",
     "atmega16u4": "atmel-dfu",

lib/python/qmk/info.py

@@ -8,10 +8,11 @@ from dotty_dict import dotty
 from milc import cli
 
 from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
-from qmk.c_parse import find_layouts
+from qmk.c_parse import find_layouts, parse_config_h_file, find_led_config
 from qmk.json_schema import deep_update, json_load, validate
 from qmk.keyboard import config_h, rules_mk
-from qmk.keymap import list_keymaps
+from qmk.keymap import list_keymaps, locate_keymap
+from qmk.commands import parse_configurator_json
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
 
@@ -68,12 +69,16 @@ def info_json(keyboard):
 
     # Merge in the data from info.json, config.h, and rules.mk
     info_data = merge_info_jsons(keyboard, info_data)
-    info_data = _extract_rules_mk(info_data)
-    info_data = _extract_config_h(info_data)
+    info_data = _process_defaults(info_data)
+    info_data = _extract_rules_mk(info_data, rules_mk(str(keyboard)))
+    info_data = _extract_config_h(info_data, config_h(str(keyboard)))
 
     # Ensure that we have matrix row and column counts
     info_data = _matrix_size(info_data)
 
+    # Merge in data from <keyboard.c>
+    info_data = _extract_led_config(info_data, str(keyboard))
+
     # Validate against the jsonschema
     try:
         validate(info_data, 'qmk.api.keyboard.v1')
@@ -166,28 +171,46 @@ def _extract_pins(pins):
     return [_pin_name(pin) for pin in pins.split(',')]
 
 
-def _extract_direct_matrix(direct_pins):
+def _extract_2d_array(raw):
+    """Return a 2d array of strings
     """
-    """
-    direct_pin_array = []
+    out_array = []
 
-    while direct_pins[-1] != '}':
-        direct_pins = direct_pins[:-1]
+    while raw[-1] != '}':
+        raw = raw[:-1]
 
-    for row in direct_pins.split('},{'):
+    for row in raw.split('},{'):
         if row.startswith('{'):
             row = row[1:]
 
         if row.endswith('}'):
             row = row[:-1]
 
-        direct_pin_array.append([])
+        out_array.append([])
 
-        for pin in row.split(','):
-            if pin == 'NO_PIN':
-                pin = None
+        for val in row.split(','):
+            out_array[-1].append(val)
 
-            direct_pin_array[-1].append(pin)
+    return out_array
+
+
+def _extract_2d_int_array(raw):
+    """Return a 2d array of ints
+    """
+    ret = _extract_2d_array(raw)
+
+    return [list(map(int, x)) for x in ret]
+
+
+def _extract_direct_matrix(direct_pins):
+    """extract direct_matrix
+    """
+    direct_pin_array = _extract_2d_array(direct_pins)
+
+    for i in range(len(direct_pin_array)):
+        for j in range(len(direct_pin_array[i])):
+            if direct_pin_array[i][j] == 'NO_PIN':
+                direct_pin_array[i][j] = None
 
     return direct_pin_array
 
@@ -205,6 +228,21 @@ def _extract_audio(info_data, config_c):
         info_data['audio'] = {'pins': audio_pins}
 
 
+def _extract_secure_unlock(info_data, config_c):
+    """Populate data about the secure unlock sequence
+    """
+    unlock = config_c.get('SECURE_UNLOCK_SEQUENCE', '').replace(' ', '')[1:-1]
+    if unlock:
+        unlock_array = _extract_2d_int_array(unlock)
+        if 'secure' not in info_data:
+            info_data['secure'] = {}
+
+        if 'unlock_sequence' in info_data['secure']:
+            _log_warning(info_data, 'Secure unlock sequence is specified in both config.h (SECURE_UNLOCK_SEQUENCE) and info.json (secure.unlock_sequence) (Value: %s), the config.h value wins.' % info_data['secure']['unlock_sequence'])
+
+        info_data['secure']['unlock_sequence'] = unlock_array
+
+
 def _extract_split_main(info_data, config_c):
     """Populate data about the split configuration
     """
@@ -270,14 +308,16 @@ def _extract_split_transport(info_data, config_c):
 
         info_data['split']['transport']['protocol'] = 'i2c'
 
-    elif 'protocol' not in info_data.get('split', {}).get('transport', {}):
+    # Ignore transport defaults if "SPLIT_KEYBOARD" is unset
+    elif 'enabled' in info_data.get('split', {}):
         if 'split' not in info_data:
             info_data['split'] = {}
 
         if 'transport' not in info_data['split']:
             info_data['split']['transport'] = {}
 
-        info_data['split']['transport']['protocol'] = 'serial'
+        if 'protocol' not in info_data['split']['transport']:
+            info_data['split']['transport']['protocol'] = 'serial'
 
 
 def _extract_split_right_pins(info_data, config_c):
@@ -400,18 +440,16 @@ def _extract_device_version(info_data):
             info_data['usb']['device_version'] = f'{major}.{minor}.{revision}'
 
 
-def _extract_config_h(info_data):
+def _extract_config_h(info_data, config_c):
     """Pull some keyboard information from existing config.h files
     """
-    config_c = config_h(info_data['keyboard_folder'])
-
     # Pull in data from the json map
     dotty_info = dotty(info_data)
     info_config_map = json_load(Path('data/mappings/info_config.json'))
 
     for config_key, info_dict in info_config_map.items():
         info_key = info_dict['info_key']
-        key_type = info_dict.get('value_type', 'str')
+        key_type = info_dict.get('value_type', 'raw')
 
         try:
             if config_key in config_c and info_dict.get('to_json', True):
@@ -443,6 +481,9 @@ def _extract_config_h(info_data):
                 elif key_type == 'int':
                     dotty_info[info_key] = int(config_c[config_key])
 
+                elif key_type == 'str':
+                    dotty_info[info_key] = config_c[config_key].strip('"')
+
                 elif key_type == 'bcd_version':
                     major = int(config_c[config_key][2:4])
                     minor = int(config_c[config_key][4])
@@ -461,6 +502,7 @@ def _extract_config_h(info_data):
     # Pull data that easily can't be mapped in json
     _extract_matrix_info(info_data, config_c)
     _extract_audio(info_data, config_c)
+    _extract_secure_unlock(info_data, config_c)
     _extract_split_main(info_data, config_c)
     _extract_split_transport(info_data, config_c)
     _extract_split_right_pins(info_data, config_c)
@@ -469,10 +511,21 @@ def _extract_config_h(info_data):
     return info_data
 
 
-def _extract_rules_mk(info_data):
+def _process_defaults(info_data):
+    """Process any additional defaults based on currently discovered information
+    """
+    defaults_map = json_load(Path('data/mappings/defaults.json'))
+    for default_type in defaults_map.keys():
+        thing_map = defaults_map[default_type]
+        if default_type in info_data:
+            for key, value in thing_map.get(info_data[default_type], {}).items():
+                info_data[key] = value
+    return info_data
+
+
+def _extract_rules_mk(info_data, rules):
     """Pull some keyboard information from existing rules.mk files
     """
-    rules = rules_mk(info_data['keyboard_folder'])
     info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
 
     if info_data['processor'] in CHIBIOS_PROCESSORS:
@@ -491,7 +544,7 @@ def _extract_rules_mk(info_data):
 
     for rules_key, info_dict in info_rules_map.items():
         info_key = info_dict['info_key']
-        key_type = info_dict.get('value_type', 'str')
+        key_type = info_dict.get('value_type', 'raw')
 
         try:
             if rules_key in rules and info_dict.get('to_json', True):
@@ -523,6 +576,9 @@ def _extract_rules_mk(info_data):
                 elif key_type == 'int':
                     dotty_info[info_key] = int(rules[rules_key])
 
+                elif key_type == 'str':
+                    dotty_info[info_key] = rules[rules_key].strip('"')
+
                 else:
                     dotty_info[info_key] = rules[rules_key]
 
@@ -537,6 +593,46 @@ def _extract_rules_mk(info_data):
     return info_data
 
 
+def find_keyboard_c(keyboard):
+    """Find all <keyboard>.c files
+    """
+    keyboard = Path(keyboard)
+    current_path = Path('keyboards/')
+
+    files = []
+    for directory in keyboard.parts:
+        current_path = current_path / directory
+        keyboard_c_path = current_path / f'{directory}.c'
+        if keyboard_c_path.exists():
+            files.append(keyboard_c_path)
+
+    return files
+
+
+def _extract_led_config(info_data, keyboard):
+    """Scan all <keyboard>.c files for led config
+    """
+    cols = info_data['matrix_size']['cols']
+    rows = info_data['matrix_size']['rows']
+
+    # Assume what feature owns g_led_config
+    feature = "rgb_matrix"
+    if info_data.get("features", {}).get("led_matrix", False):
+        feature = "led_matrix"
+
+    # Process
+    for file in find_keyboard_c(keyboard):
+        try:
+            ret = find_led_config(file, cols, rows)
+            if ret:
+                info_data[feature] = info_data.get(feature, {})
+                info_data[feature]["layout"] = ret
+        except Exception as e:
+            _log_warning(info_data, f'led_config: {file.name}: {e}')
+
+    return info_data
+
+
 def _matrix_size(info_data):
     """Add info_data['matrix_size'] if it doesn't exist.
     """
@@ -753,10 +849,43 @@ def find_info_json(keyboard):
 
     # Add in parent folders for least specific
     for _ in range(5):
-        info_jsons.append(keyboard_parent / 'info.json')
-        if keyboard_parent.parent == base_path:
+        if keyboard_parent == base_path:
             break
+        info_jsons.append(keyboard_parent / 'info.json')
         keyboard_parent = keyboard_parent.parent
 
     # Return a list of the info.json files that actually exist
     return [info_json for info_json in info_jsons if info_json.exists()]
+
+
+def keymap_json_config(keyboard, keymap):
+    """Extract keymap level config
+    """
+    keymap_folder = locate_keymap(keyboard, keymap).parent
+
+    km_info_json = parse_configurator_json(keymap_folder / 'keymap.json')
+    return km_info_json.get('config', {})
+
+
+def keymap_json(keyboard, keymap):
+    """Generate the info.json data for a specific keymap.
+    """
+    keymap_folder = locate_keymap(keyboard, keymap).parent
+
+    # Files to scan
+    keymap_config = keymap_folder / 'config.h'
+    keymap_rules = keymap_folder / 'rules.mk'
+    keymap_file = keymap_folder / 'keymap.json'
+
+    # Build the info.json file
+    kb_info_json = info_json(keyboard)
+
+    # Merge in the data from keymap.json
+    km_info_json = keymap_json_config(keyboard, keymap) if keymap_file.exists() else {}
+    deep_update(kb_info_json, km_info_json)
+
+    # Merge in the data from config.h, and rules.mk
+    _extract_rules_mk(kb_info_json, parse_rules_mk_file(keymap_rules))
+    _extract_config_h(kb_info_json, parse_config_h_file(keymap_config))
+
+    return kb_info_json

lib/python/qmk/json_encoders.py

@@ -75,8 +75,8 @@ class InfoJSONEncoder(QMKJSONEncoder):
         """Encode info.json dictionaries.
         """
         if obj:
-            if self.indentation_level == 4:
-                # These are part of a layout, put them on a single line.
+            if set(("x", "y")).issubset(obj.keys()):
+                # These are part of a layout/led_config, put them on a single line.
                 return "{ " + ", ".join(f"{self.encode(key)}: {self.encode(element)}" for key, element in sorted(obj.items())) + " }"
 
             else:

lib/python/qmk/json_schema.py

@@ -68,7 +68,11 @@ def create_validator(schema):
     schema_store = compile_schema_store()
     resolver = jsonschema.RefResolver.from_schema(schema_store[schema], store=schema_store)
 
-    return jsonschema.Draft7Validator(schema_store[schema], resolver=resolver).validate
+    # TODO: Remove this after the jsonschema>=4 requirement had time to reach users
+    try:
+        return jsonschema.Draft202012Validator(schema_store[schema], resolver=resolver).validate
+    except AttributeError:
+        return jsonschema.Draft7Validator(schema_store[schema], resolver=resolver).validate
 
 
 def validate(data, schema):

lib/python/qmk/keyboard.py

@@ -218,7 +218,7 @@ def render_key_rect(textpad, x, y, w, h, label, style):
 
     label_blank = ' ' * label_len
     label_border = box_chars['h'] * label_len
-    label_middle = label + ' '*label_leftover  # noqa: yapf insists there be no whitespace around *
+    label_middle = label + ' ' * label_leftover
 
     top_line = array('u', box_chars['tl'] + label_border + box_chars['tr'])
     lab_line = array('u', box_chars['v'] + label_middle + box_chars['v'])
@@ -245,10 +245,10 @@ def render_key_isoenter(textpad, x, y, w, h, label, style):
     if len(label) > label_len:
         label = label[:label_len]
 
-    label_blank = ' ' * (label_len-1)  # noqa: yapf insists there be no whitespace around - and *
+    label_blank = ' ' * (label_len - 1)
     label_border_top = box_chars['h'] * label_len
-    label_border_bottom = box_chars['h'] * (label_len-1)  # noqa
-    label_middle = label + ' '*label_leftover  # noqa
+    label_border_bottom = box_chars['h'] * (label_len - 1)
+    label_middle = label + ' ' * label_leftover
 
     top_line = array('u', box_chars['tl'] + label_border_top + box_chars['tr'])
     lab_line = array('u', box_chars['v'] + label_middle + box_chars['v'])
@@ -277,10 +277,10 @@ def render_key_baenter(textpad, x, y, w, h, label, style):
     if len(label) > label_len:
         label = label[:label_len]
 
-    label_blank = ' ' * (label_len-3)  # noqa: yapf insists there be no whitespace around - and *
-    label_border_top = box_chars['h'] * (label_len-3)  # noqa
+    label_blank = ' ' * (label_len - 3)
+    label_border_top = box_chars['h'] * (label_len - 3)
     label_border_bottom = box_chars['h'] * label_len
-    label_middle = label + ' '*label_leftover  # noqa
+    label_middle = label + ' ' * label_leftover
 
     top_line = array('u', box_chars['tl'] + label_border_top + box_chars['tr'])
     mid_line = array('u', box_chars['v'] + label_blank + box_chars['v'])

lib/python/qmk/painter.py

@@ -0,0 +1,268 @@
+"""Functions that help us work with Quantum Painter's file formats.
+"""
+import math
+import re
+from string import Template
+from PIL import Image, ImageOps
+
+# The list of valid formats Quantum Painter supports
+valid_formats = {
+    'pal256': {
+        'image_format': 'IMAGE_FORMAT_PALETTE',
+        'bpp': 8,
+        'has_palette': True,
+        'num_colors': 256,
+        'image_format_byte': 0x07,  # see qp_internal_formats.h
+    },
+    'pal16': {
+        'image_format': 'IMAGE_FORMAT_PALETTE',
+        'bpp': 4,
+        'has_palette': True,
+        'num_colors': 16,
+        'image_format_byte': 0x06,  # see qp_internal_formats.h
+    },
+    'pal4': {
+        'image_format': 'IMAGE_FORMAT_PALETTE',
+        'bpp': 2,
+        'has_palette': True,
+        'num_colors': 4,
+        'image_format_byte': 0x05,  # see qp_internal_formats.h
+    },
+    'pal2': {
+        'image_format': 'IMAGE_FORMAT_PALETTE',
+        'bpp': 1,
+        'has_palette': True,
+        'num_colors': 2,
+        'image_format_byte': 0x04,  # see qp_internal_formats.h
+    },
+    'mono256': {
+        'image_format': 'IMAGE_FORMAT_GRAYSCALE',
+        'bpp': 8,
+        'has_palette': False,
+        'num_colors': 256,
+        'image_format_byte': 0x03,  # see qp_internal_formats.h
+    },
+    'mono16': {
+        'image_format': 'IMAGE_FORMAT_GRAYSCALE',
+        'bpp': 4,
+        'has_palette': False,
+        'num_colors': 16,
+        'image_format_byte': 0x02,  # see qp_internal_formats.h
+    },
+    'mono4': {
+        'image_format': 'IMAGE_FORMAT_GRAYSCALE',
+        'bpp': 2,
+        'has_palette': False,
+        'num_colors': 4,
+        'image_format_byte': 0x01,  # see qp_internal_formats.h
+    },
+    'mono2': {
+        'image_format': 'IMAGE_FORMAT_GRAYSCALE',
+        'bpp': 1,
+        'has_palette': False,
+        'num_colors': 2,
+        'image_format_byte': 0x00,  # see qp_internal_formats.h
+    }
+}
+
+license_template = """\
+// Copyright ${year} QMK -- generated source code only, ${generated_type} retains original copyright
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+// This file was auto-generated by `${generator_command}`
+"""
+
+
+def render_license(subs):
+    license_txt = Template(license_template)
+    return license_txt.substitute(subs)
+
+
+header_file_template = """\
+${license}
+#pragma once
+
+#include <qp.h>
+
+extern const uint32_t ${var_prefix}_${sane_name}_length;
+extern const uint8_t  ${var_prefix}_${sane_name}[${byte_count}];
+"""
+
+
+def render_header(subs):
+    header_txt = Template(header_file_template)
+    return header_txt.substitute(subs)
+
+
+source_file_template = """\
+${license}
+#include <qp.h>
+
+const uint32_t ${var_prefix}_${sane_name}_length = ${byte_count};
+
+// clang-format off
+const uint8_t ${var_prefix}_${sane_name}[${byte_count}] = {
+${bytes_lines}
+};
+// clang-format on
+"""
+
+
+def render_source(subs):
+    source_txt = Template(source_file_template)
+    return source_txt.substitute(subs)
+
+
+def render_bytes(bytes, newline_after=16):
+    lines = ''
+    for n in range(len(bytes)):
+        if n % newline_after == 0 and n > 0 and n != len(bytes):
+            lines = lines + "\n   "
+        elif n == 0:
+            lines = lines + "   "
+        lines = lines + " 0x{0:02X},".format(bytes[n])
+    return lines.rstrip()
+
+
+def clean_output(str):
+    str = re.sub(r'\r', '', str)
+    str = re.sub(r'[\n]{3,}', r'\n\n', str)
+    return str
+
+
+def rescale_byte(val, maxval):
+    """Rescales a byte value to the supplied range, i.e. [0,255] -> [0,maxval].
+    """
+    return int(round(val * maxval / 255.0))
+
+
+def convert_requested_format(im, format):
+    """Convert an image to the requested format.
+    """
+
+    # Work out the requested format
+    ncolors = format["num_colors"]
+    image_format = format["image_format"]
+
+    # Ensure we have a valid number of colors for the palette
+    if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0):
+        raise ValueError("Number of colors must be 2, 4, 16, or 256.")
+
+    # Work out where we're getting the bytes from
+    if image_format == 'IMAGE_FORMAT_GRAYSCALE':
+        # If mono, convert input to grayscale, then to RGB, then grab the raw bytes corresponding to the intensity of the red channel
+        im = ImageOps.grayscale(im)
+        im = im.convert("RGB")
+    elif image_format == 'IMAGE_FORMAT_PALETTE':
+        # If color, convert input to RGB, palettize based on the supplied number of colors, then get the raw palette bytes
+        im = im.convert("RGB")
+        im = im.convert("P", palette=Image.ADAPTIVE, colors=ncolors)
+
+    return im
+
+
+def convert_image_bytes(im, format):
+    """Convert the supplied image to the equivalent bytes required by the QMK firmware.
+    """
+
+    # Work out the requested format
+    ncolors = format["num_colors"]
+    image_format = format["image_format"]
+    shifter = int(math.log2(ncolors))
+    pixels_per_byte = int(8 / math.log2(ncolors))
+    (width, height) = im.size
+    expected_byte_count = ((width * height) + (pixels_per_byte - 1)) // pixels_per_byte
+
+    if image_format == 'IMAGE_FORMAT_GRAYSCALE':
+        # Take the red channel
+        image_bytes = im.tobytes("raw", "R")
+        image_bytes_len = len(image_bytes)
+
+        # No palette
+        palette = None
+
+        bytearray = []
+        for x in range(expected_byte_count):
+            byte = 0
+            for n in range(pixels_per_byte):
+                byte_offset = x * pixels_per_byte + n
+                if byte_offset < image_bytes_len:
+                    # If mono, each input byte is a grayscale [0,255] pixel -- rescale to the range we want then pack together
+                    byte = byte | (rescale_byte(image_bytes[byte_offset], ncolors - 1) << int(n * shifter))
+            bytearray.append(byte)
+
+    elif image_format == 'IMAGE_FORMAT_PALETTE':
+        # Convert each pixel to the palette bytes
+        image_bytes = im.tobytes("raw", "P")
+        image_bytes_len = len(image_bytes)
+
+        # Export the palette
+        palette = []
+        pal = im.getpalette()
+        for n in range(0, ncolors * 3, 3):
+            palette.append((pal[n + 0], pal[n + 1], pal[n + 2]))
+
+        bytearray = []
+        for x in range(expected_byte_count):
+            byte = 0
+            for n in range(pixels_per_byte):
+                byte_offset = x * pixels_per_byte + n
+                if byte_offset < image_bytes_len:
+                    # If color, each input byte is the index into the color palette -- pack them together
+                    byte = byte | ((image_bytes[byte_offset] & (ncolors - 1)) << int(n * shifter))
+            bytearray.append(byte)
+
+    if len(bytearray) != expected_byte_count:
+        raise Exception(f"Wrong byte count, was {len(bytearray)}, expected {expected_byte_count}")
+
+    return (palette, bytearray)
+
+
+def compress_bytes_qmk_rle(bytearray):
+    debug_dump = False
+    output = []
+    temp = []
+    repeat = False
+
+    def append_byte(c):
+        if debug_dump:
+            print('Appending byte:', '0x{0:02X}'.format(int(c)), '=', c)
+        output.append(c)
+
+    def append_range(r):
+        append_byte(127 + len(r))
+        if debug_dump:
+            print('Appending {0} byte(s):'.format(len(r)), '[', ', '.join(['{0:02X}'.format(e) for e in r]), ']')
+        output.extend(r)
+
+    for n in range(0, len(bytearray) + 1):
+        end = True if n == len(bytearray) else False
+        if not end:
+            c = bytearray[n]
+            temp.append(c)
+            if len(temp) <= 1:
+                continue
+
+        if debug_dump:
+            print('Temp buffer state {0:3d} bytes:'.format(len(temp)), '[', ', '.join(['{0:02X}'.format(e) for e in temp]), ']')
+
+        if repeat:
+            if temp[-1] != temp[-2]:
+                repeat = False
+            if not repeat or len(temp) == 128 or end:
+                append_byte(len(temp) if end else len(temp) - 1)
+                append_byte(temp[0])
+                temp = [temp[-1]]
+                repeat = False
+        else:
+            if len(temp) >= 2 and temp[-1] == temp[-2]:
+                repeat = True
+                if len(temp) > 2:
+                    append_range(temp[0:(len(temp) - 2)])
+                    temp = [temp[-1], temp[-1]]
+                continue
+            if len(temp) == 128 or end:
+                append_range(temp)
+                temp = []
+                repeat = False
+    return output

lib/python/qmk/painter_qff.py

@@ -0,0 +1,401 @@
+# Copyright 2021 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Quantum Font File "QFF" Font File Format.
+# See https://docs.qmk.fm/#/quantum_painter_qff for more information.
+
+from pathlib import Path
+from typing import Dict, Any
+from colorsys import rgb_to_hsv
+from PIL import Image, ImageDraw, ImageFont, ImageChops
+from PIL._binary import o8, o16le as o16, o32le as o32
+from qmk.painter_qgf import QGFBlockHeader, QGFFramePaletteDescriptorV1
+from milc.attrdict import AttrDict
+import qmk.painter
+
+
+def o24(i):
+    return o16(i & 0xFFFF) + o8((i & 0xFF0000) >> 16)
+
+
+########################################################################################################################
+
+
+class QFFGlyphInfo(AttrDict):
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+
+        for n, value in enumerate(args):
+            self[f'arg:{n}'] = value
+
+        for key, value in kwargs.items():
+            self[key] = value
+
+    def write(self, fp, include_code_point):
+        if include_code_point is True:
+            fp.write(o24(ord(self.code_point)))
+
+        value = ((self.data_offset << 6) & 0xFFFFC0) | (self.w & 0x3F)
+        fp.write(o24(value))
+
+
+########################################################################################################################
+
+
+class QFFFontDescriptor:
+    type_id = 0x00
+    length = 20
+    magic = 0x464651
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QFFFontDescriptor.type_id
+        self.header.length = QFFFontDescriptor.length
+        self.version = 1
+        self.total_file_size = 0
+        self.line_height = 0
+        self.has_ascii_table = False
+        self.unicode_glyph_count = 0
+        self.format = 0xFF
+        self.flags = 0
+        self.compression = 0xFF
+        self.transparency_index = 0xFF  # TODO: Work out how to retrieve the transparent palette entry from the PIL gif loader
+
+    def write(self, fp):
+        self.header.write(fp)
+        fp.write(
+            b''  # start off with empty bytes...
+            + o24(QFFFontDescriptor.magic)  # magic
+            + o8(self.version)  # version
+            + o32(self.total_file_size)  # file size
+            + o32((~self.total_file_size) & 0xFFFFFFFF)  # negated file size
+            + o8(self.line_height)  # line height
+            + o8(1 if self.has_ascii_table is True else 0)  # whether or not we have an ascii table present
+            + o16(self.unicode_glyph_count & 0xFFFF)  # number of unicode glyphs present
+            + o8(self.format)  # format
+            + o8(self.flags)  # flags
+            + o8(self.compression)  # compression
+            + o8(self.transparency_index)  # transparency index
+        )
+
+    @property
+    def is_transparent(self):
+        return (self.flags & 0x01) == 0x01
+
+    @is_transparent.setter
+    def is_transparent(self, val):
+        if val:
+            self.flags |= 0x01
+        else:
+            self.flags &= ~0x01
+
+
+########################################################################################################################
+
+
+class QFFAsciiGlyphTableV1:
+    type_id = 0x01
+    length = 95 * 3  # We have 95 glyphs: [0x20...0x7E]
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QFFAsciiGlyphTableV1.type_id
+        self.header.length = QFFAsciiGlyphTableV1.length
+
+        # Each glyph is key=code_point, value=QFFGlyphInfo
+        self.glyphs = {}
+
+    def add_glyph(self, glyph: QFFGlyphInfo):
+        self.glyphs[ord(glyph.code_point)] = glyph
+
+    def write(self, fp):
+        self.header.write(fp)
+
+        for n in range(0x20, 0x7F):
+            self.glyphs[n].write(fp, False)
+
+
+########################################################################################################################
+
+
+class QFFUnicodeGlyphTableV1:
+    type_id = 0x02
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QFFUnicodeGlyphTableV1.type_id
+        self.header.length = 0
+
+        # Each glyph is key=code_point, value=QFFGlyphInfo
+        self.glyphs = {}
+
+    def add_glyph(self, glyph: QFFGlyphInfo):
+        self.glyphs[ord(glyph.code_point)] = glyph
+
+    def write(self, fp):
+        self.header.length = len(self.glyphs.keys()) * 6
+        self.header.write(fp)
+
+        for n in sorted(self.glyphs.keys()):
+            self.glyphs[n].write(fp, True)
+
+
+########################################################################################################################
+
+
+class QFFFontDataDescriptorV1:
+    type_id = 0x04
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QFFFontDataDescriptorV1.type_id
+        self.data = []
+
+    def write(self, fp):
+        self.header.length = len(self.data)
+        self.header.write(fp)
+        fp.write(bytes(self.data))
+
+
+########################################################################################################################
+
+
+def _generate_font_glyphs_list(use_ascii, unicode_glyphs):
+    # The set of glyphs that we want to generate images for
+    glyphs = {}
+
+    # Add ascii charset if requested
+    if use_ascii is True:
+        for c in range(0x20, 0x7F):  # does not include 0x7F!
+            glyphs[chr(c)] = True
+
+    # Append any extra unicode glyphs
+    unicode_glyphs = list(unicode_glyphs)
+    for c in unicode_glyphs:
+        glyphs[c] = True
+
+    return sorted(glyphs.keys())
+
+
+class QFFFont:
+    def __init__(self, logger):
+        self.logger = logger
+        self.image = None
+        self.glyph_data = {}
+        self.glyph_height = 0
+        return
+
+    def _extract_glyphs(self, format):
+        total_data_size = 0
+        total_rle_data_size = 0
+
+        converted_img = qmk.painter.convert_requested_format(self.image, format)
+        (self.palette, _) = qmk.painter.convert_image_bytes(converted_img, format)
+
+        # Work out how many bytes used for RLE vs. non-RLE
+        for _, glyph_entry in self.glyph_data.items():
+            glyph_img = converted_img.crop((glyph_entry.x, 1, glyph_entry.x + glyph_entry.w, 1 + self.glyph_height))
+            (_, this_glyph_image_bytes) = qmk.painter.convert_image_bytes(glyph_img, format)
+            this_glyph_rle_bytes = qmk.painter.compress_bytes_qmk_rle(this_glyph_image_bytes)
+            total_data_size += len(this_glyph_image_bytes)
+            total_rle_data_size += len(this_glyph_rle_bytes)
+            glyph_entry['image_uncompressed_bytes'] = this_glyph_image_bytes
+            glyph_entry['image_compressed_bytes'] = this_glyph_rle_bytes
+
+        return (total_data_size, total_rle_data_size)
+
+    def _parse_image(self, img, include_ascii_glyphs: bool = True, unicode_glyphs: str = ''):
+        # Clear out any existing font metadata
+        self.image = None
+        # Each glyph is key=code_point, value={ x: ?, w: ? }
+        self.glyph_data = {}
+        self.glyph_height = 0
+
+        # Work out the list of glyphs required
+        glyphs = _generate_font_glyphs_list(include_ascii_glyphs, unicode_glyphs)
+
+        # Work out the geometry
+        (width, height) = img.size
+
+        # Work out the glyph offsets/widths
+        glyph_pixel_offsets = []
+        glyph_pixel_widths = []
+        pixels = img.load()
+
+        # Run through the markers and work out where each glyph starts/stops
+        glyph_split_color = pixels[0, 0]  # top left pixel is the marker color we're going to use to split each glyph
+        glyph_pixel_offsets.append(0)
+        last_offset = 0
+        for x in range(1, width):
+            if pixels[x, 0] == glyph_split_color:
+                glyph_pixel_offsets.append(x)
+                glyph_pixel_widths.append(x - last_offset)
+                last_offset = x
+        glyph_pixel_widths.append(width - last_offset)
+
+        # Make sure the number of glyphs we're attempting to generate matches the input image
+        if len(glyph_pixel_offsets) != len(glyphs):
+            self.logger.error('The number of glyphs to generate doesn\'t match the number of detected glyphs in the input image.')
+            return
+
+        # Set up the required metadata for each glyph
+        for n in range(0, len(glyph_pixel_offsets)):
+            self.glyph_data[glyphs[n]] = QFFGlyphInfo(code_point=glyphs[n], x=glyph_pixel_offsets[n], w=glyph_pixel_widths[n])
+
+        # Parsing was successful, keep the image in this instance
+        self.image = img
+        self.glyph_height = height - 1  # subtract the line with the markers
+
+    def generate_image(self, ttf_file: Path, font_size: int, include_ascii_glyphs: bool = True, unicode_glyphs: str = '', include_before_left: bool = False, use_aa: bool = True):
+        # Load the font
+        font = ImageFont.truetype(str(ttf_file), int(font_size))
+        # Work out the max font size
+        max_font_size = font.font.ascent + abs(font.font.descent)
+        # Work out the list of glyphs required
+        glyphs = _generate_font_glyphs_list(include_ascii_glyphs, unicode_glyphs)
+
+        baseline_offset = 9999999
+        total_glyph_width = 0
+        max_glyph_height = -1
+
+        # Measure each glyph to determine the overall baseline offset required
+        for glyph in glyphs:
+            (ls_l, ls_t, ls_r, ls_b) = font.getbbox(glyph, anchor='ls')
+            glyph_width = (ls_r - ls_l) if include_before_left else (ls_r)
+            glyph_height = font.getbbox(glyph, anchor='la')[3]
+            if max_glyph_height < glyph_height:
+                max_glyph_height = glyph_height
+            total_glyph_width += glyph_width
+            if baseline_offset > ls_t:
+                baseline_offset = ls_t
+
+        # Create the output image
+        img = Image.new("RGB", (total_glyph_width + 1, max_font_size * 2 + 1), (0, 0, 0, 255))
+        cur_x_pos = 0
+
+        # Loop through each glyph...
+        for glyph in glyphs:
+            # Work out this glyph's bounding box
+            (ls_l, ls_t, ls_r, ls_b) = font.getbbox(glyph, anchor='ls')
+            glyph_width = (ls_r - ls_l) if include_before_left else (ls_r)
+            glyph_height = ls_b - ls_t
+            x_offset = -ls_l
+            y_offset = ls_t - baseline_offset
+
+            # Draw each glyph to its own image so we don't get anti-aliasing applied to the final image when straddling edges
+            glyph_img = Image.new("RGB", (glyph_width, max_font_size), (0, 0, 0, 255))
+            glyph_draw = ImageDraw.Draw(glyph_img)
+            if not use_aa:
+                glyph_draw.fontmode = "1"
+            glyph_draw.text((x_offset, y_offset), glyph, font=font, anchor='lt')
+
+            # Place the glyph-specific image in the correct location overall
+            img.paste(glyph_img, (cur_x_pos, 1))
+
+            # Set up the marker for start of each glyph
+            pixels = img.load()
+            pixels[cur_x_pos, 0] = (255, 0, 255)
+
+            # Increment for the next glyph's position
+            cur_x_pos += glyph_width
+
+        # Add the ending marker so that the difference/crop works
+        pixels = img.load()
+        pixels[cur_x_pos, 0] = (255, 0, 255)
+
+        # Determine the usable font area
+        dummy_img = Image.new("RGB", (total_glyph_width + 1, max_font_size + 1), (0, 0, 0, 255))
+        bbox = ImageChops.difference(img, dummy_img).getbbox()
+        bbox = (bbox[0], bbox[1], bbox[2] - 1, bbox[3])  # remove the unused end-marker
+
+        # Crop and re-parse the resulting image to ensure we're generating the correct format
+        self._parse_image(img.crop(bbox), include_ascii_glyphs, unicode_glyphs)
+
+    def save_to_image(self, img_file: Path):
+        # Drop out if there's no image loaded
+        if self.image is None:
+            self.logger.error('No image is loaded.')
+            return
+
+        # Save the image to the supplied file
+        self.image.save(str(img_file))
+
+    def read_from_image(self, img_file: Path, include_ascii_glyphs: bool = True, unicode_glyphs: str = ''):
+        # Load and parse the supplied image file
+        self._parse_image(Image.open(str(img_file)), include_ascii_glyphs, unicode_glyphs)
+        return
+
+    def save_to_qff(self, format: Dict[str, Any], use_rle: bool, fp):
+        # Drop out if there's no image loaded
+        if self.image is None:
+            self.logger.error('No image is loaded.')
+            return
+
+        # Work out if we want to use RLE at all, skipping it if it's not any smaller (it's applied per-glyph)
+        (total_data_size, total_rle_data_size) = self._extract_glyphs(format)
+        if use_rle:
+            use_rle = (total_rle_data_size < total_data_size)
+
+        # For each glyph, work out which image data we want to use and append it to the image buffer, recording the byte-wise offset
+        img_buffer = bytes()
+        for _, glyph_entry in self.glyph_data.items():
+            glyph_entry['data_offset'] = len(img_buffer)
+            glyph_img_bytes = glyph_entry.image_compressed_bytes if use_rle else glyph_entry.image_uncompressed_bytes
+            img_buffer += bytes(glyph_img_bytes)
+
+        font_descriptor = QFFFontDescriptor()
+        ascii_table = QFFAsciiGlyphTableV1()
+        unicode_table = QFFUnicodeGlyphTableV1()
+        data_descriptor = QFFFontDataDescriptorV1()
+        data_descriptor.data = img_buffer
+
+        # Check if we have all the ASCII glyphs present
+        include_ascii_glyphs = all([chr(n) in self.glyph_data for n in range(0x20, 0x7F)])
+
+        # Helper for populating the blocks
+        for code_point, glyph_entry in self.glyph_data.items():
+            if ord(code_point) >= 0x20 and ord(code_point) <= 0x7E and include_ascii_glyphs:
+                ascii_table.add_glyph(glyph_entry)
+            else:
+                unicode_table.add_glyph(glyph_entry)
+
+        # Configure the font descriptor
+        font_descriptor.line_height = self.glyph_height
+        font_descriptor.has_ascii_table = include_ascii_glyphs
+        font_descriptor.unicode_glyph_count = len(unicode_table.glyphs.keys())
+        font_descriptor.is_transparent = False
+        font_descriptor.format = format['image_format_byte']
+        font_descriptor.compression = 0x01 if use_rle else 0x00
+
+        # Write a dummy font descriptor -- we'll have to come back and write it properly once we've rendered out everything else
+        font_descriptor_location = fp.tell()
+        font_descriptor.write(fp)
+
+        # Write out the ASCII table if required
+        if font_descriptor.has_ascii_table:
+            ascii_table.write(fp)
+
+        # Write out the unicode table if required
+        if font_descriptor.unicode_glyph_count > 0:
+            unicode_table.write(fp)
+
+        # Write out the palette if required
+        if format['has_palette']:
+            palette_descriptor = QGFFramePaletteDescriptorV1()
+
+            # Helper to convert from RGB888 to the QMK "dialect" of HSV888
+            def rgb888_to_qmk_hsv888(e):
+                hsv = rgb_to_hsv(e[0] / 255.0, e[1] / 255.0, e[2] / 255.0)
+                return (int(hsv[0] * 255.0), int(hsv[1] * 255.0), int(hsv[2] * 255.0))
+
+            # Convert all palette entries to HSV888 and write to the output
+            palette_descriptor.palette_entries = list(map(rgb888_to_qmk_hsv888, self.palette))
+            palette_descriptor.write(fp)
+
+        # Write out the image data
+        data_descriptor.write(fp)
+
+        # Now fix up the overall font descriptor, then write it in the correct location
+        font_descriptor.total_file_size = fp.tell()
+        fp.seek(font_descriptor_location, 0)
+        font_descriptor.write(fp)

lib/python/qmk/painter_qgf.py

@@ -0,0 +1,408 @@
+# Copyright 2021 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Quantum Graphics File "QGF" Image File Format.
+# See https://docs.qmk.fm/#/quantum_painter_qgf for more information.
+
+from colorsys import rgb_to_hsv
+from types import FunctionType
+from PIL import Image, ImageFile, ImageChops
+from PIL._binary import o8, o16le as o16, o32le as o32
+import qmk.painter
+
+
+def o24(i):
+    return o16(i & 0xFFFF) + o8((i & 0xFF0000) >> 16)
+
+
+########################################################################################################################
+
+
+class QGFBlockHeader:
+    block_size = 5
+
+    def write(self, fp):
+        fp.write(b''  # start off with empty bytes...
+                 + o8(self.type_id)  # block type id
+                 + o8((~self.type_id) & 0xFF)  # negated block type id
+                 + o24(self.length)  # blob length
+                 )
+
+
+########################################################################################################################
+
+
+class QGFGraphicsDescriptor:
+    type_id = 0x00
+    length = 18
+    magic = 0x464751
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QGFGraphicsDescriptor.type_id
+        self.header.length = QGFGraphicsDescriptor.length
+        self.version = 1
+        self.total_file_size = 0
+        self.image_width = 0
+        self.image_height = 0
+        self.frame_count = 0
+
+    def write(self, fp):
+        self.header.write(fp)
+        fp.write(
+            b''  # start off with empty bytes...
+            + o24(QGFGraphicsDescriptor.magic)  # magic
+            + o8(self.version)  # version
+            + o32(self.total_file_size)  # file size
+            + o32((~self.total_file_size) & 0xFFFFFFFF)  # negated file size
+            + o16(self.image_width)  # width
+            + o16(self.image_height)  # height
+            + o16(self.frame_count)  # frame count
+        )
+
+
+########################################################################################################################
+
+
+class QGFFrameOffsetDescriptorV1:
+    type_id = 0x01
+
+    def __init__(self, frame_count):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QGFFrameOffsetDescriptorV1.type_id
+        self.frame_offsets = [0xFFFFFFFF] * frame_count
+        self.frame_count = frame_count
+
+    def write(self, fp):
+        self.header.length = len(self.frame_offsets) * 4
+        self.header.write(fp)
+        for offset in self.frame_offsets:
+            fp.write(b''  # start off with empty bytes...
+                     + o32(offset)  # offset
+                     )
+
+
+########################################################################################################################
+
+
+class QGFFrameDescriptorV1:
+    type_id = 0x02
+    length = 6
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QGFFrameDescriptorV1.type_id
+        self.header.length = QGFFrameDescriptorV1.length
+        self.format = 0xFF
+        self.flags = 0
+        self.compression = 0xFF
+        self.transparency_index = 0xFF  # TODO: Work out how to retrieve the transparent palette entry from the PIL gif loader
+        self.delay = 1000  # Placeholder until it gets read from the animation
+
+    def write(self, fp):
+        self.header.write(fp)
+        fp.write(b''  # start off with empty bytes...
+                 + o8(self.format)  # format
+                 + o8(self.flags)  # flags
+                 + o8(self.compression)  # compression
+                 + o8(self.transparency_index)  # transparency index
+                 + o16(self.delay)  # delay
+                 )
+
+    @property
+    def is_transparent(self):
+        return (self.flags & 0x01) == 0x01
+
+    @is_transparent.setter
+    def is_transparent(self, val):
+        if val:
+            self.flags |= 0x01
+        else:
+            self.flags &= ~0x01
+
+    @property
+    def is_delta(self):
+        return (self.flags & 0x02) == 0x02
+
+    @is_delta.setter
+    def is_delta(self, val):
+        if val:
+            self.flags |= 0x02
+        else:
+            self.flags &= ~0x02
+
+
+########################################################################################################################
+
+
+class QGFFramePaletteDescriptorV1:
+    type_id = 0x03
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QGFFramePaletteDescriptorV1.type_id
+        self.header.length = 0
+        self.palette_entries = [(0xFF, 0xFF, 0xFF)] * 4
+
+    def write(self, fp):
+        self.header.length = len(self.palette_entries) * 3
+        self.header.write(fp)
+        for entry in self.palette_entries:
+            fp.write(b''  # start off with empty bytes...
+                     + o8(entry[0])  # h
+                     + o8(entry[1])  # s
+                     + o8(entry[2])  # v
+                     )
+
+
+########################################################################################################################
+
+
+class QGFFrameDeltaDescriptorV1:
+    type_id = 0x04
+    length = 8
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QGFFrameDeltaDescriptorV1.type_id
+        self.header.length = QGFFrameDeltaDescriptorV1.length
+        self.left = 0
+        self.top = 0
+        self.right = 0
+        self.bottom = 0
+
+    def write(self, fp):
+        self.header.write(fp)
+        fp.write(b''  # start off with empty bytes...
+                 + o16(self.left)  # left
+                 + o16(self.top)  # top
+                 + o16(self.right)  # right
+                 + o16(self.bottom)  # bottom
+                 )
+
+
+########################################################################################################################
+
+
+class QGFFrameDataDescriptorV1:
+    type_id = 0x05
+
+    def __init__(self):
+        self.header = QGFBlockHeader()
+        self.header.type_id = QGFFrameDataDescriptorV1.type_id
+        self.data = []
+
+    def write(self, fp):
+        self.header.length = len(self.data)
+        self.header.write(fp)
+        fp.write(bytes(self.data))
+
+
+########################################################################################################################
+
+
+class QGFImageFile(ImageFile.ImageFile):
+
+    format = "QGF"
+    format_description = "Quantum Graphics File Format"
+
+    def _open(self):
+        raise NotImplementedError("Reading QGF files is not supported")
+
+
+########################################################################################################################
+
+
+def _accept(prefix):
+    """Helper method used by PIL to work out if it can parse an input file.
+
+    Currently unimplemented.
+    """
+    return False
+
+
+def _save(im, fp, filename):
+    """Helper method used by PIL to write to an output file.
+    """
+    # Work out from the parameters if we need to do anything special
+    encoderinfo = im.encoderinfo.copy()
+    append_images = list(encoderinfo.get("append_images", []))
+    verbose = encoderinfo.get("verbose", False)
+    use_deltas = encoderinfo.get("use_deltas", True)
+    use_rle = encoderinfo.get("use_rle", True)
+
+    # Helper for inline verbose prints
+    def vprint(s):
+        if verbose:
+            print(s)
+
+    # Helper to iterate through all frames in the input image
+    def _for_all_frames(x: FunctionType):
+        frame_num = 0
+        last_frame = None
+        for frame in [im] + append_images:
+            # Get number of of frames in this image
+            nfr = getattr(frame, "n_frames", 1)
+            for idx in range(nfr):
+                frame.seek(idx)
+                frame.load()
+                copy = frame.copy().convert("RGB")
+                x(frame_num, copy, last_frame)
+                last_frame = copy
+                frame_num += 1
+
+    # Collect all the frame sizes
+    frame_sizes = []
+    _for_all_frames(lambda idx, frame, last_frame: frame_sizes.append(frame.size))
+
+    # Make sure all frames are the same size
+    if len(list(set(frame_sizes))) != 1:
+        raise ValueError("Mismatching sizes on frames")
+
+    # Write out the initial graphics descriptor (and write a dummy value), so that we can come back and fill in the
+    # correct values once we've written all the frames to the output
+    graphics_descriptor_location = fp.tell()
+    graphics_descriptor = QGFGraphicsDescriptor()
+    graphics_descriptor.frame_count = len(frame_sizes)
+    graphics_descriptor.image_width = frame_sizes[0][0]
+    graphics_descriptor.image_height = frame_sizes[0][1]
+    vprint(f'{"Graphics descriptor block":26s} {fp.tell():5d}d / {fp.tell():04X}h')
+    graphics_descriptor.write(fp)
+
+    # Work out the frame offset descriptor location (and write a dummy value), so that we can come back and fill in the
+    # correct offsets once we've written all the frames to the output
+    frame_offset_location = fp.tell()
+    frame_offsets = QGFFrameOffsetDescriptorV1(graphics_descriptor.frame_count)
+    vprint(f'{"Frame offsets block":26s} {fp.tell():5d}d / {fp.tell():04X}h')
+    frame_offsets.write(fp)
+
+    # Helper function to save each frame to the output file
+    def _write_frame(idx, frame, last_frame):
+        # If we replace the frame we're going to output with a delta, we can override it here
+        this_frame = frame
+        location = (0, 0)
+        size = frame.size
+
+        # Work out the format we're going to use
+        format = encoderinfo["qmk_format"]
+
+        # Convert the original frame so we can do comparisons
+        converted = qmk.painter.convert_requested_format(this_frame, format)
+        graphic_data = qmk.painter.convert_image_bytes(converted, format)
+
+        # Convert the raw data to RLE-encoded if requested
+        raw_data = graphic_data[1]
+        if use_rle:
+            rle_data = qmk.painter.compress_bytes_qmk_rle(graphic_data[1])
+        use_raw_this_frame = not use_rle or len(raw_data) <= len(rle_data)
+        image_data = raw_data if use_raw_this_frame else rle_data
+
+        # Work out if a delta frame is smaller than injecting it directly
+        use_delta_this_frame = False
+        if use_deltas and last_frame is not None:
+            # If we want to use deltas, then find the difference
+            diff = ImageChops.difference(frame, last_frame)
+
+            # Get the bounding box of those differences
+            bbox = diff.getbbox()
+
+            # If we have a valid bounding box...
+            if bbox:
+                # ...create the delta frame by cropping the original.
+                delta_frame = frame.crop(bbox)
+                delta_location = (bbox[0], bbox[1])
+                delta_size = (bbox[2] - bbox[0], bbox[3] - bbox[1])
+
+                # Convert the delta frame to the requested format
+                delta_converted = qmk.painter.convert_requested_format(delta_frame, format)
+                delta_graphic_data = qmk.painter.convert_image_bytes(delta_converted, format)
+
+                # Work out how large the delta frame is going to be with compression etc.
+                delta_raw_data = delta_graphic_data[1]
+                if use_rle:
+                    delta_rle_data = qmk.painter.compress_bytes_qmk_rle(delta_graphic_data[1])
+                delta_use_raw_this_frame = not use_rle or len(delta_raw_data) <= len(delta_rle_data)
+                delta_image_data = delta_raw_data if delta_use_raw_this_frame else delta_rle_data
+
+                # If the size of the delta frame (plus delta descriptor) is smaller than the original, use that instead
+                # This ensures that if a non-delta is overall smaller in size, we use that in preference due to flash
+                # sizing constraints.
+                if (len(delta_image_data) + QGFFrameDeltaDescriptorV1.length) < len(image_data):
+                    # Copy across all the delta equivalents so that the rest of the processing acts on those
+                    this_frame = delta_frame
+                    location = delta_location
+                    size = delta_size
+                    converted = delta_converted
+                    graphic_data = delta_graphic_data
+                    raw_data = delta_raw_data
+                    rle_data = delta_rle_data
+                    use_raw_this_frame = delta_use_raw_this_frame
+                    image_data = delta_image_data
+                    use_delta_this_frame = True
+
+        # Write out the frame descriptor
+        frame_offsets.frame_offsets[idx] = fp.tell()
+        vprint(f'{f"Frame {idx:3d} base":26s} {fp.tell():5d}d / {fp.tell():04X}h')
+        frame_descriptor = QGFFrameDescriptorV1()
+        frame_descriptor.is_delta = use_delta_this_frame
+        frame_descriptor.is_transparent = False
+        frame_descriptor.format = format['image_format_byte']
+        frame_descriptor.compression = 0x00 if use_raw_this_frame else 0x01  # See qp.h, painter_compression_t
+        frame_descriptor.delay = frame.info['duration'] if 'duration' in frame.info else 1000  # If we're not an animation, just pretend we're delaying for 1000ms
+        frame_descriptor.write(fp)
+
+        # Write out the palette if required
+        if format['has_palette']:
+            palette = graphic_data[0]
+            palette_descriptor = QGFFramePaletteDescriptorV1()
+
+            # Helper to convert from RGB888 to the QMK "dialect" of HSV888
+            def rgb888_to_qmk_hsv888(e):
+                hsv = rgb_to_hsv(e[0] / 255.0, e[1] / 255.0, e[2] / 255.0)
+                return (int(hsv[0] * 255.0), int(hsv[1] * 255.0), int(hsv[2] * 255.0))
+
+            # Convert all palette entries to HSV888 and write to the output
+            palette_descriptor.palette_entries = list(map(rgb888_to_qmk_hsv888, palette))
+            vprint(f'{f"Frame {idx:3d} palette":26s} {fp.tell():5d}d / {fp.tell():04X}h')
+            palette_descriptor.write(fp)
+
+        # Write out the delta info if required
+        if use_delta_this_frame:
+            # Set up the rendering location of where the delta frame should be situated
+            delta_descriptor = QGFFrameDeltaDescriptorV1()
+            delta_descriptor.left = location[0]
+            delta_descriptor.top = location[1]
+            delta_descriptor.right = location[0] + size[0]
+            delta_descriptor.bottom = location[1] + size[1]
+
+            # Write the delta frame to the output
+            vprint(f'{f"Frame {idx:3d} delta":26s} {fp.tell():5d}d / {fp.tell():04X}h')
+            delta_descriptor.write(fp)
+
+        # Write out the data for this frame to the output
+        data_descriptor = QGFFrameDataDescriptorV1()
+        data_descriptor.data = image_data
+        vprint(f'{f"Frame {idx:3d} data":26s} {fp.tell():5d}d / {fp.tell():04X}h')
+        data_descriptor.write(fp)
+
+    # Iterate over each if the input frames, writing it to the output in the process
+    _for_all_frames(_write_frame)
+
+    # Go back and update the graphics descriptor now that we can determine the final file size
+    graphics_descriptor.total_file_size = fp.tell()
+    fp.seek(graphics_descriptor_location, 0)
+    graphics_descriptor.write(fp)
+
+    # Go back and update the frame offsets now that they're written to the file
+    fp.seek(frame_offset_location, 0)
+    frame_offsets.write(fp)
+
+
+########################################################################################################################
+
+# Register with PIL so that it knows about the QGF format
+Image.register_open(QGFImageFile.format, QGFImageFile, _accept)
+Image.register_save(QGFImageFile.format, _save)
+Image.register_save_all(QGFImageFile.format, _save)
+Image.register_extension(QGFImageFile.format, f".{QGFImageFile.format.lower()}")
+Image.register_mime(QGFImageFile.format, f"image/{QGFImageFile.format.lower()}")

lib/python/qmk/tests/test_cli_commands.py

@@ -248,7 +248,7 @@ def test_clean():
 
 
 def test_generate_api():
-    result = check_subcommand('generate-api', '--dry-run')
+    result = check_subcommand('generate-api', '--dry-run', '--filter', 'handwired/pytest')
     check_returncode(result)
 
 
@@ -263,7 +263,7 @@ def test_generate_config_h():
     result = check_subcommand('generate-config-h', '-kb', 'handwired/pytest/basic')
     check_returncode(result)
     assert '#   define DEVICE_VER 0x0001' in result.stdout
-    assert '#   define DESCRIPTION handwired/pytest/basic' in result.stdout
+    assert '#   define DESCRIPTION "handwired/pytest/basic"' in result.stdout
     assert '#   define DIODE_DIRECTION COL2ROW' in result.stdout
     assert '#   define MANUFACTURER none' in result.stdout
     assert '#   define PRODUCT pytest' in result.stdout