Merge commit '6d0a62920410f50d7f6707960ca1ca0c8fd1d1fa' into firmware21

2021-12-07 09:27:44 -08:00
parent 43002bdf77 6d0a629204
commit 7c18b1c9d3
593 changed files with 14869 additions and 17518 deletions
--- a/lib/python/qmk/cli/init.py
+++ b/lib/python/qmk/cli/init.py
@@ -31,11 +31,11 @@ safe_commands = [
 subcommands = [
    'qmk.cli.bux',
    'qmk.cli.c2json',
+    'qmk.cli.cd',
    'qmk.cli.cformat',
    'qmk.cli.chibios.confmigrate',
    'qmk.cli.clean',
    'qmk.cli.compile',
-    'qmk.cli.console',
    'qmk.cli.docs',
    'qmk.cli.doctor',
    'qmk.cli.fileformat',
@@ -45,7 +45,9 @@ subcommands = [
    'qmk.cli.format.python',
    'qmk.cli.format.text',
    'qmk.cli.generate.api',
+    'qmk.cli.generate.compilation_database',
    'qmk.cli.generate.config_h',
+    'qmk.cli.generate.develop_pr_list',
    'qmk.cli.generate.dfu_header',
    'qmk.cli.generate.docs',
    'qmk.cli.generate.info_json',
--- a/lib/python/qmk/cli/cd.py
+++ b/lib/python/qmk/cli/cd.py
@@ -0,0 +1,46 @@
+"""Open a shell in the QMK Home directory
+"""
+import sys
+import os
+
+from milc import cli
+
+from qmk.path import under_qmk_firmware
+
+
+@cli.subcommand('Go to QMK Home')
+def cd(cli):
+    """Go to QMK Home
+    """
+    if not sys.stdout.isatty():
+        cli.log.error("This command is for interactive usage only. For non-interactive usage, 'cd $(qmk env QMK_HOME)' is more robust.")
+        sys.exit(1)
+
+    if not under_qmk_firmware():
+        # Only do anything if the user is not under qmk_firmware already
+        # in order to reduce the possibility of starting multiple shells
+        cli.log.info("Spawning a subshell in your QMK_HOME directory.")
+        cli.log.info("Type 'exit' to get back to the parent shell.")
+        if not cli.platform.lower().startswith('windows'):
+            # For Linux/Mac/etc
+            # Check the user's login shell from 'passwd'
+            # alternatively fall back to $SHELL env var
+            # and finally to '/bin/bash'.
+            import getpass
+            import pwd
+            shell = pwd.getpwnam(getpass.getuser()).pw_shell
+            if not shell:
+                shell = os.environ.get('SHELL', '/bin/bash')
+            # Start the new subshell
+            os.execl(shell, shell)
+        else:
+            # For Windows
+            # Check the $SHELL env var
+            # and fall back to '/usr/bin/bash'.
+            qmk_env = os.environ.copy()
+            # Set the prompt for the new shell
+            qmk_env['MSYS2_PS1'] = qmk_env['PS1']
+            # Start the new subshell
+            cli.run([os.environ.get('SHELL', '/usr/bin/bash')], env=qmk_env)
+    else:
+        cli.log.info("Already within qmk_firmware directory.")
--- a/lib/python/qmk/cli/console.py
+++ b/lib/python/qmk/cli/console.py
@@ -1,303 +0,0 @@
-"""Acquire debugging information from usb hid devices
-
-cli implementation of https://www.pjrc.com/teensy/hid_listen.html
-"""
-from pathlib import Path
-from threading import Thread
-from time import sleep, strftime
-
-import hid
-import usb.core
-
-from milc import cli
-
-LOG_COLOR = {
-    'next': 0,
-    'colors': [
-        '{fg_blue}',
-        '{fg_cyan}',
-        '{fg_green}',
-        '{fg_magenta}',
-        '{fg_red}',
-        '{fg_yellow}',
-    ],
-}
-
-KNOWN_BOOTLOADERS = {
-    # VID  ,  PID
-    ('03EB', '2FEF'): 'atmel-dfu: ATmega16U2',
-    ('03EB', '2FF0'): 'atmel-dfu: ATmega32U2',
-    ('03EB', '2FF3'): 'atmel-dfu: ATmega16U4',
-    ('03EB', '2FF4'): 'atmel-dfu: ATmega32U4',
-    ('03EB', '2FF9'): 'atmel-dfu: AT90USB64',
-    ('03EB', '2FFA'): 'atmel-dfu: AT90USB162',
-    ('03EB', '2FFB'): 'atmel-dfu: AT90USB128',
-    ('03EB', '6124'): 'Microchip SAM-BA',
-    ('0483', 'DF11'): 'stm32-dfu: STM32 BOOTLOADER',
-    ('16C0', '05DC'): 'usbasploader: USBaspLoader',
-    ('16C0', '05DF'): 'bootloadhid: HIDBoot',
-    ('16C0', '0478'): 'halfkay: Teensy Halfkay',
-    ('1B4F', '9203'): 'caterina: Pro Micro 3.3V',
-    ('1B4F', '9205'): 'caterina: Pro Micro 5V',
-    ('1B4F', '9207'): 'caterina: LilyPadUSB',
-    ('1C11', 'B007'): 'kiibohd: Kiibohd DFU Bootloader',
-    ('1EAF', '0003'): 'stm32duino: Maple 003',
-    ('1FFB', '0101'): 'caterina: Polou A-Star 32U4 Bootloader',
-    ('2341', '0036'): 'caterina: Arduino Leonardo',
-    ('2341', '0037'): 'caterina: Arduino Micro',
-    ('239A', '000C'): 'caterina: Adafruit Feather 32U4',
-    ('239A', '000D'): 'caterina: Adafruit ItsyBitsy 32U4 3v',
-    ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v',
-    ('2A03', '0036'): 'caterina: Arduino Leonardo',
-    ('2A03', '0037'): 'caterina: Arduino Micro',
-    ('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode',
-    ('03EB', '2067'): 'qmk-hid: HID Bootloader',
-    ('03EB', '2045'): 'lufa-ms: LUFA Mass Storage Bootloader'
-}
-
-
-class MonitorDevice(object):
-    def __init__(self, hid_device, numeric):
-        self.hid_device = hid_device
-        self.numeric = numeric
-        self.device = hid.Device(path=hid_device['path'])
-        self.current_line = ''
-
-        cli.log.info('Console Connected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', hid_device)
-
-    def read(self, size, encoding='ascii', timeout=1):
-        """Read size bytes from the device.
-        """
-        return self.device.read(size, timeout).decode(encoding)
-
-    def read_line(self):
-        """Read from the device's console until we get a \n.
-        """
-        while '\n' not in self.current_line:
-            self.current_line += self.read(32).replace('\x00', '')
-
-        lines = self.current_line.split('\n', 1)
-        self.current_line = lines[1]
-
-        return lines[0]
-
-    def run_forever(self):
-        while True:
-            try:
-                message = {**self.hid_device, 'text': self.read_line()}
-                identifier = (int2hex(message['vendor_id']), int2hex(message['product_id'])) if self.numeric else (message['manufacturer_string'], message['product_string'])
-                message['identifier'] = ':'.join(identifier)
-                message['ts'] = '{style_dim}{fg_green}%s{style_reset_all} ' % (strftime(cli.config.general.datetime_fmt),) if cli.args.timestamp else ''
-
-                cli.echo('%(ts)s%(color)s%(identifier)s:%(index)d{style_reset_all}: %(text)s' % message)
-
-            except hid.HIDException:
-                break
-
-
-class FindDevices(object):
-    def __init__(self, vid, pid, index, numeric):
-        self.vid = vid
-        self.pid = pid
-        self.index = index
-        self.numeric = numeric
-
-    def run_forever(self):
-        """Process messages from our queue in a loop.
-        """
-        live_devices = {}
-        live_bootloaders = {}
-
-        while True:
-            try:
-                for device in list(live_devices):
-                    if not live_devices[device]['thread'].is_alive():
-                        cli.log.info('Console Disconnected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', live_devices[device])
-                        del live_devices[device]
-
-                for device in self.find_devices():
-                    if device['path'] not in live_devices:
-                        device['color'] = LOG_COLOR['colors'][LOG_COLOR['next']]
-                        LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors'])
-                        live_devices[device['path']] = device
-
-                        try:
-                            monitor = MonitorDevice(device, self.numeric)
-                            device['thread'] = Thread(target=monitor.run_forever, daemon=True)
-
-                            device['thread'].start()
-                        except Exception as e:
-                            device['e'] = e
-                            device['e_name'] = e.__class__.__name__
-                            cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s:%(vendor_id)04X:%(product_id)04X:%(index)d): %(e_name)s: %(e)s", device)
-                            if cli.config.general.verbose:
-                                cli.log.exception(e)
-                            del live_devices[device['path']]
-
-                if cli.args.bootloaders:
-                    for device in self.find_bootloaders():
-                        if device.address in live_bootloaders:
-                            live_bootloaders[device.address]._qmk_found = True
-                        else:
-                            name = KNOWN_BOOTLOADERS[(int2hex(device.idVendor), int2hex(device.idProduct))]
-                            cli.log.info('Bootloader Connected: {style_bright}{fg_magenta}%s', name)
-                            device._qmk_found = True
-                            live_bootloaders[device.address] = device
-
-                    for device in list(live_bootloaders):
-                        if live_bootloaders[device]._qmk_found:
-                            live_bootloaders[device]._qmk_found = False
-                        else:
-                            name = KNOWN_BOOTLOADERS[(int2hex(live_bootloaders[device].idVendor), int2hex(live_bootloaders[device].idProduct))]
-                            cli.log.info('Bootloader Disconnected: {style_bright}{fg_magenta}%s', name)
-                            del live_bootloaders[device]
-
-                sleep(.1)
-
-            except KeyboardInterrupt:
-                break
-
-    def is_bootloader(self, hid_device):
-        """Returns true if the device in question matches a known bootloader vid/pid.
-        """
-        return (int2hex(hid_device.idVendor), int2hex(hid_device.idProduct)) in KNOWN_BOOTLOADERS
-
-    def is_console_hid(self, hid_device):
-        """Returns true when the usage page indicates it's a teensy-style console.
-        """
-        return hid_device['usage_page'] == 0xFF31 and hid_device['usage'] == 0x0074
-
-    def is_filtered_device(self, hid_device):
-        """Returns True if the device should be included in the list of available consoles.
-        """
-        return int2hex(hid_device['vendor_id']) == self.vid and int2hex(hid_device['product_id']) == self.pid
-
-    def find_devices_by_report(self, hid_devices):
-        """Returns a list of available teensy-style consoles by doing a brute-force search.
-
-        Some versions of linux don't report usage and usage_page. In that case we fallback to reading the report (possibly inaccurately) ourselves.
-        """
-        devices = []
-
-        for device in hid_devices:
-            path = device['path'].decode('utf-8')
-
-            if path.startswith('/dev/hidraw'):
-                number = path[11:]
-                report = Path(f'/sys/class/hidraw/hidraw{number}/device/report_descriptor')
-
-                if report.exists():
-                    rp = report.read_bytes()
-
-                    if rp[1] == 0x31 and rp[3] == 0x09:
-                        devices.append(device)
-
-        return devices
-
-    def find_bootloaders(self):
-        """Returns a list of available bootloader devices.
-        """
-        return list(filter(self.is_bootloader, usb.core.find(find_all=True)))
-
-    def find_devices(self):
-        """Returns a list of available teensy-style consoles.
-        """
-        hid_devices = hid.enumerate()
-        devices = list(filter(self.is_console_hid, hid_devices))
-
-        if not devices:
-            devices = self.find_devices_by_report(hid_devices)
-
-        if self.vid and self.pid:
-            devices = list(filter(self.is_filtered_device, devices))
-
-        # Add index numbers
-        device_index = {}
-        for device in devices:
-            id = ':'.join((int2hex(device['vendor_id']), int2hex(device['product_id'])))
-
-            if id not in device_index:
-                device_index[id] = 0
-
-            device_index[id] += 1
-            device['index'] = device_index[id]
-
-        return devices
-
-
-def int2hex(number):
-    """Returns a string representation of the number as hex.
-    """
-    return "%04X" % number
-
-
-def list_devices(device_finder):
-    """Show the user a nicely formatted list of devices.
-    """
-    devices = device_finder.find_devices()
-
-    if devices:
-        cli.log.info('Available devices:')
-        for dev in devices:
-            color = LOG_COLOR['colors'][LOG_COLOR['next']]
-            LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors'])
-            cli.log.info("\t%s%s:%s:%d{style_reset_all}\t%s %s", color, int2hex(dev['vendor_id']), int2hex(dev['product_id']), dev['index'], dev['manufacturer_string'], dev['product_string'])
-
-    if cli.args.bootloaders:
-        bootloaders = device_finder.find_bootloaders()
-
-        if bootloaders:
-            cli.log.info('Available Bootloaders:')
-
-            for dev in bootloaders:
-                cli.log.info("\t%s:%s\t%s", int2hex(dev.idVendor), int2hex(dev.idProduct), KNOWN_BOOTLOADERS[(int2hex(dev.idVendor), int2hex(dev.idProduct))])
-
-
-@cli.argument('--bootloaders', arg_only=True, default=True, action='store_boolean', help='displaying bootloaders.')
-@cli.argument('-d', '--device', help='Device to select - uses format <pid>:<vid>[:<index>].')
-@cli.argument('-l', '--list', arg_only=True, action='store_true', help='List available hid_listen devices.')
-@cli.argument('-n', '--numeric', arg_only=True, action='store_true', help='Show VID/PID instead of names.')
-@cli.argument('-t', '--timestamp', arg_only=True, action='store_true', help='Print the timestamp for received messages as well.')
-@cli.argument('-w', '--wait', type=int, default=1, help="How many seconds to wait between checks (Default: 1)")
-@cli.subcommand('Acquire debugging information from usb hid devices.', hidden=False if cli.config.user.developer else True)
-def console(cli):
-    """Acquire debugging information from usb hid devices
-    """
-    vid = None
-    pid = None
-    index = 1
-
-    if cli.config.console.device:
-        device = cli.config.console.device.split(':')
-
-        if len(device) == 2:
-            vid, pid = device
-
-        elif len(device) == 3:
-            vid, pid, index = device
-
-            if not index.isdigit():
-                cli.log.error('Device index must be a number! Got "%s" instead.', index)
-                exit(1)
-
-            index = int(index)
-
-            if index < 1:
-                cli.log.error('Device index must be greater than 0! Got %s', index)
-                exit(1)
-
-        else:
-            cli.log.error('Invalid format for device, expected "<pid>:<vid>[:<index>]" but got "%s".', cli.config.console.device)
-            cli.print_help()
-            exit(1)
-
-        vid = vid.upper()
-        pid = pid.upper()
-
-    device_finder = FindDevices(vid, pid, index, cli.args.numeric)
-
-    if cli.args.list:
-        return list_devices(device_finder)
-
-    print('Looking for devices...', flush=True)
-    device_finder.run_forever()
--- a/lib/python/qmk/cli/docs.py
+++ b/lib/python/qmk/cli/docs.py
@@ -2,6 +2,7 @@
 """
 import http.server
 import os
+import shutil
 import webbrowser

 from milc import cli
@@ -11,20 +12,33 @@ from milc import cli
@cli.argument('-b', '--browser', action='store_true', help='Open the docs in the default browser.')
@cli.subcommand('Run a local webserver for QMK documentation.', hidden=False if cli.config.user.developer else True)
 def docs(cli):
-    """Spin up a local HTTPServer instance for the QMK docs.
+    """Spin up a local HTTP server for the QMK docs.
    """
    os.chdir('docs')

-    with http.server.HTTPServer(('', cli.config.docs.port), http.server.SimpleHTTPRequestHandler) as httpd:
-        cli.log.info(f"Serving QMK docs at http://localhost:{cli.config.docs.port}/")
+    # If docsify-cli is installed, run that instead so we get live reload
+    if shutil.which('docsify'):
+        command = ['docsify', 'serve', '--port', f'{cli.config.docs.port}', '--open' if cli.config.docs.browser else '']
+
+        cli.log.info(f"Running {{fg_cyan}}{str.join(' ', command)}{{fg_reset}}")
        cli.log.info("Press Control+C to exit.")

-        if cli.config.docs.browser:
-            webbrowser.open(f'http://localhost:{cli.config.docs.port}')
-
        try:
-            httpd.serve_forever()
+            cli.run(command, capture_output=False)
        except KeyboardInterrupt:
            cli.log.info("Stopping HTTP server...")
-        finally:
-            httpd.shutdown()
+    else:
+        # Fall back to Python HTTPServer
+        with http.server.HTTPServer(('', cli.config.docs.port), http.server.SimpleHTTPRequestHandler) as httpd:
+            cli.log.info(f"Serving QMK docs at http://localhost:{cli.config.docs.port}/")
+            cli.log.info("Press Control+C to exit.")
+
+            if cli.config.docs.browser:
+                webbrowser.open(f'http://localhost:{cli.config.docs.port}')
+
+            try:
+                httpd.serve_forever()
+            except KeyboardInterrupt:
+                cli.log.info("Stopping HTTP server...")
+            finally:
+                httpd.shutdown()
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -26,7 +26,6 @@ ESSENTIAL_BINARIES = {
    'arm-none-eabi-gcc': {
        'version_arg': '-dumpversion'
    },
-    'bin/qmk': {},
 }


--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -18,17 +18,21 @@ def print_bootloader_help():
    """Prints the available bootloaders listed in docs.qmk.fm.
    """
    cli.log.info('Here are the available bootloaders:')
+    cli.echo('\tavrdude')
+    cli.echo('\tbootloadhid')
    cli.echo('\tdfu')
+    cli.echo('\tdfu-util')
+    cli.echo('\tmdloader')
+    cli.echo('\tst-flash')
+    cli.echo('\tst-link-cli')
+    cli.log.info('Enhanced variants for split keyboards:')
+    cli.echo('\tavrdude-split-left')
+    cli.echo('\tavrdude-split-right')
    cli.echo('\tdfu-ee')
    cli.echo('\tdfu-split-left')
    cli.echo('\tdfu-split-right')
-    cli.echo('\tavrdude')
-    cli.echo('\tBootloadHID')
-    cli.echo('\tdfu-util')
    cli.echo('\tdfu-util-split-left')
    cli.echo('\tdfu-util-split-right')
-    cli.echo('\tst-link-cli')
-    cli.echo('\tst-flash')
    cli.echo('For more info, visit https://docs.qmk.fm/#/flashing')


--- a/lib/python/qmk/cli/format/c.py
+++ b/lib/python/qmk/cli/format/c.py
@@ -74,7 +74,7 @@ def filter_files(files, core_only=False):
            # The following statement checks each file to see if the file path is
            # - in the core directories
            # - not in the ignored directories
-            if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored):
+            if not any(str(file).startswith(i) for i in core_dirs) or any(str(file).startswith(i) for i in ignored):
                files[index] = None
                cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file)

--- a/lib/python/qmk/cli/format/python.py
+++ b/lib/python/qmk/cli/format/python.py
@@ -4,23 +4,66 @@ from subprocess import CalledProcessError, DEVNULL

 from milc import cli

+from qmk.path import normpath
+
+py_file_suffixes = ('py',)
+py_dirs = ['lib/python']
+
+
+def yapf_run(files):
+    edit = '--diff' if cli.args.dry_run else '--in-place'
+    yapf_cmd = ['yapf', '-vv', '--recursive', edit, *files]
+    try:
+        cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL)
+        cli.log.info('Successfully formatted the python code.')
+
+    except CalledProcessError:
+        cli.log.error(f'Python code in {",".join(py_dirs)} incorrectly formatted!')
+        return False
+
+
+def filter_files(files):
+    """Yield only files to be formatted and skip the rest
+    """
+    for file in files:
+        if file and normpath(file).name.split('.')[-1] in py_file_suffixes:
+            yield file
+        else:
+            cli.log.debug('Skipping file %s', file)
+

@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
+@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
+@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all files.')
+@cli.argument('files', nargs='*', arg_only=True, type=normpath, help='Filename(s) to format.')
@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True)
 def format_python(cli):
    """Format python code according to QMK's style.
    """
-    edit = '--diff' if cli.args.dry_run else '--in-place'
-    yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python']
-    try:
-        cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL)
-        cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.')
-        return True
+    # Find the list of files to format
+    if cli.args.files:
+        files = list(filter_files(cli.args.files))

-    except CalledProcessError:
-        if cli.args.dry_run:
-            cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!')
-        else:
-            cli.log.error('Error formatting python code!')
+        if not files:
+            cli.log.error('No Python files in filelist: %s', ', '.join(map(str, cli.args.files)))
+            exit(0)

-    return False
+        if cli.args.all_files:
+            cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))
+
+    elif cli.args.all_files:
+        git_ls_cmd = ['git', 'ls-files', *py_dirs]
+        git_ls = cli.run(git_ls_cmd, stdin=DEVNULL)
+        files = list(filter_files(git_ls.stdout.split('\n')))
+
+    else:
+        git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *py_dirs]
+        git_diff = cli.run(git_diff_cmd, stdin=DEVNULL)
+        files = list(filter_files(git_diff.stdout.split('\n')))
+
+    # Sanity check
+    if not files:
+        cli.log.error('No changed files detected. Use "qmk format-python -a" to format all files')
+        return False
+
+    return yapf_run(files)
--- a/lib/python/qmk/cli/format/text.py
+++ b/lib/python/qmk/cli/format/text.py
@@ -1,27 +1,57 @@
 """Ensure text files have the proper line endings.
 """
-from subprocess import CalledProcessError
+from itertools import islice
+from subprocess import DEVNULL

 from milc import cli

+from qmk.path import normpath

+
+def _get_chunks(it, size):
+    """Break down a collection into smaller parts
+    """
+    it = iter(it)
+    return iter(lambda: tuple(islice(it, size)), ())
+
+
+def dos2unix_run(files):
+    """Spawn multiple dos2unix subprocess avoiding too long commands on formatting everything
+    """
+    for chunk in _get_chunks(files, 10):
+        dos2unix = cli.run(['dos2unix', *chunk])
+
+        if dos2unix.returncode:
+            return False
+
+
+@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
+@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all files.')
+@cli.argument('files', nargs='*', arg_only=True, type=normpath, help='Filename(s) to format.')
@cli.subcommand("Ensure text files have the proper line endings.", hidden=True)
 def format_text(cli):
    """Ensure text files have the proper line endings.
    """
-    try:
-        file_list_cmd = cli.run(['git', 'ls-files', '-z'], check=True)
-    except CalledProcessError as e:
-        cli.log.error('Could not get file list: %s', e)
-        exit(1)
-    except Exception as e:
-        cli.log.error('Unhandled exception: %s: %s', e.__class__.__name__, e)
-        cli.log.exception(e)
-        exit(1)
+    # Find the list of files to format
+    if cli.args.files:
+        files = list(cli.args.files)

-    dos2unix = cli.run(['xargs', '-0', 'dos2unix'], stdin=None, input=file_list_cmd.stdout)
+        if cli.args.all_files:
+            cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))

-    if dos2unix.returncode != 0:
-        print(dos2unix.stderr)
+    elif cli.args.all_files:
+        git_ls_cmd = ['git', 'ls-files']
+        git_ls = cli.run(git_ls_cmd, stdin=DEVNULL)
+        files = list(filter(None, git_ls.stdout.split('\n')))

-    return dos2unix.returncode
+    else:
+        git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch]
+        git_diff = cli.run(git_diff_cmd, stdin=DEVNULL)
+        files = list(filter(None, git_diff.stdout.split('\n')))
+
+    # Sanity check
+    if not files:
+        cli.log.error('No changed files detected. Use "qmk format-text -a" to format all files')
+        return False
+
+    return dos2unix_run(files)
--- a/lib/python/qmk/cli/generate/compilation_database.py
+++ b/lib/python/qmk/cli/generate/compilation_database.py
@@ -0,0 +1,133 @@
+"""Creates a compilation database for the given keyboard build.
+"""
+
+import json
+import os
+import re
+import shlex
+import shutil
+from functools import lru_cache
+from pathlib import Path
+from typing import Dict, Iterator, List, Union
+
+from milc import cli, MILC
+
+from qmk.commands import create_make_command
+from qmk.constants import QMK_FIRMWARE
+from qmk.decorators import automagic_keyboard, automagic_keymap
+
+
+@lru_cache(maxsize=10)
+def system_libs(binary: str) -> List[Path]:
+    """Find the system include directory that the given build tool uses.
+    """
+    cli.log.debug("searching for system library directory for binary: %s", binary)
+    bin_path = shutil.which(binary)
+
+    # Actually query xxxxxx-gcc to find its include paths.
+    if binary.endswith("gcc") or binary.endswith("g++"):
+        result = cli.run([binary, '-E', '-Wp,-v', '-'], capture_output=True, check=True, input='\n')
+        paths = []
+        for line in result.stderr.splitlines():
+            if line.startswith(" "):
+                paths.append(Path(line.strip()).resolve())
+        return paths
+
+    return list(Path(bin_path).resolve().parent.parent.glob("*/include")) if bin_path else []
+
+
+file_re = re.compile(r'printf "Compiling: ([^"]+)')
+cmd_re = re.compile(r'LOG=\$\((.+?)&&')
+
+
+def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
+    """parse the output of `make -n <target>`
+
+    This function makes many assumptions about the format of your build log.
+    This happens to work right now for qmk.
+    """
+
+    state = 'start'
+    this_file = None
+    records = []
+    for line in f:
+        if state == 'start':
+            m = file_re.search(line)
+            if m:
+                this_file = m.group(1)
+                state = 'cmd'
+
+        if state == 'cmd':
+            assert this_file
+            m = cmd_re.search(line)
+            if m:
+                # we have a hit!
+                this_cmd = m.group(1)
+                args = shlex.split(this_cmd)
+                for s in system_libs(args[0]):
+                    args += ['-isystem', '%s' % s]
+                new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork')
+                records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file})
+                state = 'start'
+
+    return records
+
+
+@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Create a compilation database.')
+@automagic_keyboard
+@automagic_keymap
+def generate_compilation_database(cli: MILC) -> Union[bool, int]:
+    """Creates a compilation database for the given keyboard build.
+
+    Does a make clean, then a make -n for this target and uses the dry-run output to create
+    a compilation database (compile_commands.json). This file can help some IDEs and
+    IDE-like editors work better. For more information about this:
+
+        https://clang.llvm.org/docs/JSONCompilationDatabase.html
+    """
+    command = None
+    # check both config domains: the magic decorator fills in `generate_compilation_database` but the user is
+    # more likely to have set `compile` in their config file.
+    current_keyboard = cli.config.generate_compilation_database.keyboard or cli.config.user.keyboard
+    current_keymap = cli.config.generate_compilation_database.keymap or cli.config.user.keymap
+
+    if current_keyboard and current_keymap:
+        # Generate the make command for a specific keyboard/keymap.
+        command = create_make_command(current_keyboard, current_keymap, dry_run=True)
+    elif not current_keyboard:
+        cli.log.error('Could not determine keyboard!')
+    elif not current_keymap:
+        cli.log.error('Could not determine keymap!')
+
+    if not command:
+        cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
+        cli.echo('usage: qmk compiledb [-kb KEYBOARD] [-km KEYMAP]')
+        return False
+
+    # remove any environment variable overrides which could trip us up
+    env = os.environ.copy()
+    env.pop("MAKEFLAGS", None)
+
+    # re-use same executable as the main make invocation (might be gmake)
+    clean_command = [command[0], 'clean']
+    cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
+    cli.run(clean_command, capture_output=False, check=True, env=env)
+
+    cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
+
+    result = cli.run(command, capture_output=True, check=True, env=env)
+    db = parse_make_n(result.stdout.splitlines())
+    if not db:
+        cli.log.error("Failed to parse output from make output:\n%s", result.stdout)
+        return False
+
+    cli.log.info("Found %s compile commands", len(db))
+
+    dbpath = QMK_FIRMWARE / 'compile_commands.json'
+
+    cli.log.info(f"Writing build database to {dbpath}")
+    dbpath.write_text(json.dumps(db, indent=4))
+
+    return True
--- a/lib/python/qmk/cli/generate/develop_pr_list.py
+++ b/lib/python/qmk/cli/generate/develop_pr_list.py
@@ -0,0 +1,119 @@
+"""Export the initial list of PRs associated with a `develop` merge to `master`.
+"""
+import os
+import re
+from pathlib import Path
+from subprocess import DEVNULL
+
+from milc import cli
+
+cache_timeout = 7 * 86400
+fix_expr = re.compile(r'fix', flags=re.IGNORECASE)
+clean1_expr = re.compile(r'\[(develop|keyboard|keymap|core|cli|bug|docs|feature)\]', flags=re.IGNORECASE)
+clean2_expr = re.compile(r'^(develop|keyboard|keymap|core|cli|bug|docs|feature):', flags=re.IGNORECASE)
+
+
+def _get_pr_info(cache, gh, pr_num):
+    pull = cache.get(f'pull:{pr_num}')
+    if pull is None:
+        print(f'Retrieving info for PR #{pr_num}')
+        pull = gh.pulls.get(owner='qmk', repo='qmk_firmware', pull_number=pr_num)
+        cache.set(f'pull:{pr_num}', pull, cache_timeout)
+    return pull
+
+
+def _try_open_cache(cli):
+    # These dependencies are manually handled because people complain. Fun.
+    try:
+        from sqlite_cache.sqlite_cache import SqliteCache
+    except ImportError:
+        return None
+
+    cache_loc = Path(cli.config_file).parent
+    return SqliteCache(cache_loc)
+
+
+def _get_github():
+    try:
+        from ghapi.all import GhApi
+    except ImportError:
+        return None
+
+    return GhApi()
+
+
+@cli.argument('-f', '--from-ref', default='0.11.0', help='Git revision/tag/reference/branch to begin search')
+@cli.argument('-b', '--branch', default='upstream/develop', help='Git branch to iterate (default: "upstream/develop")')
+@cli.subcommand('Creates the develop PR list.', hidden=False if cli.config.user.developer else True)
+def generate_develop_pr_list(cli):
+    """Retrieves information from GitHub regarding the list of PRs associated
+    with a merge of `develop` branch into `master`.
+
+    Requires environment variable GITHUB_TOKEN to be set.
+    """
+
+    if 'GITHUB_TOKEN' not in os.environ or os.environ['GITHUB_TOKEN'] == '':
+        cli.log.error('Environment variable "GITHUB_TOKEN" is not set.')
+        return 1
+
+    cache = _try_open_cache(cli)
+    gh = _get_github()
+
+    git_args = ['git', 'rev-list', '--oneline', '--no-merges', '--reverse', f'{cli.args.from_ref}...{cli.args.branch}', '^upstream/master']
+    commit_list = cli.run(git_args, capture_output=True, stdin=DEVNULL)
+
+    if cache is None or gh is None:
+        cli.log.error('Missing one or more dependent python packages: "ghapi", "python-sqlite-cache"')
+        return 1
+
+    pr_list_bugs = []
+    pr_list_dependencies = []
+    pr_list_core = []
+    pr_list_keyboards = []
+    pr_list_keyboard_fixes = []
+    pr_list_cli = []
+    pr_list_others = []
+
+    def _categorise_commit(commit_info):
+        def fix_or_normal(info, fixes_collection, normal_collection):
+            if "bug" in info['pr_labels'] or fix_expr.search(info['title']):
+                fixes_collection.append(info)
+            else:
+                normal_collection.append(info)
+
+        if "dependencies" in commit_info['pr_labels']:
+            fix_or_normal(commit_info, pr_list_bugs, pr_list_dependencies)
+        elif "core" in commit_info['pr_labels']:
+            fix_or_normal(commit_info, pr_list_bugs, pr_list_core)
+        elif "keyboard" in commit_info['pr_labels'] or "keymap" in commit_info['pr_labels'] or "via" in commit_info['pr_labels']:
+            fix_or_normal(commit_info, pr_list_keyboard_fixes, pr_list_keyboards)
+        elif "cli" in commit_info['pr_labels']:
+            fix_or_normal(commit_info, pr_list_bugs, pr_list_cli)
+        else:
+            fix_or_normal(commit_info, pr_list_bugs, pr_list_others)
+
+    git_expr = re.compile(r'^(?P<hash>[a-f0-9]+) (?P<title>.*) \(#(?P<pr>[0-9]+)\)$')
+    for line in commit_list.stdout.split('\n'):
+        match = git_expr.search(line)
+        if match:
+            pr_info = _get_pr_info(cache, gh, match.group("pr"))
+            commit_info = {'hash': match.group("hash"), 'title': match.group("title"), 'pr_num': int(match.group("pr")), 'pr_labels': [label.name for label in pr_info.labels.items]}
+            _categorise_commit(commit_info)
+
+    def _dump_commit_list(name, collection):
+        if len(collection) == 0:
+            return
+        print("")
+        print(f"{name}:")
+        for commit in sorted(collection, key=lambda x: x['pr_num']):
+            title = clean1_expr.sub('', clean2_expr.sub('', commit['title'])).strip()
+            pr_num = commit['pr_num']
+            print(f'* {title} ([#{pr_num}](https://github.com/qmk/qmk_firmware/pull/{pr_num}))')
+
+    _dump_commit_list("Core", pr_list_core)
+    _dump_commit_list("CLI", pr_list_cli)
+    _dump_commit_list("Submodule updates", pr_list_dependencies)
+    _dump_commit_list("Keyboards", pr_list_keyboards)
+    _dump_commit_list("Keyboard fixes", pr_list_keyboard_fixes)
+    _dump_commit_list("Others", pr_list_others)
+    _dump_commit_list("Bugs", pr_list_bugs)
--- a/lib/python/qmk/cli/generate/dfu_header.py
+++ b/lib/python/qmk/cli/generate/dfu_header.py
@@ -32,7 +32,7 @@ def generate_dfu_header(cli):

    keyboard_h_lines = ['/* This file was generated by `qmk generate-dfu-header`. Do not edit or copy.', ' */', '', '#pragma once']
    keyboard_h_lines.append(f'#define MANUFACTURER {kb_info_json["manufacturer"]}')
-    keyboard_h_lines.append(f'#define PRODUCT {cli.config.generate_dfu_header.keyboard} Bootloader')
+    keyboard_h_lines.append(f'#define PRODUCT {kb_info_json["keyboard_name"]} Bootloader')

    # Optional
    if 'qmk_lufa_bootloader.esc_output' in kb_info_json:
--- a/lib/python/qmk/cli/generate/rules_mk.py
+++ b/lib/python/qmk/cli/generate/rules_mk.py
@@ -67,12 +67,9 @@ def generate_rules_mk(cli):
    # Iterate through features to enable/disable them
    if 'features' in kb_info_json:
        for feature, enabled in kb_info_json['features'].items():
-            if feature == 'bootmagic_lite' and enabled:
-                rules_mk_lines.append('BOOTMAGIC_ENABLE ?= lite')
-            else:
-                feature = feature.upper()
-                enabled = 'yes' if enabled else 'no'
-                rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}')
+            feature = feature.upper()
+            enabled = 'yes' if enabled else 'no'
+            rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}')

    # Set SPLIT_TRANSPORT, if needed
    if kb_info_json.get('split', {}).get('transport', {}).get('protocol') == 'custom':
--- a/lib/python/qmk/cli/json2c.py
+++ b/lib/python/qmk/cli/json2c.py
@@ -33,7 +33,7 @@ def json2c(cli):
        cli.args.output = None

    # Generate the keymap
-    keymap_c = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+    keymap_c = qmk.keymap.generate_c(user_keymap)

    if cli.args.output:
        cli.args.output.parent.mkdir(parents=True, exist_ok=True)
--- a/lib/python/qmk/cli/lint.py
+++ b/lib/python/qmk/cli/lint.py
@@ -1,72 +1,129 @@
 """Command to look over a keyboard/keymap and check for common mistakes.
 """
+from pathlib import Path
+
 from milc import cli

 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.info import info_json
-from qmk.keyboard import find_readme, keyboard_completer
+from qmk.keyboard import keyboard_completer, list_keyboards
 from qmk.keymap import locate_keymap
 from qmk.path import is_keyboard, keyboard


-@cli.argument('--strict', action='store_true', help='Treat warnings as errors.')
-@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='The keyboard to check.')
-@cli.argument('-km', '--keymap', help='The keymap to check.')
+def keymap_check(kb, km):
+    """Perform the keymap level checks.
+    """
+    ok = True
+    keymap_path = locate_keymap(kb, km)
+
+    if not keymap_path:
+        ok = False
+        cli.log.error("%s: Can't find %s keymap.", kb, km)
+
+    return ok
+
+
+def rules_mk_assignment_only(keyboard_path):
+    """Check the keyboard-level rules.mk to ensure it only has assignments.
+    """
+    current_path = Path()
+    errors = []
+
+    for path_part in keyboard_path.parts:
+        current_path = current_path / path_part
+        rules_mk = current_path / 'rules.mk'
+
+        if rules_mk.exists():
+            continuation = None
+
+            for i, line in enumerate(rules_mk.open()):
+                line = line.strip()
+
+                if '#' in line:
+                    line = line[:line.index('#')]
+
+                if continuation:
+                    line = continuation + line
+                    continuation = None
+
+                if line:
+                    if line[-1] == '\\':
+                        continuation = line[:-1]
+                        continue
+
+                    if line and '=' not in line:
+                        errors.append(f'Non-assignment code on line +{i} {rules_mk}: {line}')
+
+    return errors
+
+
+@cli.argument('--strict', action='store_true', help='Treat warnings as errors')
+@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='Comma separated list of keyboards to check')
+@cli.argument('-km', '--keymap', help='The keymap to check')
+@cli.argument('--all-kb', action='store_true', arg_only=True, help='Check all keyboards')
@cli.subcommand('Check keyboard and keymap for common mistakes.')
@automagic_keyboard
@automagic_keymap
 def lint(cli):
    """Check keyboard and keymap for common mistakes.
    """
-    if not cli.config.lint.keyboard:
-        cli.log.error('Missing required argument: --keyboard')
+    failed = []
+
+    # Determine our keyboard list
+    if cli.args.all_kb:
+        if cli.args.keyboard:
+            cli.log.warning('Both --all-kb and --keyboard passed, --all-kb takes presidence.')
+
+        keyboard_list = list_keyboards()
+    elif not cli.config.lint.keyboard:
+        cli.log.error('Missing required arguments: --keyboard or --all-kb')
        cli.print_help()
        return False
+    else:
+        keyboard_list = cli.config.lint.keyboard.split(',')

-    if not is_keyboard(cli.config.lint.keyboard):
-        cli.log.error('No such keyboard: %s', cli.config.lint.keyboard)
-        return False
+    # Lint each keyboard
+    for kb in keyboard_list:
+        if not is_keyboard(kb):
+            cli.log.error('No such keyboard: %s', kb)
+            continue

-    # Gather data about the keyboard.
-    ok = True
-    keyboard_path = keyboard(cli.config.lint.keyboard)
-    keyboard_info = info_json(cli.config.lint.keyboard)
-    readme_path = find_readme(cli.config.lint.keyboard)
-    missing_readme_path = keyboard_path / 'readme.md'
+        # Gather data about the keyboard.
+        ok = True
+        keyboard_path = keyboard(kb)
+        keyboard_info = info_json(kb)

-    # Check for errors in the info.json
-    if keyboard_info['parse_errors']:
-        ok = False
-        cli.log.error('Errors found when generating info.json.')
-
-    if cli.config.lint.strict and keyboard_info['parse_warnings']:
-        ok = False
-        cli.log.error('Warnings found when generating info.json (Strict mode enabled.)')
-
-    # Check for a readme.md and warn if it doesn't exist
-    if not readme_path:
-        ok = False
-        cli.log.error('Missing %s', missing_readme_path)
-
-    # Keymap specific checks
-    if cli.config.lint.keymap:
-        keymap_path = locate_keymap(cli.config.lint.keyboard, cli.config.lint.keymap)
-
-        if not keymap_path:
+        # Check for errors in the info.json
+        if keyboard_info['parse_errors']:
            ok = False
-            cli.log.error("Can't find %s keymap for %s keyboard.", cli.config.lint.keymap, cli.config.lint.keyboard)
-        else:
-            keymap_readme = keymap_path.parent / 'readme.md'
-            if not keymap_readme.exists():
-                cli.log.warning('Missing %s', keymap_readme)
+            cli.log.error('%s: Errors found when generating info.json.', kb)

-                if cli.config.lint.strict:
-                    ok = False
+        if cli.config.lint.strict and keyboard_info['parse_warnings']:
+            ok = False
+            cli.log.error('%s: Warnings found when generating info.json (Strict mode enabled.)', kb)
+
+        # Check the rules.mk file(s)
+        rules_mk_assignment_errors = rules_mk_assignment_only(keyboard_path)
+        if rules_mk_assignment_errors:
+            ok = False
+            cli.log.error('%s: Non-assignment code found in rules.mk. Move it to post_rules.mk instead.', kb)
+            for assignment_error in rules_mk_assignment_errors:
+                cli.log.error(assignment_error)
+
+        # Keymap specific checks
+        if cli.config.lint.keymap:
+            if not keymap_check(kb, cli.config.lint.keymap):
+                ok = False
+
+        # Report status
+        if not ok:
+            failed.append(kb)

    # Check and report the overall status
-    if ok:
-        cli.log.info('Lint check passed!')
-        return True
+    if failed:
+        cli.log.error('Lint check failed for: %s', ', '.join(failed))
+        return False

-    cli.log.error('Lint check failed!')
-    return False
+    cli.log.info('Lint check passed!')
+    return True
--- a/lib/python/qmk/cli/list/keymaps.py
+++ b/lib/python/qmk/cli/list/keymaps.py
@@ -13,5 +13,10 @@ from qmk.keyboard import keyboard_completer, keyboard_folder
 def list_keymaps(cli):
    """List the keymaps for a specific keyboard
    """
+    if not cli.config.list_keymaps.keyboard:
+        cli.log.error('Missing required arguments: --keyboard')
+        cli.subcommands['list-keymaps'].print_help()
+        return False
+
    for name in qmk.keymap.list_keymaps(cli.config.list_keymaps.keyboard):
        print(name)
--- a/lib/python/qmk/cli/list/layouts.py
+++ b/lib/python/qmk/cli/list/layouts.py
@@ -13,6 +13,11 @@ from qmk.info import info_json
 def list_layouts(cli):
    """List the layouts for a specific keyboard
    """
+    if not cli.config.list_layouts.keyboard:
+        cli.log.error('Missing required arguments: --keyboard')
+        cli.subcommands['list-layouts'].print_help()
+        return False
+
    info_data = info_json(cli.config.list_layouts.keyboard)
    for name in sorted(info_data.get('community_layouts', [])):
        print(name)
--- a/lib/python/qmk/cli/new/keyboard.py
+++ b/lib/python/qmk/cli/new/keyboard.py
@@ -1,10 +1,8 @@
 """This script automates the creation of new keyboard directories using a starter template.
 """
 from datetime import date
-import fileinput
 from pathlib import Path
 import re
-import shutil

 from qmk.commands import git_get_username
 import qmk.path
@@ -32,6 +30,7 @@ def validate_keyboard_name(name):
@cli.argument('-kb', '--keyboard', help='Specify the name for the new keyboard directory', arg_only=True, type=keyboard_name)
@cli.argument('-t', '--type', help='Specify the keyboard type', arg_only=True, choices=KEYBOARD_TYPES)
@cli.argument('-u', '--username', help='Specify your username (default from Git config)', arg_only=True)
+@cli.argument('-n', '--realname', help='Specify your real name if you want to use that. Defaults to username', arg_only=True)
@cli.subcommand('Creates a new keyboard directory')
 def new_keyboard(cli):
    """Creates a new keyboard.
@@ -69,7 +68,7 @@ def new_keyboard(cli):
    # Get username
    user_name = None
    while not user_name:
-        user_name = question('Your Name:', default=find_user_name())
+        user_name = question('Your GitHub User Name:', default=find_user_name())

        if not user_name:
            cli.log.error('You didn\'t provide a username, and we couldn\'t find one set in your QMK or Git configs. Please try again.')
@@ -78,26 +77,21 @@ def new_keyboard(cli):
            if cli.args.username:
                return False

-    # Copy all the files
-    copy_templates(keyboard_type, keyboard_path)
+    real_name = None
+    while not real_name:
+        real_name = question('Your real name:', default=user_name)

-    # Replace all the placeholders
    keyboard_basename = keyboard_path.name
-    replacements = [
-        ('%YEAR%', str(date.today().year)),
-        ('%KEYBOARD%', keyboard_basename),
-        ('%YOUR_NAME%', user_name),
-    ]
-    filenames = [
-        keyboard_path / 'config.h',
-        keyboard_path / 'info.json',
-        keyboard_path / 'readme.md',
-        keyboard_path / f'{keyboard_basename}.c',
-        keyboard_path / f'{keyboard_basename}.h',
-        keyboard_path / 'keymaps/default/readme.md',
-        keyboard_path / 'keymaps/default/keymap.c',
-    ]
-    replace_placeholders(replacements, filenames)
+    replacements = {
+        "YEAR": str(date.today().year),
+        "KEYBOARD": keyboard_basename,
+        "USER_NAME": user_name,
+        "YOUR_NAME": real_name,
+    }
+
+    template_dir = Path('data/templates')
+    template_tree(template_dir / 'base', keyboard_path, replacements)
+    template_tree(template_dir / keyboard_type, keyboard_path, replacements)

    cli.echo('')
    cli.log.info(f'{{fg_green}}Created a new keyboard called {{fg_cyan}}{new_keyboard_name}{{fg_green}}.{{fg_reset}}')
@@ -114,29 +108,32 @@ def find_user_name():
        return git_get_username()


-def copy_templates(keyboard_type, keyboard_path):
-    """Copies the template files from data/templates to the new keyboard directory.
+def template_tree(src: Path, dst: Path, replacements: dict):
+    """Recursively copy template and replace placeholders
+
+    Args:
+        src (Path)
+            The source folder to copy from
+        dst (Path)
+            The destination folder to copy to
+        replacements (dict)
+            a dictionary with "key":"value" pairs to replace.
+
+    Raises:
+        FileExistsError
+            When trying to overwrite existing files
    """
-    template_base_path = Path('data/templates')
-    keyboard_basename = keyboard_path.name

-    cli.log.info('Copying base template files...')
-    shutil.copytree(template_base_path / 'base', keyboard_path)
+    dst.mkdir(parents=True, exist_ok=True)

-    cli.log.info(f'Copying {{fg_cyan}}{keyboard_type}{{fg_reset}} template files...')
-    shutil.copytree(template_base_path / keyboard_type, keyboard_path, dirs_exist_ok=True)
+    for child in src.iterdir():
+        if child.is_dir():
+            template_tree(child, dst / child.name, replacements=replacements)

-    cli.log.info(f'Renaming {{fg_cyan}}keyboard.[ch]{{fg_reset}} to {{fg_cyan}}{keyboard_basename}.[ch]{{fg_reset}}...')
-    shutil.move(keyboard_path / 'keyboard.c', keyboard_path / f'{keyboard_basename}.c')
-    shutil.move(keyboard_path / 'keyboard.h', keyboard_path / f'{keyboard_basename}.h')
+        if child.is_file():
+            file_name = dst / (child.name % replacements)

-
-def replace_placeholders(replacements, filenames):
-    """Replaces the given placeholders in each template file.
-    """
-    for replacement in replacements:
-        cli.log.info(f'Replacing {{fg_cyan}}{replacement[0]}{{fg_reset}} with {{fg_cyan}}{replacement[1]}{{fg_reset}}...')
-
-        with fileinput.input(files=filenames, inplace=True) as file:
-            for line in file:
-                print(line.replace(replacement[0], replacement[1]), end='')
+            with file_name.open(mode='x') as dst_f:
+                with child.open() as src_f:
+                    template = src_f.read()
+                    dst_f.write(template % replacements)
--- a/lib/python/qmk/cli/pytest.py
+++ b/lib/python/qmk/cli/pytest.py
@@ -12,6 +12,6 @@ def pytest(cli):
    """Run several linting/testing commands.
    """
    nose2 = cli.run(['nose2', '-v'], capture_output=False, stdin=DEVNULL)
-    flake8 = cli.run(['flake8', 'lib/python', 'bin/qmk'], capture_output=False, stdin=DEVNULL)
+    flake8 = cli.run(['flake8', 'lib/python'], capture_output=False, stdin=DEVNULL)

    return flake8.returncode | nose2.returncode
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -28,7 +28,7 @@ def _find_make():
    return make_cmd


-def create_make_target(target, parallel=1, **env_vars):
+def create_make_target(target, dry_run=False, parallel=1, **env_vars):
    """Create a make command

    Args:
@@ -36,6 +36,9 @@ def create_make_target(target, parallel=1, **env_vars):
        target
            Usually a make rule, such as 'clean' or 'all'.

+        dry_run
+            make -n -- don't actually build
+
        parallel
            The number of make jobs to run in parallel

@@ -52,10 +55,10 @@ def create_make_target(target, parallel=1, **env_vars):
    for key, value in env_vars.items():
        env.append(f'{key}={value}')

-    return [make_cmd, *get_make_parallel_args(parallel), *env, target]
+    return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]


-def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
+def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
    """Create a make compile command

    Args:
@@ -69,6 +72,9 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
        target
            Usually a bootloader.

+        dry_run
+            make -n -- don't actually build
+
        parallel
            The number of make jobs to run in parallel

@@ -84,7 +90,7 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
    if target:
        make_args.append(target)

-    return create_make_target(':'.join(make_args), parallel, **env_vars)
+    return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)


 def get_git_version(current_time, repo_dir='.', check_dir='.'):
@@ -184,7 +190,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
    target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
    keyboard_output = Path(f'{KEYBOARD_OUTPUT_PREFIX}{keyboard_filesafe}')
    keymap_output = Path(f'{keyboard_output}_{user_keymap["keymap"]}')
-    c_text = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+    c_text = qmk.keymap.generate_c(user_keymap)
    keymap_dir = keymap_output / 'src'
    keymap_c = keymap_dir / 'keymap.c'

@@ -233,7 +239,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
        f'VERBOSE={verbose}',
        f'COLOR={color}',
        'SILENT=false',
-        f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}',
+        'QMK_BIN="qmk"',
    ])

    return make_command
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -13,7 +13,7 @@ QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
 MAX_KEYBOARD_SUBFOLDERS = 5

 # Supported processor types
-CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66F18', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L433', 'STM32L443'
+CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66FX1M0', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L433', 'STM32L443', 'GD32VF103', 'WB32F3G71'
 LUFA_PROCESSORS = 'at90usb162', 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None
 VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'

--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -25,6 +25,13 @@ def _valid_community_layout(layout):
    return (Path('layouts/default') / layout).exists()


+def _remove_newlines_from_labels(layouts):
+    for layout_name, layout_json in layouts.items():
+        for key in layout_json['layout']:
+            if '\n' in key['label']:
+                key['label'] = key['label'].split('\n')[0]
+
+
 def info_json(keyboard):
    """Generate the info.json data for a specific keyboard.
    """
@@ -99,6 +106,9 @@ def info_json(keyboard):
    # Check that the reported matrix size is consistent with the actual matrix size
    _check_matrix(info_data)

+    # Remove newline characters from layout labels
+    _remove_newlines_from_labels(layouts)
+
    return info_data


@@ -112,11 +122,6 @@ def _extract_features(info_data, rules):
    if rules.get('BOOTMAGIC_ENABLE') == 'full':
        rules['BOOTMAGIC_ENABLE'] = 'on'

-    # Skip non-boolean features we haven't implemented special handling for
-    for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
-        if rules.get(feature):
-            del rules[feature]
-
    # Process the rest of the rules as booleans
    for key, value in rules.items():
        if key.endswith('_ENABLE'):
@@ -619,6 +624,8 @@ def arm_processor_rules(info_data, rules):
    if 'bootloader' not in info_data:
        if 'STM32' in info_data['processor']:
            info_data['bootloader'] = 'stm32-dfu'
+        elif 'WB32' in info_data['processor']:
+            info_data['bootloader'] = 'wb32-dfu'
        else:
            info_data['bootloader'] = 'unknown'

--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -17,6 +17,7 @@ from qmk.errors import CppError

 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
+__INCLUDES__

 /* THIS FILE WAS GENERATED!
 *
@@ -27,6 +28,7 @@ DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
 const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
 __KEYMAP_GOES_HERE__
 };
+
 """


@@ -180,10 +182,11 @@ def generate_json(keymap, keyboard, layout, layers):
    return new_keymap


-def generate_c(keyboard, layout, layers):
-    """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers.
+def generate_c(keymap_json):
+    """Returns a `keymap.c`.
+
+    `keymap_json` is a dictionary with the following keys:

-    Args:
        keyboard
            The name of the keyboard

@@ -192,19 +195,89 @@ def generate_c(keyboard, layout, layers):

        layers
            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+
+        macros
+            A sequence of strings containing macros to implement for this keyboard.
    """
-    new_keymap = template_c(keyboard)
+    new_keymap = template_c(keymap_json['keyboard'])
    layer_txt = []
-    for layer_num, layer in enumerate(layers):
+
+    for layer_num, layer in enumerate(keymap_json['layers']):
        if layer_num != 0:
            layer_txt[-1] = layer_txt[-1] + ','
        layer = map(_strip_any, layer)
        layer_keys = ', '.join(layer)
-        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, keymap_json['layout'], layer_keys))

    keymap = '\n'.join(layer_txt)
    new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap)

+    if keymap_json.get('macros'):
+        macro_txt = [
+            'bool process_record_user(uint16_t keycode, keyrecord_t *record) {',
+            '    if (record->event.pressed) {',
+            '        switch (keycode) {',
+        ]
+
+        for i, macro_array in enumerate(keymap_json['macros']):
+            macro = []
+
+            for macro_fragment in macro_array:
+                if isinstance(macro_fragment, str):
+                    macro_fragment = macro_fragment.replace('\\', '\\\\')
+                    macro_fragment = macro_fragment.replace('\r\n', r'\n')
+                    macro_fragment = macro_fragment.replace('\n', r'\n')
+                    macro_fragment = macro_fragment.replace('\r', r'\n')
+                    macro_fragment = macro_fragment.replace('\t', r'\t')
+                    macro_fragment = macro_fragment.replace('"', r'\"')
+
+                    macro.append(f'"{macro_fragment}"')
+
+                elif isinstance(macro_fragment, dict):
+                    newstring = []
+
+                    if macro_fragment['action'] == 'delay':
+                        newstring.append(f"SS_DELAY({macro_fragment['duration']})")
+
+                    elif macro_fragment['action'] == 'beep':
+                        newstring.append(r'"\a"')
+
+                    elif macro_fragment['action'] == 'tap' and len(macro_fragment['keycodes']) > 1:
+                        last_keycode = macro_fragment['keycodes'].pop()
+
+                        for keycode in macro_fragment['keycodes']:
+                            newstring.append(f'SS_DOWN(X_{keycode})')
+
+                        newstring.append(f'SS_TAP(X_{last_keycode})')
+
+                        for keycode in reversed(macro_fragment['keycodes']):
+                            newstring.append(f'SS_UP(X_{keycode})')
+
+                    else:
+                        for keycode in macro_fragment['keycodes']:
+                            newstring.append(f"SS_{macro_fragment['action'].upper()}(X_{keycode})")
+
+                    macro.append(''.join(newstring))
+
+            new_macro = "".join(macro)
+            new_macro = new_macro.replace('""', '')
+            macro_txt.append(f'            case MACRO_{i}:')
+            macro_txt.append(f'                SEND_STRING({new_macro});')
+            macro_txt.append('                return false;')
+
+        macro_txt.append('        }')
+        macro_txt.append('    }')
+        macro_txt.append('\n    return true;')
+        macro_txt.append('};')
+        macro_txt.append('')
+
+        new_keymap = '\n'.join((new_keymap, *macro_txt))
+
+    if keymap_json.get('host_language'):
+        new_keymap = new_keymap.replace('__INCLUDES__', f'#include "keymap_{keymap_json["host_language"]}.h"\n#include "sendstring_{keymap_json["host_language"]}.h"\n')
+    else:
+        new_keymap = new_keymap.replace('__INCLUDES__', '')
+
    return new_keymap


@@ -217,7 +290,7 @@ def write_file(keymap_filename, keymap_content):
    return keymap_filename


-def write_json(keyboard, keymap, layout, layers):
+def write_json(keyboard, keymap, layout, layers, macros=None):
    """Generate the `keymap.json` and write it to disk.

    Returns the filename written to.
@@ -235,19 +308,19 @@ def write_json(keyboard, keymap, layout, layers):
        layers
            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
    """
-    keymap_json = generate_json(keyboard, keymap, layout, layers)
+    keymap_json = generate_json(keyboard, keymap, layout, layers, macros=None)
    keymap_content = json.dumps(keymap_json)
    keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json'

    return write_file(keymap_file, keymap_content)


-def write(keyboard, keymap, layout, layers):
+def write(keymap_json):
    """Generate the `keymap.c` and write it to disk.

    Returns the filename written to.

-    Args:
+    `keymap_json` should be a dict with the following keys:
        keyboard
            The name of the keyboard

@@ -259,9 +332,12 @@ def write(keyboard, keymap, layout, layers):

        layers
            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+
+        macros
+            A list of macros for this keymap.
    """
-    keymap_content = generate_c(keyboard, layout, layers)
-    keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c'
+    keymap_content = generate_c(keymap_json)
+    keymap_file = qmk.path.keymap(keymap_json['keyboard']) / keymap_json['keymap'] / 'keymap.c'

    return write_file(keymap_file, keymap_content)

--- a/lib/python/qmk/tests/minimal_info.json
+++ b/lib/python/qmk/tests/minimal_info.json
@@ -4,7 +4,7 @@
    "layouts": {
        "LAYOUT": {
            "layout": [
-                { "label": "KC_A", "x": 0, "y": 0, "matrix": [0, 0] }
+                { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }
            ]
        }
    }
--- a/lib/python/qmk/tests/test_cli_commands.py
+++ b/lib/python/qmk/tests/test_cli_commands.py
@@ -81,9 +81,9 @@ def test_hello():


 def test_format_python():
-    result = check_subcommand('format-python', '--dry-run')
+    result = check_subcommand('format-python', '-n', '-a')
    check_returncode(result)
-    assert 'Python code in `bin/qmk` and `lib/python` is correctly formatted.' in result.stdout
+    assert 'Successfully formatted the python code.' in result.stdout


 def test_list_keyboards():
@@ -142,6 +142,14 @@ def test_json2c():
    assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n'


+def test_json2c_macros():
+    result = check_subcommand("json2c", 'keyboards/handwired/pytest/macro/keymaps/default/keymap.json')
+    check_returncode(result)
+    assert 'LAYOUT_ortho_1x1(MACRO_0)' in result.stdout
+    assert 'case MACRO_0:' in result.stdout
+    assert 'SEND_STRING("Hello, World!"SS_TAP(X_ENTER));' in result.stdout
+
+
 def test_json2c_stdin():
    result = check_subcommand_stdin('keyboards/handwired/pytest/has_template/keymaps/default_json/keymap.json', 'json2c', '-')
    check_returncode(result)
@@ -151,7 +159,7 @@ def test_json2c_stdin():
 def test_info():
    result = check_subcommand('info', '-kb', 'handwired/pytest/basic')
    check_returncode(result)
-    assert 'Keyboard Name: handwired/pytest/basic' in result.stdout
+    assert 'Keyboard Name: pytest' in result.stdout
    assert 'Processor: atmega32u4' in result.stdout
    assert 'Layout:' not in result.stdout
    assert 'k0' not in result.stdout
@@ -160,7 +168,7 @@ def test_info():
 def test_info_keyboard_render():
    result = check_subcommand('info', '-kb', 'handwired/pytest/basic', '-l')
    check_returncode(result)
-    assert 'Keyboard Name: handwired/pytest/basic' in result.stdout
+    assert 'Keyboard Name: pytest' in result.stdout
    assert 'Processor: atmega32u4' in result.stdout
    assert 'Layouts:' in result.stdout
    assert 'k0' in result.stdout
@@ -169,7 +177,7 @@ def test_info_keyboard_render():
 def test_info_keymap_render():
    result = check_subcommand('info', '-kb', 'handwired/pytest/basic', '-km', 'default_json')
    check_returncode(result)
-    assert 'Keyboard Name: handwired/pytest/basic' in result.stdout
+    assert 'Keyboard Name: pytest' in result.stdout
    assert 'Processor: atmega32u4' in result.stdout

    if is_windows:
@@ -181,7 +189,7 @@ def test_info_keymap_render():
 def test_info_matrix_render():
    result = check_subcommand('info', '-kb', 'handwired/pytest/basic', '-m')
    check_returncode(result)
-    assert 'Keyboard Name: handwired/pytest/basic' in result.stdout
+    assert 'Keyboard Name: pytest' in result.stdout
    assert 'Processor: atmega32u4' in result.stdout
    assert 'LAYOUT_ortho_1x1' in result.stdout

@@ -242,7 +250,7 @@ def test_generate_config_h():
    assert '#   define DESCRIPTION handwired/pytest/basic' in result.stdout
    assert '#   define DIODE_DIRECTION COL2ROW' in result.stdout
    assert '#   define MANUFACTURER none' in result.stdout
-    assert '#   define PRODUCT handwired/pytest/basic' in result.stdout
+    assert '#   define PRODUCT pytest' in result.stdout
    assert '#   define PRODUCT_ID 0x6465' in result.stdout
    assert '#   define VENDOR_ID 0xFEED' in result.stdout
    assert '#   define MATRIX_COLS 1' in result.stdout
--- a/lib/python/qmk/tests/test_qmk_keymap.py
+++ b/lib/python/qmk/tests/test_qmk_keymap.py
@@ -22,7 +22,13 @@ def test_template_json_pytest_has_template():


 def test_generate_c_pytest_has_template():
-    templ = qmk.keymap.generate_c('handwired/pytest/has_template', 'LAYOUT', [['KC_A']])
+    keymap_json = {
+        'keyboard': 'handwired/pytest/has_template',
+        'layout': 'LAYOUT',
+        'layers': [['KC_A']],
+        'macros': None,
+    }
+    templ = qmk.keymap.generate_c(keymap_json)
    assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n'