QMK CLI and JSON keymap support (#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function

.editorconfig

@@ -16,6 +16,10 @@ insert_final_newline = true
 trim_trailing_whitespace = false
 indent_size = 4
 
+[{qmk,*.py}]
+charset = utf-8
+max_line_length = 200
+
 # Make these match what we have in .gitattributes
 [*.mk]
 end_of_line = lf

.gitignore

@@ -63,3 +63,6 @@ util/Win_Check_Output.txt
 secrets.tar
 id_rsa_*
 /.vs
+
+# python things
+__pycache__

bin/qmk

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""CLI wrapper for running QMK commands.
+"""
+import os
+import subprocess
+import sys
+from glob import glob
+from time import strftime
+from importlib import import_module
+from importlib.util import find_spec
+
+# Add the QMK python libs to our path
+script_dir = os.path.dirname(os.path.realpath(__file__))
+qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
+python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
+sys.path.append(python_lib_dir)
+
+# Change to the root of our checkout
+os.environ['ORIG_CWD'] = os.getcwd()
+os.chdir(qmk_dir)
+
+# Make sure our modules have been setup
+with open('requirements.txt', 'r') as fd:
+    for line in fd.readlines():
+        line = line.strip().replace('<', '=').replace('>', '=')
+
+        if line[0] == '#':
+            continue
+
+        if '#' in line:
+            line = line.split('#')[0]
+
+        module = line.split('=')[0] if '=' in line else line
+        if not find_spec(module):
+            print('Your QMK build environment is not fully setup!\n')
+            print('Please run `./util/qmk_install.sh` to setup QMK.')
+            exit(255)
+
+# Figure out our version
+command = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags']
+result = subprocess.run(command, text=True, capture_output=True)
+
+if result.returncode == 0:
+    os.environ['QMK_VERSION'] = 'QMK ' + result.stdout.strip()
+else:
+    os.environ['QMK_VERSION'] = 'QMK ' + strftime('%Y-%m-%d-%H:%M:%S')
+
+# Setup the CLI
+import milc
+milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}ψ{style_reset_all}'
+
+# If we were invoked as `qmk <cmd>` massage sys.argv into `qmk-<cmd>`.
+# This means we can't accept arguments to the qmk script itself.
+script_name = os.path.basename(sys.argv[0])
+if script_name == 'qmk':
+    if len(sys.argv) == 1:
+        milc.cli.log.error('No subcommand specified!\n')
+
+    if len(sys.argv) == 1 or sys.argv[1] in ['-h', '--help']:
+        milc.cli.echo('usage: qmk <subcommand> [...]')
+        milc.cli.echo('\nsubcommands:')
+        subcommands = glob(os.path.join(qmk_dir, 'bin', 'qmk-*'))
+        for subcommand in sorted(subcommands):
+            subcommand = os.path.basename(subcommand).split('-', 1)[1]
+            milc.cli.echo('\t%s', subcommand)
+        milc.cli.echo('\nqmk <subcommand> --help for more information')
+        exit(1)
+
+    if sys.argv[1] in ['-V', '--version']:
+        milc.cli.echo(os.environ['QMK_VERSION'])
+        exit(0)
+
+    sys.argv[0] = script_name = '-'.join((script_name, sys.argv[1]))
+    del sys.argv[1]
+
+# Look for which module to import
+if script_name == 'qmk':
+    milc.cli.print_help()
+    exit(0)
+elif not script_name.startswith('qmk-'):
+    milc.cli.log.error('Invalid symlink, must start with "qmk-": %s', script_name)
+else:
+    subcommand = script_name.replace('-', '.').replace('_', '.').split('.')
+    subcommand.insert(1, 'cli')
+    subcommand = '.'.join(subcommand)
+
+    try:
+        import_module(subcommand)
+    except ModuleNotFoundError as e:
+        if e.__class__.__name__ != subcommand:
+            raise
+
+        milc.cli.log.error('Invalid subcommand! Could not import %s.', subcommand)
+        exit(1)
+
+if __name__ == '__main__':
+    milc.cli()

bin/qmk-compile-json

@@ -0,0 +1 @@
+qmk

bin/qmk-doctor

@@ -0,0 +1 @@
+qmk

bin/qmk-hello

@@ -0,0 +1 @@
+qmk

bin/qmk-json-keymap

@@ -0,0 +1 @@
+qmk

build_json.mk

@@ -0,0 +1,27 @@
+# Look for a json keymap file
+ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+endif
+
+# Generate the keymap.c
+ifneq ("$(KEYMAP_JSON)","")
+    _ = $(shell bin/qmk-json-keymap -f $(KEYMAP_JSON) -o $(KEYMAP_C))
+endif

build_keyboard.mk

@@ -98,31 +98,38 @@ MAIN_KEYMAP_PATH_3 := $(KEYBOARD_PATH_3)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_4 := $(KEYBOARD_PATH_4)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 
-ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+# Check for keymap.json first, so we can regenerate keymap.c
+include build_json.mk
+
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look through the possible keymap folders until we find a matching keymap.c
+    ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
         -include $(MAIN_KEYMAP_PATH_5)/rules.mk
         KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
         KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
         -include $(MAIN_KEYMAP_PATH_4)/rules.mk
         KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
         KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
         -include $(MAIN_KEYMAP_PATH_3)/rules.mk
         KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
         KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
         -include $(MAIN_KEYMAP_PATH_2)/rules.mk
         KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
         KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
         -include $(MAIN_KEYMAP_PATH_1)/rules.mk
         KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
         KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
-else ifneq ($(LAYOUTS),)
+    else ifneq ($(LAYOUTS),)
+        # If we haven't found a keymap yet fall back to community layouts
         include build_layout.mk
-else
+    else
         $(error Could not find keymap)
         # this state should never be reached
+    endif
 endif
 
 ifeq ($(strip $(CTPC)), yes)
@@ -313,7 +320,6 @@ ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
 endif
 
-
 # Object files directory
 #     To put object files in current directory, use a dot (.), do NOT make
 #     this an empty or blank macro!
@@ -323,7 +329,7 @@ ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif
 
-# # project specific files
+# project specific files
 SRC += $(KEYBOARD_SRC) \
     $(KEYMAP_C) \
     $(QUANTUM_SRC)

lib/python/milc.py

@@ -0,0 +1,716 @@
+#!/usr/bin/env python3
+# coding=utf-8
+"""MILC - A CLI Framework
+
+PYTHON_ARGCOMPLETE_OK
+
+MILC is an opinionated framework for writing CLI apps. It optimizes for the
+most common unix tool pattern- small tools that are run from the command
+line but generally do not feature any user interaction while they run.
+
+For more details see the MILC documentation:
+
+    <https://github.com/clueboard/milc/tree/master/docs>
+"""
+from __future__ import division, print_function, unicode_literals
+import argparse
+import logging
+import os
+import re
+import sys
+from decimal import Decimal
+from tempfile import NamedTemporaryFile
+from time import sleep
+
+try:
+    from ConfigParser import RawConfigParser
+except ImportError:
+    from configparser import RawConfigParser
+
+try:
+    import thread
+    import threading
+except ImportError:
+    thread = None
+
+import argcomplete
+import colorama
+
+# Log Level Representations
+EMOJI_LOGLEVELS = {
+    'CRITICAL': '{bg_red}{fg_white}¬_¬{style_reset_all}',
+    'ERROR': '{fg_red}☒{style_reset_all}',
+    'WARNING': '{fg_yellow}⚠{style_reset_all}',
+    'INFO': '{fg_blue}ℹ{style_reset_all}',
+    'DEBUG': '{fg_cyan}☐{style_reset_all}',
+    'NOTSET': '{style_reset_all}¯\\_(o_o)_/¯'
+}
+EMOJI_LOGLEVELS['FATAL'] = EMOJI_LOGLEVELS['CRITICAL']
+EMOJI_LOGLEVELS['WARN'] = EMOJI_LOGLEVELS['WARNING']
+
+# ANSI Color setup
+# Regex was gratefully borrowed from kfir on stackoverflow:
+# https://stackoverflow.com/a/45448194
+ansi_regex = r'\x1b(' \
+             r'(\[\??\d+[hl])|' \
+             r'([=<>a-kzNM78])|' \
+             r'([\(\)][a-b0-2])|' \
+             r'(\[\d{0,2}[ma-dgkjqi])|' \
+             r'(\[\d+;\d+[hfy]?)|' \
+             r'(\[;?[hf])|' \
+             r'(#[3-68])|' \
+             r'([01356]n)|' \
+             r'(O[mlnp-z]?)|' \
+             r'(/Z)|' \
+             r'(\d+)|' \
+             r'(\[\?\d;\d0c)|' \
+             r'(\d;\dR))'
+ansi_escape = re.compile(ansi_regex, flags=re.IGNORECASE)
+ansi_styles = (
+    ('fg', colorama.ansi.AnsiFore()),
+    ('bg', colorama.ansi.AnsiBack()),
+    ('style', colorama.ansi.AnsiStyle()),
+)
+ansi_colors = {}
+
+for prefix, obj in ansi_styles:
+    for color in [x for x in obj.__dict__ if not x.startswith('_')]:
+        ansi_colors[prefix + '_' + color.lower()] = getattr(obj, color)
+
+
+def format_ansi(text):
+    """Return a copy of text with certain strings replaced with ansi.
+    """
+    # Avoid .format() so we don't have to worry about the log content
+    for color in ansi_colors:
+        text = text.replace('{%s}' % color, ansi_colors[color])
+    return text + ansi_colors['style_reset_all']
+
+
+class ANSIFormatter(logging.Formatter):
+    """A log formatter that inserts ANSI color.
+    """
+
+    def format(self, record):
+        msg = super(ANSIFormatter, self).format(record)
+        return format_ansi(msg)
+
+
+class ANSIEmojiLoglevelFormatter(ANSIFormatter):
+    """A log formatter that makes the loglevel an emoji.
+    """
+
+    def format(self, record):
+        record.levelname = EMOJI_LOGLEVELS[record.levelname].format(**ansi_colors)
+        return super(ANSIEmojiLoglevelFormatter, self).format(record)
+
+
+class ANSIStrippingFormatter(ANSIFormatter):
+    """A log formatter that strips ANSI.
+    """
+
+    def format(self, record):
+        msg = super(ANSIStrippingFormatter, self).format(record)
+        return ansi_escape.sub('', msg)
+
+
+class Configuration(object):
+    """Represents the running configuration.
+
+    This class never raises IndexError, instead it will return None if a
+    section or option does not yet exist.
+    """
+
+    def __contains__(self, key):
+        return self._config.__contains__(key)
+
+    def __iter__(self):
+        return self._config.__iter__()
+
+    def __len__(self):
+        return self._config.__len__()
+
+    def __repr__(self):
+        return self._config.__repr__()
+
+    def keys(self):
+        return self._config.keys()
+
+    def items(self):
+        return self._config.items()
+
+    def values(self):
+        return self._config.values()
+
+    def __init__(self, *args, **kwargs):
+        self._config = {}
+        self.default_container = ConfigurationOption
+
+    def __getitem__(self, key):
+        """Returns a config section, creating it if it doesn't exist yet.
+        """
+        if key not in self._config:
+            self.__dict__[key] = self._config[key] = ConfigurationOption()
+
+        return self._config[key]
+
+    def __setitem__(self, key, value):
+        self.__dict__[key] = value
+        self._config[key] = value
+
+    def __delitem__(self, key):
+        if key in self.__dict__ and key[0] != '_':
+            del self.__dict__[key]
+        del self._config[key]
+
+
+class ConfigurationOption(Configuration):
+    def __init__(self, *args, **kwargs):
+        super(ConfigurationOption, self).__init__(*args, **kwargs)
+        self.default_container = dict
+
+    def __getitem__(self, key):
+        """Returns a config section, creating it if it doesn't exist yet.
+        """
+        if key not in self._config:
+            self.__dict__[key] = self._config[key] = None
+
+        return self._config[key]
+
+
+def handle_store_boolean(self, *args, **kwargs):
+    """Does the add_argument for action='store_boolean'.
+    """
+    kwargs['add_dest'] = False
+    disabled_args = None
+    disabled_kwargs = kwargs.copy()
+    disabled_kwargs['action'] = 'store_false'
+    disabled_kwargs['help'] = 'Disable ' + kwargs['help']
+    kwargs['action'] = 'store_true'
+    kwargs['help'] = 'Enable ' + kwargs['help']
+
+    for flag in args:
+        if flag[:2] == '--':
+            disabled_args = ('--no-' + flag[2:],)
+            break
+
+    self.add_argument(*args, **kwargs)
+    self.add_argument(*disabled_args, **disabled_kwargs)
+
+    return (args, kwargs, disabled_args, disabled_kwargs)
+
+
+class SubparserWrapper(object):
+    """Wrap subparsers so we can populate the normal and the shadow parser.
+    """
+
+    def __init__(self, cli, submodule, subparser):
+        self.cli = cli
+        self.submodule = submodule
+        self.subparser = subparser
+
+        for attr in dir(subparser):
+            if not hasattr(self, attr):
+                setattr(self, attr, getattr(subparser, attr))
+
+    def completer(self, completer):
+        """Add an arpcomplete completer to this subcommand.
+        """
+        self.subparser.completer = completer
+
+    def add_argument(self, *args, **kwargs):
+        if kwargs.get('add_dest', True):
+            kwargs['dest'] = self.submodule + '_' + self.cli.get_argument_name(*args, **kwargs)
+        if 'add_dest' in kwargs:
+            del kwargs['add_dest']
+
+        if 'action' in kwargs and kwargs['action'] == 'store_boolean':
+            return handle_store_boolean(self, *args, **kwargs)
+
+        self.cli.acquire_lock()
+        self.subparser.add_argument(*args, **kwargs)
+
+        if 'default' in kwargs:
+            del kwargs['default']
+        if 'action' in kwargs and kwargs['action'] == 'store_false':
+            kwargs['action'] == 'store_true'
+        self.cli.subcommands_default[self.submodule].add_argument(*args, **kwargs)
+        self.cli.release_lock()
+
+
+class MILC(object):
+    """MILC - An Opinionated Batteries Included Framework
+    """
+
+    def __init__(self):
+        """Initialize the MILC object.
+        """
+        # Setup a lock for thread safety
+        self._lock = threading.RLock() if thread else None
+
+        # Define some basic info
+        self.acquire_lock()
+        self._description = None
+        self._entrypoint = None
+        self._inside_context_manager = False
+        self.ansi = ansi_colors
+        self.config = Configuration()
+        self.config_file = None
+        self.prog_name = sys.argv[0][:-3] if sys.argv[0].endswith('.py') else sys.argv[0]
+        self.version = os.environ.get('QMK_VERSION', 'unknown')
+        self.release_lock()
+
+        # Initialize all the things
+        self.initialize_argparse()
+        self.initialize_logging()
+
+    @property
+    def description(self):
+        return self._description
+
+    @description.setter
+    def description(self, value):
+        self._description = self._arg_parser.description = self._arg_defaults.description = value
+
+    def echo(self, text, *args, **kwargs):
+        """Print colorized text to stdout, as long as stdout is a tty.
+
+        ANSI color strings (such as {fg-blue}) will be converted into ANSI
+        escape sequences, and the ANSI reset sequence will be added to all
+        strings.
+
+        If *args or **kwargs are passed they will be used to %-format the strings.
+        """
+        if args and kwargs:
+            raise RuntimeError('You can only specify *args or **kwargs, not both!')
+
+        if sys.stdout.isatty():
+            args = args or kwargs
+            text = format_ansi(text)
+
+            print(text % args)
+
+    def initialize_argparse(self):
+        """Prepare to process arguments from sys.argv.
+        """
+        kwargs = {
+            'fromfile_prefix_chars': '@',
+            'conflict_handler': 'resolve',
+        }
+
+        self.acquire_lock()
+        self.subcommands = {}
+        self.subcommands_default = {}
+        self._subparsers = None
+        self._subparsers_default = None
+        self.argwarn = argcomplete.warn
+        self.args = None
+        self._arg_defaults = argparse.ArgumentParser(**kwargs)
+        self._arg_parser = argparse.ArgumentParser(**kwargs)
+        self.set_defaults = self._arg_parser.set_defaults
+        self.print_usage = self._arg_parser.print_usage
+        self.print_help = self._arg_parser.print_help
+        self.release_lock()
+
+    def completer(self, completer):
+        """Add an arpcomplete completer to this subcommand.
+        """
+        self._arg_parser.completer = completer
+
+    def add_argument(self, *args, **kwargs):
+        """Wrapper to add arguments to both the main and the shadow argparser.
+        """
+        if kwargs.get('add_dest', True) and args[0][0] == '-':
+            kwargs['dest'] = 'general_' + self.get_argument_name(*args, **kwargs)
+        if 'add_dest' in kwargs:
+            del kwargs['add_dest']
+
+        if 'action' in kwargs and kwargs['action'] == 'store_boolean':
+            return handle_store_boolean(self, *args, **kwargs)
+
+        self.acquire_lock()
+        self._arg_parser.add_argument(*args, **kwargs)
+
+        # Populate the shadow parser
+        if 'default' in kwargs:
+            del kwargs['default']
+        if 'action' in kwargs and kwargs['action'] == 'store_false':
+            kwargs['action'] == 'store_true'
+        self._arg_defaults.add_argument(*args, **kwargs)
+        self.release_lock()
+
+    def initialize_logging(self):
+        """Prepare the defaults for the logging infrastructure.
+        """
+        self.acquire_lock()
+        self.log_file = None
+        self.log_file_mode = 'a'
+        self.log_file_handler = None
+        self.log_print = True
+        self.log_print_to = sys.stderr
+        self.log_print_level = logging.INFO
+        self.log_file_level = logging.DEBUG
+        self.log_level = logging.INFO
+        self.log = logging.getLogger(self.__class__.__name__)
+        self.log.setLevel(logging.DEBUG)
+        logging.root.setLevel(logging.DEBUG)
+        self.release_lock()
+
+        self.add_argument('-V', '--version', version=self.version, action='version', help='Display the version and exit')
+        self.add_argument('-v', '--verbose', action='store_true', help='Make the logging more verbose')
+        self.add_argument('--datetime-fmt', default='%Y-%m-%d %H:%M:%S', help='Format string for datetimes')
+        self.add_argument('--log-fmt', default='%(levelname)s %(message)s', help='Format string for printed log output')
+        self.add_argument('--log-file-fmt', default='[%(levelname)s] [%(asctime)s] [file:%(pathname)s] [line:%(lineno)d] %(message)s', help='Format string for log file.')
+        self.add_argument('--log-file', help='File to write log messages to')
+        self.add_argument('--color', action='store_boolean', default=True, help='color in output')
+        self.add_argument('-c', '--config-file', help='The config file to read and/or write')
+        self.add_argument('--save-config', action='store_true', help='Save the running configuration to the config file')
+
+    def add_subparsers(self, title='Sub-commands', **kwargs):
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before the with statement!')
+
+        self.acquire_lock()
+        self._subparsers_default = self._arg_defaults.add_subparsers(title=title, dest='subparsers', **kwargs)
+        self._subparsers = self._arg_parser.add_subparsers(title=title, dest='subparsers', **kwargs)
+        self.release_lock()
+
+    def acquire_lock(self):
+        """Acquire the MILC lock for exclusive access to properties.
+        """
+        if self._lock:
+            self._lock.acquire()
+
+    def release_lock(self):
+        """Release the MILC lock.
+        """
+        if self._lock:
+            self._lock.release()
+
+    def find_config_file(self):
+        """Locate the config file.
+        """
+        if self.config_file:
+            return self.config_file
+
+        if self.args and self.args.general_config_file:
+            return self.args.general_config_file
+
+        return os.path.abspath(os.path.expanduser('~/.%s.ini' % self.prog_name))
+
+    def get_argument_name(self, *args, **kwargs):
+        """Takes argparse arguments and returns the dest name.
+        """
+        try:
+            return self._arg_parser._get_optional_kwargs(*args, **kwargs)['dest']
+        except ValueError:
+            return self._arg_parser._get_positional_kwargs(*args, **kwargs)['dest']
+
+    def argument(self, *args, **kwargs):
+        """Decorator to call self.add_argument or self.<subcommand>.add_argument.
+        """
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before the with statement!')
+
+        def argument_function(handler):
+            if handler is self._entrypoint:
+                self.add_argument(*args, **kwargs)
+
+            elif handler.__name__ in self.subcommands:
+                self.subcommands[handler.__name__].add_argument(*args, **kwargs)
+
+            else:
+                raise RuntimeError('Decorated function is not entrypoint or subcommand!')
+
+            return handler
+
+        return argument_function
+
+    def arg_passed(self, arg):
+        """Returns True if arg was passed on the command line.
+        """
+        return self.args_passed[arg] in (None, False)
+
+    def parse_args(self):
+        """Parse the CLI args.
+        """
+        if self.args:
+            self.log.debug('Warning: Arguments have already been parsed, ignoring duplicate attempt!')
+            return
+
+        argcomplete.autocomplete(self._arg_parser)
+
+        self.acquire_lock()
+        self.args = self._arg_parser.parse_args()
+        self.args_passed = self._arg_defaults.parse_args()
+
+        if 'entrypoint' in self.args:
+            self._entrypoint = self.args.entrypoint
+
+        if self.args.general_config_file:
+            self.config_file = self.args.general_config_file
+
+        self.release_lock()
+
+    def read_config(self):
+        """Parse the configuration file and determine the runtime configuration.
+        """
+        self.acquire_lock()
+        self.config_file = self.find_config_file()
+
+        if self.config_file and os.path.exists(self.config_file):
+            config = RawConfigParser(self.config)
+            config.read(self.config_file)
+
+            # Iterate over the config file options and write them into self.config
+            for section in config.sections():
+                for option in config.options(section):
+                    value = config.get(section, option)
+
+                    # Coerce values into useful datatypes
+                    if value.lower() in ['1', 'yes', 'true', 'on']:
+                        value = True
+                    elif value.lower() in ['0', 'no', 'false', 'none', 'off']:
+                        value = False
+                    elif value.replace('.', '').isdigit():
+                        if '.' in value:
+                            value = Decimal(value)
+                        else:
+                            value = int(value)
+
+                    self.config[section][option] = value
+
+        # Fold the CLI args into self.config
+        for argument in vars(self.args):
+            if argument in ('subparsers', 'entrypoint'):
+                continue
+
+            if '_' not in argument:
+                continue
+
+            section, option = argument.split('_', 1)
+            if hasattr(self.args_passed, argument):
+                self.config[section][option] = getattr(self.args, argument)
+            else:
+                if option not in self.config[section]:
+                    self.config[section][option] = getattr(self.args, argument)
+
+        self.release_lock()
+
+    def save_config(self):
+        """Save the current configuration to the config file.
+        """
+        self.log.debug("Saving config file to '%s'", self.config_file)
+
+        if not self.config_file:
+            self.log.warning('%s.config_file file not set, not saving config!', self.__class__.__name__)
+            return
+
+        self.acquire_lock()
+
+        config = RawConfigParser()
+        for section_name, section in self.config._config.items():
+            config.add_section(section_name)
+            for option_name, value in section.items():
+                if section_name == 'general':
+                    if option_name in ['save_config']:
+                        continue
+                config.set(section_name, option_name, str(value))
+
+        with NamedTemporaryFile(mode='w', dir=os.path.dirname(self.config_file), delete=False) as tmpfile:
+            config.write(tmpfile)
+
+        # Move the new config file into place atomically
+        if os.path.getsize(tmpfile.name) > 0:
+            os.rename(tmpfile.name, self.config_file)
+        else:
+            self.log.warning('Config file saving failed, not replacing %s with %s.', self.config_file, tmpfile.name)
+
+        self.release_lock()
+
+    def __call__(self):
+        """Execute the entrypoint function.
+        """
+        if not self._inside_context_manager:
+            # If they didn't use the context manager use it ourselves
+            with self:
+                self.__call__()
+                return
+
+        if not self._entrypoint:
+            raise RuntimeError('No entrypoint provided!')
+
+        return self._entrypoint(self)
+
+    def entrypoint(self, description):
+        """Set the entrypoint for when no subcommand is provided.
+        """
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before cli()!')
+
+        self.acquire_lock()
+        self.description = description
+        self.release_lock()
+
+        def entrypoint_func(handler):
+            self.acquire_lock()
+            self._entrypoint = handler
+            self.release_lock()
+
+            return handler
+
+        return entrypoint_func
+
+    def add_subcommand(self, handler, description, name=None, **kwargs):
+        """Register a subcommand.
+
+        If name is not provided we use `handler.__name__`.
+        """
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before the with statement!')
+
+        if self._subparsers is None:
+            self.add_subparsers()
+
+        if not name:
+            name = handler.__name__
+
+        self.acquire_lock()
+        kwargs['help'] = description
+        self.subcommands_default[name] = self._subparsers_default.add_parser(name, **kwargs)
+        self.subcommands[name] = SubparserWrapper(self, name, self._subparsers.add_parser(name, **kwargs))
+        self.subcommands[name].set_defaults(entrypoint=handler)
+
+        if name not in self.__dict__:
+            self.__dict__[name] = self.subcommands[name]
+        else:
+            self.log.debug("Could not add subcommand '%s' to attributes, key already exists!", name)
+
+        self.release_lock()
+
+        return handler
+
+    def subcommand(self, description, **kwargs):
+        """Decorator to register a subcommand.
+        """
+
+        def subcommand_function(handler):
+            return self.add_subcommand(handler, description, **kwargs)
+
+        return subcommand_function
+
+    def setup_logging(self):
+        """Called by __enter__() to setup the logging configuration.
+        """
+        if len(logging.root.handlers) != 0:
+            # This is not a design decision. This is what I'm doing for now until I can examine and think about this situation in more detail.
+            raise RuntimeError('MILC should be the only system installing root log handlers!')
+
+        self.acquire_lock()
+
+        if self.config['general']['verbose']:
+            self.log_print_level = logging.DEBUG
+
+        self.log_file = self.config['general']['log_file'] or self.log_file
+        self.log_file_format = self.config['general']['log_file_fmt']
+        self.log_file_format = ANSIStrippingFormatter(self.config['general']['log_file_fmt'], self.config['general']['datetime_fmt'])
+        self.log_format = self.config['general']['log_fmt']
+
+        if self.config.general.color:
+            self.log_format = ANSIEmojiLoglevelFormatter(self.args.general_log_fmt, self.config.general.datetime_fmt)
+        else:
+            self.log_format = ANSIStrippingFormatter(self.args.general_log_fmt, self.config.general.datetime_fmt)
+
+        if self.log_file:
+            self.log_file_handler = logging.FileHandler(self.log_file, self.log_file_mode)
+            self.log_file_handler.setLevel(self.log_file_level)
+            self.log_file_handler.setFormatter(self.log_file_format)
+            logging.root.addHandler(self.log_file_handler)
+
+        if self.log_print:
+            self.log_print_handler = logging.StreamHandler(self.log_print_to)
+            self.log_print_handler.setLevel(self.log_print_level)
+            self.log_print_handler.setFormatter(self.log_format)
+            logging.root.addHandler(self.log_print_handler)
+
+        self.release_lock()
+
+    def __enter__(self):
+        if self._inside_context_manager:
+            self.log.debug('Warning: context manager was entered again. This usually means that self.__call__() was called before the with statement. You probably do not want to do that.')
+            return
+
+        self.acquire_lock()
+        self._inside_context_manager = True
+        self.release_lock()
+
+        colorama.init()
+        self.parse_args()
+        self.read_config()
+        self.setup_logging()
+
+        if self.config.general.save_config:
+            self.save_config()
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.acquire_lock()
+        self._inside_context_manager = False
+        self.release_lock()
+
+        if exc_type is not None and not isinstance(SystemExit(), exc_type):
+            print(exc_type)
+            logging.exception(exc_val)
+            exit(255)
+
+
+cli = MILC()
+
+if __name__ == '__main__':
+
+    @cli.argument('-c', '--comma', help='comma in output', default=True, action='store_boolean')
+    @cli.entrypoint('My useful CLI tool with subcommands.')
+    def main(cli):
+        comma = ',' if cli.config.general.comma else ''
+        cli.log.info('{bg_green}{fg_red}Hello%s World!', comma)
+
+    @cli.argument('-n', '--name', help='Name to greet', default='World')
+    @cli.subcommand('Description of hello subcommand here.')
+    def hello(cli):
+        comma = ',' if cli.config.general.comma else ''
+        cli.log.info('{fg_blue}Hello%s %s!', comma, cli.config.hello.name)
+
+    def goodbye(cli):
+        comma = ',' if cli.config.general.comma else ''
+        cli.log.info('{bg_red}Goodbye%s %s!', comma, cli.config.goodbye.name)
+
+    @cli.argument('-n', '--name', help='Name to greet', default='World')
+    @cli.subcommand('Think a bit before greeting the user.')
+    def thinking(cli):
+        comma = ',' if cli.config.general.comma else ''
+        spinner = cli.spinner(text='Just a moment...', spinner='earth')
+        spinner.start()
+        sleep(2)
+        spinner.stop()
+
+        with cli.spinner(text='Almost there!', spinner='moon'):
+            sleep(2)
+
+        cli.log.info('{fg_cyan}Hello%s %s!', comma, cli.config.thinking.name)
+
+    @cli.subcommand('Show off our ANSI colors.')
+    def pride(cli):
+        cli.echo('{bg_red}                    ')
+        cli.echo('{bg_lightred_ex}                    ')
+        cli.echo('{bg_lightyellow_ex}                    ')
+        cli.echo('{bg_green}                    ')
+        cli.echo('{bg_blue}                    ')
+        cli.echo('{bg_magenta}                    ')
+
+    # You can register subcommands using decorators as seen above, or using functions like like this:
+    cli.add_subcommand(goodbye, 'This will show up in --help output.')
+    cli.goodbye.add_argument('-n', '--name', help='Name to bid farewell to', default='World')
+
+    cli()  # Automatically picks between main(), hello() and goodbye()
+    print(sorted(ansi_colors.keys()))

lib/python/qmk/cli/compile/json.py

@@ -0,0 +1,44 @@
+"""Create a keymap directory from a configurator export.
+"""
+import json
+import os
+import sys
+import subprocess
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', help='Configurator JSON export')
+@cli.entrypoint('Compile a QMK Configurator export.')
+def main(cli):
+    """Compile a QMK Configurator export.
+
+    This command creates a new keymap from a configurator export, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        exit(1)
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+    cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+    cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    # Compile the keymap
+    command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)

lib/python/qmk/cli/doctor.py

@@ -0,0 +1,47 @@
+"""QMK Python Doctor
+
+Check up for QMK environment.
+"""
+import shutil
+import platform
+import os
+
+from milc import cli
+
+
+@cli.entrypoint('Basic QMK environment checks')
+def main(cli):
+    """Basic QMK environment checks.
+
+    This is currently very simple, it just checks that all the expected binaries are on your system.
+
+    TODO(unclaimed):
+        * [ ] Run the binaries to make sure they work
+        * [ ] Compile a trivial program with each compiler
+        * [ ] Check for udev entries on linux
+    """
+
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
+
+    cli.log.info('QMK Doctor is Checking your environment')
+
+    ok = True
+    for binary in binaries:
+        res = shutil.which(binary)
+        if res is None:
+            cli.log.error('{fg_red}QMK can\'t find ' + binary + ' in your path')
+            ok = False
+
+    OS = platform.system()
+    if OS == "Darwin":
+        cli.log.info("Detected {fg_cyan}macOS")
+    elif OS == "Linux":
+        cli.log.info("Detected {fg_cyan}linux")
+        test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
+        if os.system(test) == 0:
+            cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+    else:
+        cli.log.info("Assuming {fg_cyan}Windows")
+
+    if ok:
+        cli.log.info('{fg_green}QMK is ready to go')

lib/python/qmk/cli/hello.py

@@ -0,0 +1,13 @@
+"""QMK Python Hello World
+
+This is an example QMK CLI script.
+"""
+from milc import cli
+
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Hello World.')
+def main(cli):
+    """Log a friendly greeting.
+    """
+    cli.log.info('Hello, %s!', cli.config.general.name)

lib/python/qmk/cli/json/keymap.py

@@ -0,0 +1,54 @@
+"""Generate a keymap.c from a configurator export.
+"""
+import json
+import os
+import sys
+
+from milc import cli
+
+import qmk.keymap
+
+
+@cli.argument('-o', '--output', help='File to write to')
+@cli.argument('filename', help='Configurator JSON file')
+@cli.entrypoint('Create a keymap.c from a QMK Configurator export.')
+def main(cli):
+    """Generate a keymap.c from a configurator export.
+
+    This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        output_file = qmk.path.normpath(cli.args.output)
+        with open(output_file, 'w') as keymap_fd:
+            keymap_fd.write(keymap_c)
+
+        cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(keymap_c)

lib/python/qmk/errors.py

@@ -0,0 +1,6 @@
+class NoSuchKeyboardError(Exception):
+    """Raised when we can't find a keyboard/keymap directory.
+    """
+
+    def __init__(self, message):
+        self.message = message

lib/python/qmk/keymap.py

@@ -0,0 +1,100 @@
+"""Functions that help you work with QMK keymaps.
+"""
+import json
+import logging
+import os
+from traceback import format_exc
+
+import qmk.path
+from qmk.errors import NoSuchKeyboardError
+
+# The `keymap.c` template to use when a keyboard doesn't have its own
+DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
+
+/* THIS FILE WAS GENERATED!
+ *
+ * This file was generated by qmk-compile-json. You may or may not want to
+ * edit it directly.
+ */
+
+const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
+__KEYMAP_GOES_HERE__
+};
+"""
+
+
+def template(keyboard):
+    """Returns the `keymap.c` template for a keyboard.
+
+    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
+    text will be used instead of `DEFAULT_KEYMAP_C`.
+
+    Args:
+        keyboard
+            The keyboard to return a template for.
+    """
+    template_name = 'keyboards/%s/templates/keymap.c' % keyboard
+
+    if os.path.exists(template_name):
+        with open(template_name, 'r') as fd:
+            return fd.read()
+
+    return DEFAULT_KEYMAP_C
+
+
+def generate(keyboard, layout, layers):
+    """Returns a keymap.c for the specified keyboard, layout, and layers.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    layer_txt = []
+    for layer_num, layer in enumerate(layers):
+        if layer_num != 0:
+            layer_txt[-1] = layer_txt[-1] + ','
+        layer_keys = ', '.join(layer)
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+    keymap = '\n'.join(layer_txt)
+    keymap_c = template(keyboard, keymap)
+
+    return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap)
+
+
+def write(keyboard, keymap, layout, layers):
+    """Generate the `keymap.c` and write it to disk.
+
+    Returns the filename written to.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        keymap
+            The name of the keymap
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    keymap_c = generate(keyboard, layout, layers)
+    keymap_path = qmk.path.keymap(keyboard)
+    keymap_dir = os.path.join(keymap_path, keymap)
+    keymap_file = os.path.join(keymap_dir, 'keymap.c')
+
+    if not os.path.exists(keymap_dir):
+        os.makedirs(keymap_dir)
+
+    with open(keymap_file, 'w') as keymap_fd:
+        keymap_fd.write(keymap_c)
+
+    return keymap_file

lib/python/qmk/path.py

@@ -0,0 +1,32 @@
+"""Functions that help us work with files and folders.
+"""
+import os
+
+
+def keymap(keyboard):
+    """Locate the correct directory for storing a keymap.
+
+    Args:
+        keyboard
+            The name of the keyboard. Example: clueboard/66/rev3
+    """
+    for directory in ['.', '..', '../..', '../../..', '../../../..', '../../../../..']:
+        basepath = os.path.normpath(os.path.join('keyboards', keyboard, directory, 'keymaps'))
+
+        if os.path.exists(basepath):
+            return basepath
+
+    logging.error('Could not find keymaps directory!')
+    raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard)
+
+
+def normpath(path):
+    """Returns the fully resolved absolute path to a file.
+
+    This function will return the absolute path to a file as seen from the
+    directory the script was called from.
+    """
+    if path and path[0] == '/':
+        return os.path.normpath(path)
+
+    return os.path.normpath(os.path.join(os.environ['ORIG_CWD'], path))

requirements.txt

@@ -0,0 +1,5 @@
+# Python requirements
+# milc FIXME(skullydazed): Included in the repo for now.
+argcomplete
+colorama
+#halo

setup.cfg

@@ -0,0 +1,330 @@
+# Python settings for QMK
+
+[yapf]
+# Align closing bracket with visual indentation.
+align_closing_bracket_with_visual_indent=True
+
+# Allow dictionary keys to exist on multiple lines. For example:
+#
+#   x = {
+#       ('this is the first element of a tuple',
+#        'this is the second element of a tuple'):
+#            value,
+#   }
+allow_multiline_dictionary_keys=False
+
+# Allow lambdas to be formatted on more than one line.
+allow_multiline_lambdas=False
+
+# Allow splitting before a default / named assignment in an argument list.
+allow_split_before_default_or_named_assigns=True
+
+# Allow splits before the dictionary value.
+allow_split_before_dict_value=True
+
+#   Let spacing indicate operator precedence. For example:
+#
+#     a = 1 * 2 + 3 / 4
+#     b = 1 / 2 - 3 * 4
+#     c = (1 + 2) * (3 - 4)
+#     d = (1 - 2) / (3 + 4)
+#     e = 1 * 2 - 3
+#     f = 1 + 2 + 3 + 4
+#
+# will be formatted as follows to indicate precedence:
+#
+#     a = 1*2 + 3/4
+#     b = 1/2 - 3*4
+#     c = (1+2) * (3-4)
+#     d = (1-2) / (3+4)
+#     e = 1*2 - 3
+#     f = 1 + 2 + 3 + 4
+#
+arithmetic_precedence_indication=True
+
+# Number of blank lines surrounding top-level function and class
+# definitions.
+blank_lines_around_top_level_definition=2
+
+# Insert a blank line before a class-level docstring.
+blank_line_before_class_docstring=False
+
+# Insert a blank line before a module docstring.
+blank_line_before_module_docstring=False
+
+# Insert a blank line before a 'def' or 'class' immediately nested
+# within another 'def' or 'class'. For example:
+#
+#   class Foo:
+#                      # <------ this blank line
+#     def method():
+#       ...
+blank_line_before_nested_class_or_def=False
+
+# Do not split consecutive brackets. Only relevant when
+# dedent_closing_brackets is set. For example:
+#
+#    call_func_that_takes_a_dict(
+#        {
+#            'key1': 'value1',
+#            'key2': 'value2',
+#        }
+#    )
+#
+# would reformat to:
+#
+#    call_func_that_takes_a_dict({
+#        'key1': 'value1',
+#        'key2': 'value2',
+#    })
+coalesce_brackets=True
+
+# The column limit.
+column_limit=256
+
+# The style for continuation alignment. Possible values are:
+#
+# - SPACE: Use spaces for continuation alignment. This is default behavior.
+# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns
+#   (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation
+#   alignment.
+# - VALIGN-RIGHT: Vertically align continuation lines with indent
+#   characters. Slightly right (one more indent character) if cannot
+#   vertically align continuation lines with indent characters.
+#
+# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is
+# enabled.
+continuation_align_style=SPACE
+
+# Indent width used for line continuations.
+continuation_indent_width=4
+
+# Put closing brackets on a separate line, dedented, if the bracketed
+# expression can't fit in a single line. Applies to all kinds of brackets,
+# including function definitions and calls. For example:
+#
+#   config = {
+#       'key1': 'value1',
+#       'key2': 'value2',
+#   }        # <--- this bracket is dedented and on a separate line
+#
+#   time_series = self.remote_client.query_entity_counters(
+#       entity='dev3246.region1',
+#       key='dns.query_latency_tcp',
+#       transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
+#       start_ts=now()-timedelta(days=3),
+#       end_ts=now(),
+#   )        # <--- this bracket is dedented and on a separate line
+dedent_closing_brackets=True
+
+# Disable the heuristic which places each list element on a separate line
+# if the list is comma-terminated.
+disable_ending_comma_heuristic=False
+
+# Place each dictionary entry onto its own line.
+each_dict_entry_on_separate_line=True
+
+# The regex for an i18n comment. The presence of this comment stops
+# reformatting of that line, because the comments are required to be
+# next to the string they translate.
+i18n_comment=
+
+# The i18n function call names. The presence of this function stops
+# reformattting on that line, because the string it has cannot be moved
+# away from the i18n comment.
+i18n_function_call=
+
+# Indent blank lines.
+indent_blank_lines=False
+
+# Indent the dictionary value if it cannot fit on the same line as the
+# dictionary key. For example:
+#
+#   config = {
+#       'key1':
+#           'value1',
+#       'key2': value1 +
+#               value2,
+#   }
+indent_dictionary_value=True
+
+# The number of columns to use for indentation.
+indent_width=4
+
+# Join short lines into one line. E.g., single line 'if' statements.
+join_multiple_lines=False
+
+# Do not include spaces around selected binary operators. For example:
+#
+#   1 + 2 * 3 - 4 / 5
+#
+# will be formatted as follows when configured with "*,/":
+#
+#   1 + 2*3 - 4/5
+no_spaces_around_selected_binary_operators=
+
+# Use spaces around default or named assigns.
+spaces_around_default_or_named_assign=False
+
+# Use spaces around the power operator.
+spaces_around_power_operator=False
+
+# The number of spaces required before a trailing comment.
+# This can be a single value (representing the number of spaces
+# before each trailing comment) or list of values (representing
+# alignment column values; trailing comments within a block will
+# be aligned to the first column value that is greater than the maximum
+# line length within the block). For example:
+#
+# With spaces_before_comment=5:
+#
+#   1 + 1 # Adding values
+#
+# will be formatted as:
+#
+#   1 + 1     # Adding values <-- 5 spaces between the end of the statement and comment
+#
+# With spaces_before_comment=15, 20:
+#
+#   1 + 1 # Adding values
+#   two + two # More adding
+#
+#   longer_statement # This is a longer statement
+#   short # This is a shorter statement
+#
+#   a_very_long_statement_that_extends_beyond_the_final_column # Comment
+#   short # This is a shorter statement
+#
+# will be formatted as:
+#
+#   1 + 1          # Adding values <-- end of line comments in block aligned to col 15
+#   two + two      # More adding
+#
+#   longer_statement    # This is a longer statement <-- end of line comments in block aligned to col 20
+#   short               # This is a shorter statement
+#
+#   a_very_long_statement_that_extends_beyond_the_final_column  # Comment <-- the end of line comments are aligned based on the line length
+#   short                                                       # This is a shorter statement
+#
+spaces_before_comment=2
+
+# Insert a space between the ending comma and closing bracket of a list,
+# etc.
+space_between_ending_comma_and_closing_bracket=False
+
+# Split before arguments
+split_all_comma_separated_values=False
+
+# Split before arguments if the argument list is terminated by a
+# comma.
+split_arguments_when_comma_terminated=True
+
+# Set to True to prefer splitting before '+', '-', '*', '/', '//', or '@'
+# rather than after.
+split_before_arithmetic_operator=False
+
+# Set to True to prefer splitting before '&', '|' or '^' rather than
+# after.
+split_before_bitwise_operator=True
+
+# Split before the closing bracket if a list or dict literal doesn't fit on
+# a single line.
+split_before_closing_bracket=True
+
+# Split before a dictionary or set generator (comp_for). For example, note
+# the split before the 'for':
+#
+#   foo = {
+#       variable: 'Hello world, have a nice day!'
+#       for variable in bar if variable != 42
+#   }
+split_before_dict_set_generator=True
+
+# Split before the '.' if we need to split a longer expression:
+#
+#   foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d))
+#
+# would reformat to something like:
+#
+#   foo = ('This is a really long string: {}, {}, {}, {}'
+#          .format(a, b, c, d))
+split_before_dot=False
+
+# Split after the opening paren which surrounds an expression if it doesn't
+# fit on a single line.
+split_before_expression_after_opening_paren=False
+
+# If an argument / parameter list is going to be split, then split before
+# the first argument.
+split_before_first_argument=False
+
+# Set to True to prefer splitting before 'and' or 'or' rather than
+# after.
+split_before_logical_operator=False
+
+# Split named assignments onto individual lines.
+split_before_named_assigns=True
+
+# Set to True to split list comprehensions and generators that have
+# non-trivial expressions and multiple clauses before each of these
+# clauses. For example:
+#
+#   result = [
+#       a_long_var + 100 for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+#
+# would reformat to something like:
+#
+#   result = [
+#       a_long_var + 100
+#       for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+split_complex_comprehension=True
+
+# The penalty for splitting right after the opening bracket.
+split_penalty_after_opening_bracket=300
+
+# The penalty for splitting the line after a unary operator.
+split_penalty_after_unary_operator=10000
+
+# The penalty of splitting the line around the '+', '-', '*', '/', '//',
+# ``%``, and '@' operators.
+split_penalty_arithmetic_operator=300
+
+# The penalty for splitting right before an if expression.
+split_penalty_before_if_expr=0
+
+# The penalty of splitting the line around the '&', '|', and '^'
+# operators.
+split_penalty_bitwise_operator=300
+
+# The penalty for splitting a list comprehension or generator
+# expression.
+split_penalty_comprehension=80
+
+# The penalty for characters over the column limit.
+split_penalty_excess_character=7000
+
+# The penalty incurred by adding a line split to the unwrapped line. The
+# more line splits added the higher the penalty.
+split_penalty_for_added_line_split=30
+
+# The penalty of splitting a list of "import as" names. For example:
+#
+#   from a_very_long_or_indented_module_name_yada_yad import (long_argument_1,
+#                                                             long_argument_2,
+#                                                             long_argument_3)
+#
+# would reformat to something like:
+#
+#   from a_very_long_or_indented_module_name_yada_yad import (
+#       long_argument_1, long_argument_2, long_argument_3)
+split_penalty_import_names=0
+
+# The penalty of splitting the line around the 'and' and 'or'
+# operators.
+split_penalty_logical_operator=300
+
+# Use the Tab character for indentation.
+use_tabs=False
+

util/freebsd_install.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+util_dir=$(dirname "$0")
 pkg update
 pkg install -y \
 	git \
@@ -17,3 +18,4 @@ pkg install -y \
 	arm-none-eabi-newlib \
 	diffutils \
 	python3
+pip3 install -r ${util_dir}/../requirements.txt

util/linux_install.sh

@@ -8,6 +8,8 @@ SLACKWARE_WARNING="You will need the following packages from slackbuilds.org:\n\
 
 SOLUS_INFO="Your tools are now installed. To start using them, open new terminal or source these scripts:\n\t/usr/share/defaults/etc/profile.d/50-arm-toolchain-path.sh\n\t/usr/share/defaults/etc/profile.d/50-avr-toolchain-path.sh"
 
+util_dir=$(dirname "$0")
+
 if grep ID /etc/os-release | grep -qE "fedora"; then
 	sudo dnf install \
 		arm-none-eabi-binutils-cs \
@@ -183,3 +185,6 @@ else
 	echo
 	echo "https://docs.qmk.fm/#/contributing"
 fi
+
+# Global install tasks
+pip3 install -r ${util_dir}/../requirements.txt

util/macos_install.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+util_dir=$(dirname "$0")
+
 if ! brew --version 2>&1 > /dev/null; then
 	echo "Error! Homebrew not installed or broken!"
 	echo -n "Would you like to install homebrew now? [y/n] "
@@ -24,3 +26,4 @@ brew tap PX4/homebrew-px4
 brew update
 brew install avr-gcc@8 gcc-arm-none-eabi dfu-programmer avrdude dfu-util python3
 brew link --force avr-gcc@8
+pip3 install -r ${util_dir}/../requirements.txt

util/msys2_install.sh

@@ -5,6 +5,7 @@ download_dir=~/qmk_utils
 avrtools=avr8-gnu-toolchain
 armtools=gcc-arm-none-eabi
 installflip=false
+util_dir=$(dirname "$0")
 
 echo "Installing dependencies needed for the installation (quazip)"
 pacman --needed -S base-devel mingw-w64-x86_64-toolchain msys/git msys/p7zip msys/python3 msys/unzip
@@ -92,6 +93,8 @@ else
 fi
 popd
 
+pip3 install -r ${util_dir}/../requirements.txt
+
 cp -f "$dir/activate_msys2.sh" "$download_dir/"
 
 if grep "^source ~/qmk_utils/activate_msys2.sh$" ~/.bashrc

util/wsl_install.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+util_dir=$(dirname "$0")
+dir=$(cd -P -- "$util_dir" && pwd -P)
 pushd "$dir";
 
 if [[ $dir != /mnt/* ]];
@@ -28,6 +29,8 @@ download_dir=wsl_downloaded
 
 source "$dir/win_shared_install.sh"
 
+pip3 install -r ${util_dir}/../requirements.txt
+
 pushd "$download_dir"
 while true; do
     echo