Mucking about with forcing the use of a venv at runtime.

Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.0.0 to 5.1.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](https://github.com/docker/build-push-action/compare/v5.0.0...v5.1.0)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.bumpversion.cfg

@@ -0,0 +1,12 @@
+[bumpversion]
+current_version = 1.1.5
+commit = True
+tag = True
+tag_name = {new_version}
+message = New release: {current_version} → {new_version}
+
+[bumpversion:file:.bumpversion.cfg]
+
+[bumpversion:file:qmk_cli/__init__.py]
+
+[bumpversion:file:setup.cfg]

.github/dependabot.yml

@@ -0,0 +1,17 @@
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/"
+    schedule:
+      interval: "daily"
+    reviewers:
+      - "qmk/collaborators"
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+    reviewers:
+      - "qmk/collaborators"

.github/workflows/cli_setup.yml

@@ -7,65 +7,70 @@ on:
   pull_request:
 
 jobs:
+  test_cli_base_container:
+    runs-on: ubuntu-latest
+    container: ghcr.io/qmk/qmk_base_container
+
+    env:
+      QMK_HOME: ~/qmk_firmware
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install dependencies
+      run: apt-get update && apt-get install -y python3-venv
+
+    - name: Run ci_tests
+      run: ./ci_tests -a
+
   test_cli_linux_macos:
+    needs: test_cli_base_container
+
     runs-on: ${{ matrix.os }}
     env:
       QMK_HOME: ~/qmk_firmware
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install Python dependencies
-      run: |
-        python3 -m pip install wheel
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
-
-    - name: Install QMK CLI from source
-      run: |
-        python3 -m pip install -r requirements.txt
-        python3 setup.py sdist bdist_wheel
-        cd ..
-        python3 -m pip install --force-reinstall --no-index --no-deps --prefix=~/.local --find-links qmk_cli/dist qmk
-
-    - name: Run qmk setup -y
-      run: qmk setup -y
+    - name: Run ci_tests
+      run: ./ci_tests -a
 
   test_cli_win:
+    needs: test_cli_base_container
+
     runs-on: windows-latest
     env:
       QMK_HOME: $HOME/qmk_firmware
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
 
     - name: (MSYS2) Setup and install dependencies
       uses: msys2/setup-msys2@v2
       with:
         update: true
-        install: git mingw-w64-x86_64-toolchain mingw-w64-x86_64-python-pip
+        install: git mingw-w64-x86_64-toolchain mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-pillow mingw-w64-x86_64-rust
 
     - name: (MSYS2) Install Python dependencies
       shell: msys2 {0}
       run: |
-        python3 -m pip install wheel
-
+        python3 -m pip install -U setuptools wheel
     - name: (MSYS2) Install QMK CLI from source
       shell: msys2 {0}
       run: |
-        python3 -m pip install -r requirements.txt
-        python3 setup.py sdist bdist_wheel
-        cd ..
-        python3 -m pip install --force-reinstall --no-index --no-deps --find-links qmk_cli/dist qmk
-
+        python3 -m pip install -U build
+        python3 -m build
+        python3 -m pip install dist/qmk-*.tar.gz
     - name: (MSYS2) Run qmk setup -y
       shell: msys2 {0}
       run: qmk setup -y

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '22 18 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-republish.yml

@@ -0,0 +1,56 @@
+name: Rebuild Docker Image
+
+on:
+  workflow_dispatch:
+
+jobs:
+  redeploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.7'
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: 'Get Previous tag'
+      id: previoustag
+      uses: "WyriHaximus/github-action-get-previous-tag@v1"
+
+    - name: Download previous artifact
+      run: |
+        pip download -d "${GITHUB_WORKSPACE}/dist" qmk=="${{ steps.previoustag.outputs.tag }}"
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+    - name: Build and Push to Docker Hub
+      uses: docker/build-push-action@v5.1.0
+      with:
+        context: .
+        push: true
+        platforms: linux/amd64,linux/arm64
+        tags: |
+          ghcr.io/qmk/qmk_cli:latest
+          qmkfm/qmk_cli:latest

.github/workflows/python-publish.yml

@@ -17,31 +17,73 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: '3.7'
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - uses: actions/checkout@v4
+
+    - name: Run ci_tests
+      run: ./ci_tests
+
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel 
+        python3 -m pip install --upgrade pip
+        pip install setuptools wheel
         pip install -r requirements-dev.txt
+
     - name: Bump version
       run: |
         git config --local user.email "hello@qmk.fm"
         git config --local user.name "QMK Bot"
         bumpversion ${{ github.event.inputs.version_part }}
+
     - name: Push changes
       uses: ad-m/github-push-action@master
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         branch: master
         tags: true
+
     - name: Build and publish
       env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_APITOKEN }}
       run: |
-        python setup.py sdist bdist_wheel
+        python3 -m build
         twine upload dist/*
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+    - name: Build and Push to Docker Hub
+      uses: docker/build-push-action@v5.1.0
+      with:
+        context: .
+        push: true
+        platforms: linux/amd64,linux/arm64
+        tags: |
+          ghcr.io/qmk/qmk_cli:latest
+          qmkfm/qmk_cli:latest
+
+    - name: Trigger OS package builds
+      run: ./trigger_packages
+      env:
+        QMK_BOT_TOKEN: ${{ secrets.QMK_BOT_TOKEN }}

Dockerfile

@@ -0,0 +1,13 @@
+FROM ghcr.io/qmk/qmk_base_container:latest
+
+# Copy package in
+ADD dist /tmp/dist
+
+# Install python packages
+RUN python3 -m pip uninstall -y qmk || true
+RUN python3 -m pip install --upgrade pip setuptools wheel nose2 && \
+    python3 -m pip install /tmp/dist/qmk-*.whl && \
+    rm -rf /tmp/dist
+
+# Set the default location for qmk_firmware
+ENV QMK_HOME /qmk_firmware

README.md

@@ -8,17 +8,52 @@ A program to help users work with [QMK Firmware](https://qmk.fm/).
 * Interact with your qmk_firmware tree from any location
 * Use `qmk clone` to pull down anyone's `qmk_firmware` fork
 * Setup your build environment with `qmk setup`
+* Use `qmk console` to get debugging information from your keyboard(s)
 * Check that your environment is correctly setup with `qmk doctor`
 * Integrates with qmk_firmware for additional functionality:
     * `qmk compile`
-    * `qmk console`
+    * `qmk info`
     * `qmk flash`
     * `qmk lint`
     * ...and many more!
 
+# Packages
+
+We provide "install and go" packages for many Operating Systems.
+
+## Linux
+
+Packages for several distributions available here: https://github.com/qmk/qmk_fpm
+
+## macOS
+
+Using [Homebrew](https://brew.sh):
+
+    brew install qmk/qmk/qmk
+
+## Windows
+
+Download our custom MSYS2 installer here: https://msys.qmk.fm/
+
 # Quickstart
 
 * `python3 -m pip install qmk`
 * `qmk setup`
 
+# Building
+
+We follow PEP517, you can install using [build](https://pypi.org/project/build/):
+
+Setup:
+
+    python3 -m pip install build
+
+Build:
+
+    python3 -m build
+
+You can read more about working with PEP517 packages in the [Python Packaging User Guide](https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-your-project).
+
+# Documentation
+
 Full documentation: <https://docs.qmk.fm/#/tutorial>

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.0.x   | :white_check_mark: |
+
+## Reporting a Vulnerability
+
+* [Open an Issue](https://github.com/qmk/qmk_cli/issues/new?assignees=&labels=bug&template=bug_report.md&title=%5BBug%5D+)

ci_tests

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -e
+#set -x
+
+ADVANCED=false
+TMPDIR=$(mktemp -d)
+QMK_HOME="$TMPDIR/qmk_firmware"
+export QMK_HOME
+
+for arg in $@; do
+    if [ "$arg" = "-a" ]; then
+        ADVANCED=true
+    fi
+done
+
+if [ $ADVANCED = true ]; then
+    echo "*** Running in advanced mode with tmp files in $TMPDIR"
+else
+    echo "*** Running in basic mode with tmp files in $TMPDIR"
+fi
+
+# Setup our virtualenv
+if [ -e .ci_venv ]; then
+    rm -rf .ci_venv
+fi
+
+python3 -m venv .ci_venv
+source .ci_venv/bin/activate
+
+# Install dependencies
+python3 -m pip install -U pip wheel
+python3 -m pip install .
+python3 -m pip install -r requirements-dev.txt
+
+# Ensure that qmk works
+echo "*** Testing 'qmk clone -h'"
+qmk clone -h
+#echo "*** Testing 'qmk config -a'" # Test disabled as `milc` at least 1.6.8+ returns False and thus non-zero exit code
+#qmk config -a
+echo "*** Testing 'qmk setup -n'"
+qmk setup -n
+
+echo
+echo "*** Basic tests completed successfully!"
+
+# Run advanced test if requested
+if [ $ADVANCED = true ]; then
+    echo
+    echo "*** Testing 'qmk setup -y'"
+    qmk setup -y
+
+    echo
+    echo "*** Advanced tests completed successfully!"
+fi
+
+# Cleanup
+deactivate
+rm -rf .ci_venv $TMPDIR

pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

qmk_cli/__init__.py

@@ -1,3 +1,3 @@
 """A program to help you work with qmk_firmware."""
 
-__version__ = '0.0.52'
+__version__ = '1.1.5'

qmk_cli/script_qmk.py

@@ -10,6 +10,8 @@ import sys
 from platform import platform
 from traceback import print_exc
 
+from . import switch_to_venv # noqa, intentionally unused
+
 import milc
 
 from . import __version__

qmk_cli/subcommands/__init__.py

@@ -3,5 +3,7 @@
 We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup.
 """
 from . import clone  # noqa
+from . import console  # noqa
 from . import env  # noqa
 from . import setup  # noqa
+from . import pip # noqa

qmk_cli/subcommands/clone.py

@@ -11,20 +11,21 @@ default_fork = 'qmk/' + default_repo
 default_branch = 'master'
 
 
-@cli.argument('--baseurl', default='https://github.com', help='The URL all git operations start from (Default: https://github.com)')
-@cli.argument('-b', '--branch', default=default_branch, help='The branch to clone. Default: %s' % default_branch)
-@cli.argument('destination', default=None, nargs='?', help='The directory to clone to. Default: (current directory)')
-@cli.argument('fork', default=default_fork, nargs='?', help='The qmk_firmware fork to clone. Default: %s' % default_fork)
+@cli.argument('--baseurl', arg_only=True, default='https://github.com', help='The URL all git operations start from (Default: https://github.com)')
+@cli.argument('-b', '--branch', arg_only=True, default=default_branch, help='The branch to clone. Default: %s' % default_branch)
+@cli.argument('destination', arg_only=True, default=None, nargs='?', help='The directory to clone to. Default: (current directory)')
+@cli.argument('fork', arg_only=True, default=default_fork, nargs='?', help='The qmk_firmware fork to clone. Default: %s' % default_fork)
 @cli.subcommand('Clone a qmk_firmware fork.')
 def clone(cli):
     if not cli.args.destination:
         cli.args.destination = os.path.join(os.environ['ORIG_CWD'], default_fork.split('/')[-1])
 
     qmk_firmware = Path(cli.args.destination)
-    git_url = '/'.join((cli.config.clone.baseurl, cli.args.fork))
+    git_url = '/'.join((cli.args.baseurl, cli.args.fork))
 
-    if qmk_firmware.exists():
+    # Exists (but not an empty dir)
+    if qmk_firmware.exists() and any(qmk_firmware.iterdir()):
         cli.log.error('Destination already exists: %s', cli.args.destination)
         exit(1)
 
-    return git_clone(git_url, cli.args.destination, cli.config.clone.branch)
+    return git_clone(git_url, cli.args.destination, cli.args.branch)

qmk_cli/subcommands/console.py

@@ -0,0 +1,360 @@
+"""Acquire debugging information from usb hid devices
+
+cli implementation of https://www.pjrc.com/teensy/hid_listen.html
+"""
+from functools import lru_cache
+from pathlib import Path
+from platform import platform
+from threading import Thread
+from time import sleep, strftime
+
+from milc import cli
+from milc.questions import yesno
+
+LOG_COLOR = {
+    'next': 0,
+    'colors': [
+        '{fg_blue}',
+        '{fg_cyan}',
+        '{fg_green}',
+        '{fg_magenta}',
+        '{fg_red}',
+        '{fg_yellow}',
+    ],
+}
+
+KNOWN_BOOTLOADERS = {
+    # VID  ,  PID
+    ('03EB', '2045'): 'lufa-ms: LUFA Mass Storage Bootloader',
+    ('03EB', '2067'): 'qmk-hid: HID Bootloader',
+    ('03EB', '2FEF'): 'atmel-dfu: ATmega16U2',
+    ('03EB', '2FF0'): 'atmel-dfu: ATmega32U2',
+    ('03EB', '2FF3'): 'atmel-dfu: ATmega16U4',
+    ('03EB', '2FF4'): 'atmel-dfu: ATmega32U4',
+    ('03EB', '2FF9'): 'atmel-dfu: AT90USB64',
+    ('03EB', '2FFA'): 'atmel-dfu: AT90USB162',
+    ('03EB', '2FFB'): 'atmel-dfu: AT90USB128',
+    ('03EB', '6124'): 'Microchip SAM-BA',
+    ('0483', 'DF11'): 'stm32-dfu: STM32 BOOTLOADER',
+    ('16C0', '05DC'): 'usbasploader: USBaspLoader',
+    ('16C0', '05DF'): 'bootloadhid: HIDBoot',
+    ('16C0', '0478'): 'halfkay: Teensy Halfkay',
+    ('1B4F', '9203'): 'caterina: Pro Micro 3.3V',
+    ('1B4F', '9205'): 'caterina: Pro Micro 5V',
+    ('1B4F', '9207'): 'caterina: LilyPadUSB',
+    ('1C11', 'B007'): 'kiibohd: Kiibohd DFU Bootloader',
+    ('1EAF', '0003'): 'stm32duino: Maple 003',
+    ('1FFB', '0101'): 'caterina: Pololu A-Star 32U4 Bootloader',
+    ('2341', '0036'): 'caterina: Arduino Leonardo',
+    ('2341', '0037'): 'caterina: Arduino Micro',
+    ('239A', '000C'): 'caterina: Adafruit Feather 32U4',
+    ('239A', '000D'): 'caterina: Adafruit ItsyBitsy 32U4 3v',
+    ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v',
+    ('2A03', '0036'): 'caterina: Arduino Leonardo',
+    ('2A03', '0037'): 'caterina: Arduino Micro',
+    ('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode',
+    ('342D', 'DFA0'): 'wb32-dfu: WB32 Device in DFU Mode'
+}
+
+
+def install_deps():
+    """Install the necessary dependencies for qmk console.
+    """
+    this_platform = platform().lower()
+
+    if 'darwin' in this_platform or 'macos' in this_platform:
+        command = ['brew', 'install', 'hidapi']
+    elif 'linux' in this_platform:
+        command = ['sudo', 'apt', 'install', '-y', 'libhidapi-hidraw0', 'libusb-dev']
+    elif 'windows' in this_platform:
+        command = ['pacboy', 'sync', '--needed', '--noconfirm', '--disable-download-timeout', 'hidapi:x']
+    else:
+        cli.log.error('Unsupported platform: %s', this_platform)
+
+    if yesno("Would you like to run `%s` to install the necessary package?", ' '.join(command)):
+        cli.run(command, capture_output=False)
+        return True
+
+
+@lru_cache(maxsize=0)
+def import_usb_core():
+    """Attempts to import the usb.core module.
+    """
+    try:
+        import usb.core
+        return usb.core
+
+    except ImportError as e:
+        cli.log.error('Could not import usb.core: %s', e)
+
+        if install_deps():
+            return import_usb_core()
+
+        raise
+
+
+def import_hid():
+    """Attempts to import the hid module.
+    """
+    try:
+        import hid
+        return hid
+
+    except ImportError as e:
+        cli.log.error('Could not import hid: %s', e)
+
+        if install_deps():
+            return import_hid()
+
+        raise
+
+
+class MonitorDevice(object):
+    def __init__(self, hid_device, numeric):
+        self.hid = import_hid()
+        self.hid_device = hid_device
+        self.numeric = numeric
+        self.device = self.hid.Device(path=hid_device['path'])
+        self.current_line = ''
+
+        cli.log.info('Console Connected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', hid_device)
+
+    def read(self, size, encoding='ascii', timeout=1):
+        """Read size bytes from the device.
+        """
+        return self.device.read(size, timeout).decode(encoding)
+
+    def read_line(self):
+        """Read from the device's console until we get a \n.
+        """
+        while '\n' not in self.current_line:
+            self.current_line += self.read(32).replace('\x00', '')
+
+        lines = self.current_line.split('\n', 1)
+        self.current_line = lines[1]
+
+        return lines[0]
+
+    def run_forever(self):
+        while True:
+            try:
+                message = {**self.hid_device, 'text': self.read_line()}
+                identifier = (int2hex(message['vendor_id']), int2hex(message['product_id'])) if self.numeric else (message['manufacturer_string'], message['product_string'])
+                message['identifier'] = ':'.join(identifier)
+                message['ts'] = '{style_dim}{fg_green}%s{style_reset_all} ' % (strftime(cli.config.general.datetime_fmt),) if cli.args.timestamp else ''
+
+                cli.echo('%s', '%(ts)s%(color)s%(identifier)s:%(index)d{style_reset_all}: %(text)s' % message)
+
+            except self.hid.HIDException:
+                break
+
+
+class FindDevices(object):
+    def __init__(self, vid, pid, index, numeric):
+        self.hid = import_hid()
+        self.vid = vid
+        self.pid = pid
+        self.index = index
+        self.numeric = numeric
+
+    def run_forever(self):
+        """Process messages from our queue in a loop.
+        """
+        live_devices = {}
+        live_bootloaders = {}
+
+        while True:
+            try:
+                for device in list(live_devices):
+                    if not live_devices[device]['thread'].is_alive():
+                        cli.log.info('Console Disconnected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', live_devices[device])
+                        del live_devices[device]
+
+                for device in self.find_devices():
+                    if device['path'] not in live_devices:
+                        device['color'] = LOG_COLOR['colors'][LOG_COLOR['next']]
+                        LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors'])
+                        live_devices[device['path']] = device
+
+                        try:
+                            monitor = MonitorDevice(device, self.numeric)
+                            device['thread'] = Thread(target=monitor.run_forever, daemon=True)
+
+                            device['thread'].start()
+                        except Exception as e:
+                            device['e'] = e
+                            device['e_name'] = e.__class__.__name__
+                            cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s:%(vendor_id)04X:%(product_id)04X:%(index)d): %(e_name)s: %(e)s", device)
+                            if cli.config.general.verbose:
+                                cli.log.exception(e)
+                            del live_devices[device['path']]
+
+                if cli.args.bootloaders:
+                    for device in self.find_bootloaders():
+                        if device.address in live_bootloaders:
+                            live_bootloaders[device.address]._qmk_found = True
+                        else:
+                            name = KNOWN_BOOTLOADERS[(int2hex(device.idVendor), int2hex(device.idProduct))]
+                            cli.log.info('Bootloader Connected: {style_bright}{fg_magenta}%s', name)
+                            device._qmk_found = True
+                            live_bootloaders[device.address] = device
+
+                    for device in list(live_bootloaders):
+                        if live_bootloaders[device]._qmk_found:
+                            live_bootloaders[device]._qmk_found = False
+                        else:
+                            name = KNOWN_BOOTLOADERS[(int2hex(live_bootloaders[device].idVendor), int2hex(live_bootloaders[device].idProduct))]
+                            cli.log.info('Bootloader Disconnected: {style_bright}{fg_magenta}%s', name)
+                            del live_bootloaders[device]
+
+                sleep(.1)
+
+            except KeyboardInterrupt:
+                break
+
+    def is_bootloader(self, hid_device):
+        """Returns true if the device in question matches a known bootloader vid/pid.
+        """
+        return (int2hex(hid_device.idVendor), int2hex(hid_device.idProduct)) in KNOWN_BOOTLOADERS
+
+    def is_console_hid(self, hid_device):
+        """Returns true when the usage page indicates it's a teensy-style console.
+        """
+        return hid_device['usage_page'] == 0xFF31 and hid_device['usage'] == 0x0074
+
+    def is_filtered_device(self, hid_device):
+        """Returns True if the device should be included in the list of available consoles.
+        """
+        return int2hex(hid_device['vendor_id']) == self.vid and int2hex(hid_device['product_id']) == self.pid
+
+    def find_devices_by_report(self, hid_devices):
+        """Returns a list of available teensy-style consoles by doing a brute-force search.
+
+        Some versions of linux don't report usage and usage_page. In that case we fallback to reading the report (possibly inaccurately) ourselves.
+        """
+        devices = []
+
+        for device in hid_devices:
+            path = device['path'].decode('utf-8')
+
+            if path.startswith('/dev/hidraw'):
+                number = path[11:]
+                report = Path(f'/sys/class/hidraw/hidraw{number}/device/report_descriptor')
+
+                if report.exists():
+                    rp = report.read_bytes()
+
+                    if rp[1] == 0x31 and rp[3] == 0x09:
+                        devices.append(device)
+
+        return devices
+
+    def find_bootloaders(self):
+        """Returns a list of available bootloader devices.
+        """
+        import usb.core
+
+        return list(filter(self.is_bootloader, usb.core.find(find_all=True)))
+
+    def find_devices(self):
+        """Returns a list of available teensy-style consoles.
+        """
+        hid_devices = self.hid.enumerate()
+        devices = list(filter(self.is_console_hid, hid_devices))
+
+        if not devices:
+            devices = self.find_devices_by_report(hid_devices)
+
+        if self.vid and self.pid:
+            devices = list(filter(self.is_filtered_device, devices))
+
+        # Add index numbers
+        device_index = {}
+        for device in devices:
+            id = ':'.join((int2hex(device['vendor_id']), int2hex(device['product_id'])))
+
+            if id not in device_index:
+                device_index[id] = 0
+
+            device_index[id] += 1
+            device['index'] = device_index[id]
+
+        return devices
+
+
+def int2hex(number):
+    """Returns a string representation of the number as hex.
+    """
+    return "%04X" % number
+
+
+def list_devices(device_finder):
+    """Show the user a nicely formatted list of devices.
+    """
+    devices = device_finder.find_devices()
+
+    if devices:
+        cli.log.info('Available devices:')
+        for dev in devices:
+            color = LOG_COLOR['colors'][LOG_COLOR['next']]
+            LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors'])
+            cli.log.info("\t%s%s:%s:%d{style_reset_all}\t%s %s", color, int2hex(dev['vendor_id']), int2hex(dev['product_id']), dev['index'], dev['manufacturer_string'], dev['product_string'])
+
+    if cli.args.bootloaders:
+        bootloaders = device_finder.find_bootloaders()
+
+        if bootloaders:
+            cli.log.info('Available Bootloaders:')
+
+            for dev in bootloaders:
+                cli.log.info("\t%s:%s\t%s", int2hex(dev.idVendor), int2hex(dev.idProduct), KNOWN_BOOTLOADERS[(int2hex(dev.idVendor), int2hex(dev.idProduct))])
+
+
+@cli.argument('--bootloaders', arg_only=True, default=True, action='store_boolean', help='displaying bootloaders.')
+@cli.argument('-d', '--device', help='Device to select - uses format <pid>:<vid>[:<index>].')
+@cli.argument('-l', '--list', arg_only=True, action='store_true', help='List available hid_listen devices.')
+@cli.argument('-n', '--numeric', arg_only=True, action='store_true', help='Show VID/PID instead of names.')
+@cli.argument('-t', '--timestamp', arg_only=True, action='store_true', help='Print the timestamp for received messages as well.')
+@cli.argument('-w', '--wait', type=int, default=1, help="How many seconds to wait between checks (Default: 1)")
+@cli.subcommand('Acquire debugging information from usb hid devices.')
+def console(cli):
+    """Acquire debugging information from usb hid devices
+    """
+    vid = None
+    pid = None
+    index = 1
+
+    if cli.config.console.device:
+        device = cli.config.console.device.split(':')
+
+        if len(device) == 2:
+            vid, pid = device
+
+        elif len(device) == 3:
+            vid, pid, index = device
+
+            if not index.isdigit():
+                cli.log.error('Device index must be a number! Got "%s" instead.', index)
+                exit(1)
+
+            index = int(index)
+
+            if index < 1:
+                cli.log.error('Device index must be greater than 0! Got %s', index)
+                exit(1)
+
+        else:
+            cli.log.error('Invalid format for device, expected "<pid>:<vid>[:<index>]" but got "%s".', cli.config.console.device)
+            cli.print_help()
+            exit(1)
+
+        vid = vid.upper()
+        pid = pid.upper()
+
+    device_finder = FindDevices(vid, pid, index, cli.args.numeric)
+
+    if cli.args.list:
+        return list_devices(device_finder)
+
+    print('Looking for devices...', flush=True)
+    device_finder.run_forever()

qmk_cli/subcommands/env.py

@@ -7,7 +7,7 @@ from milc import cli
 from qmk_cli.helpers import is_qmk_firmware
 
 
-@cli.argument('var', default=None, nargs='?', help='Optional variable to query')
+@cli.argument('var', arg_only=True, default=None, nargs='?', help='Optional variable to query')
 @cli.subcommand('Prints environment information.')
 def env(cli):
     home = os.environ.get('QMK_HOME', "")

qmk_cli/subcommands/pip.py

@@ -0,0 +1,72 @@
+"""Installs python dependencies.
+"""
+import shlex
+import shutil
+import subprocess
+from milc import cli
+
+from qmk_cli.helpers import find_qmk_firmware, is_qmk_firmware
+
+
+def _run_pip(pip_args, dry_run=False, upgrade=False):
+    pip_exe = ['pip']
+    if cli.config.user.pip_exe:
+        pip_exe = shlex.split(cli.config.user.pip_exe)
+    pip_command = [*pip_exe, *pip_args]
+    pip_command[0] = shutil.which(pip_command[0])
+    if shutil.which(pip_command[0]) is None:
+        pip_command[0] = 'pip' # Hope falling back to unqualified path works
+
+    if dry_run:
+        print(" ".join(shlex.quote(str(s)) for s in pip_command))
+        return True
+
+    try:
+        with subprocess.Popen(pip_command, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, encoding='utf-8') as p:
+            for line in p.stdout:
+                print(line, end='')
+    except Exception as e:
+        pip_cmd = ' '.join([s.replace(' ', r'\ ') for s in pip_command])
+        cli.log.error("Could not run '%s': %s: %s", pip_cmd, e.__class__.__name__, e)
+        return False
+
+    if p.returncode == 0:
+        return True
+    return False
+
+
+def install_qmk_python_deps(upgrade=False, developer=False, dry_run=False):
+    qmk_firmware_dir = find_qmk_firmware()
+    if not is_qmk_firmware(qmk_firmware_dir):
+        cli.log.error('Could not find {fg_cyan}qmk_firmware{fg_reset}!')
+        return False
+
+    reqs_path = qmk_firmware_dir / 'requirements.txt' if not developer else qmk_firmware_dir / 'requirements-dev.txt'
+
+    pip_args = ['install', '-r', reqs_path]
+    if upgrade:
+        pip_args.append('--upgrade')
+
+    if _run_pip(pip_args, dry_run=dry_run):
+        if not dry_run:
+            cli.log.info('Successfully installed python dependencies!')
+        return True
+    else:
+        if not dry_run:
+            cli.log.error('Failed to install python dependencies!')
+        return False
+
+
+@cli.argument('-u', '--upgrade', arg_only=True, action='store_true', help='Upgrades existing packages.')
+@cli.argument('-d', '--developer', arg_only=True, action='store_true', help='Installs QMK Firmware developer dependencies, too.')
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help='Prints the commands to execute, rather than executing them.')
+@cli.subcommand('Installs python dependencies.')
+def install_deps(cli):
+    return install_qmk_python_deps(upgrade=cli.args.upgrade, developer=cli.args.developer, dry_run=cli.args.dry_run)
+
+
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help='Prints the commands to execute, rather than executing them.')
+@cli.argument('arguments', arg_only=True, default=[], nargs='*', help='The arguments to pass to pip.')
+@cli.subcommand('Runs pip inside the qmk venv.')
+def pip(cli):
+    return _run_pip(cli.args.arguments, dry_run=cli.args.dry_run)

qmk_cli/subcommands/setup.py

@@ -20,7 +20,7 @@ default_branch = 'master'
 def git_upstream(destination):
     """Add the qmk/qmk_firmware upstream to a qmk_firmware clone.
     """
-    git_url = '/'.join((cli.config.setup.baseurl, default_fork))
+    git_url = '/'.join((cli.args.baseurl, default_fork))
     git_cmd = [
         'git',
         '-C',
@@ -45,10 +45,10 @@ def git_upstream(destination):
 
 @cli.argument('-n', '--no', arg_only=True, action='store_true', help='Answer no to all questions')
 @cli.argument('-y', '--yes', arg_only=True, action='store_true', help='Answer yes to all questions')
-@cli.argument('--baseurl', default=default_base, help='The URL all git operations start from. Default: %s' % default_base)
-@cli.argument('-b', '--branch', default=default_branch, help='The branch to clone. Default: %s' % default_branch)
-@cli.argument('-H', '--home', default=Path(os.environ['QMK_HOME']), type=Path, help='The location for QMK Firmware. Default: %s' % os.environ['QMK_HOME'])
-@cli.argument('fork', default=default_fork, nargs='?', help='The qmk_firmware fork to clone. Default: %s' % default_fork)
+@cli.argument('--baseurl', arg_only=True, default=default_base, help='The URL all git operations start from. Default: %s' % default_base)
+@cli.argument('-b', '--branch', arg_only=True, default=default_branch, help='The branch to clone. Default: %s' % default_branch)
+@cli.argument('-H', '--home', arg_only=True, default=Path(os.environ['QMK_HOME']), type=Path, help='The location for QMK Firmware. Default: %s' % os.environ['QMK_HOME'])
+@cli.argument('fork', arg_only=True, default=default_fork, nargs='?', help='The qmk_firmware fork to clone. Default: %s' % default_fork)
 @cli.subcommand('Setup your computer for qmk_firmware.')
 def setup(cli):
     """Guide the user through setting up their QMK environment.
@@ -65,7 +65,8 @@ def setup(cli):
     if is_qmk_firmware(cli.args.home):
         cli.log.info('Found qmk_firmware at %s.', str(cli.args.home))
 
-    elif cli.args.home.exists():
+    # Exists (but not an empty dir)
+    elif cli.args.home.exists() and any(cli.args.home.iterdir()):
         path_str = str(cli.args.home)
 
         if cli.args.home.name != 'qmk_firmware':
@@ -77,12 +78,14 @@ def setup(cli):
     else:
         cli.log.error('Could not find qmk_firmware!')
         if yesno(clone_prompt):
-            git_url = '/'.join((cli.config.setup.baseurl, cli.args.fork))
+            git_url = '/'.join((cli.args.baseurl, cli.args.fork))
 
-            if git_clone(git_url, cli.args.home, cli.config.setup.branch):
+            if git_clone(git_url, cli.args.home, cli.args.branch):
                 git_upstream(cli.args.home)
             else:
                 exit(1)
+        else:
+            cli.log.warning('Not cloning qmk_firmware due to user input or --no flag.')
 
     # Offer to set `user.qmk_home` for them.
     if str(cli.args.home) != os.environ['QMK_HOME'] and yesno(home_prompt):
@@ -91,14 +94,15 @@ def setup(cli):
         cli.write_config_option('user', 'qmk_home')
 
     # Run `qmk doctor` to check the rest of the environment out
-    color = '--color' if cli.config.general.color else '--no-color'
-    unicode = '--unicode' if cli.config.general.unicode else '--no-unicode'
-    doctor_command = [Path(sys.argv[0]).as_posix(), color, unicode, 'doctor']
+    if cli.args.home.exists():
+        color = '--color' if cli.config.general.color else '--no-color'
+        unicode = '--unicode' if cli.config.general.unicode else '--no-unicode'
+        doctor_command = [Path(sys.argv[0]).as_posix(), color, unicode, 'doctor']
 
-    if cli.args.no:
-        doctor_command.append('--no')
+        if cli.args.no:
+            doctor_command.append('-n')
 
-    if cli.args.yes:
-        doctor_command.append('--yes')
+        if cli.args.yes:
+            doctor_command.append('-y')
 
-    cli.run(doctor_command, capture_output=False)
+        cli.run(doctor_command, stdin=None, capture_output=False, cwd=cli.args.home)

qmk_cli/switch_to_venv.py

@@ -0,0 +1,37 @@
+""" Activates a virtualenv if not already present
+"""
+
+import os
+import sys
+
+
+if not os.environ.get('VIRTUAL_ENV') or True:
+    import site
+    from pathlib import Path
+
+    # Build a venv if required
+    INTENDED_VENV_PATH = Path(os.environ['HOME']) / ".local/qmk_venv"
+    if not INTENDED_VENV_PATH.exists():
+        from venv import EnvBuilder
+        builder = EnvBuilder(with_pip=True)
+        builder.create(str(INTENDED_VENV_PATH))
+
+    # Determine the venv's bin directory
+    bin_dir = INTENDED_VENV_PATH / "bin"
+
+    # Update the environment to point to the QMK venv (effectively matches `source path_to_venv/bin/activate`)
+    os.environ["PATH"] = os.pathsep.join([str(bin_dir), *os.environ.get("PATH", "").split(os.pathsep)])
+    os.environ["VIRTUAL_ENV"] = str(INTENDED_VENV_PATH)
+    if 'PYTHONHOME' in os.environ:
+        os.environ.pop('PYTHONHOME')
+
+    # Prepend the venv's library paths to the python import mechanism
+    pyver = sys.version_info[:2]
+    lib_dir = INTENDED_VENV_PATH / f'lib/python{pyver[0]}.{pyver[1]}/site-packages'
+    for d in [lib_dir, bin_dir]:
+        site.addsitedir(str(d))
+        sys.path.insert(0, str(d))
+
+    # Update the python prefixes
+    sys.base_prefix = sys.prefix
+    sys.prefix = str(INTENDED_VENV_PATH)

requirements-dev.txt

@@ -1,2 +1,4 @@
+build
 bumpversion
+requests
 twine

setup.cfg

@@ -1,13 +1,5 @@
-[bumpversion]
-current_version = 0.0.52
-commit = True
-tag = True
-tag_name = {new_version}
-message = New release: {current_version} → {new_version}
-
-[bumpversion:file:qmk_cli/__init__.py]
-
-[bumpversion:file:setup.cfg]
+#[bumpversion]
+# Bumpversion config has been moved to .bumpversion.cfg
 
 [bdist_wheel]
 universal = 1
@@ -16,17 +8,51 @@ universal = 1
 ignore = E501,E226
 
 [metadata]
+name = qmk
+version = 1.1.5
 author = skullydazed
-author-email = skullydazed@gmail.com
-description-file = README.md
-dist-name = qmk
-license_file = LICENSE
-module = qmk_cli
-home-page = https://github.com/qmk/qmk_cli
-requires-python = >=3.7
+author_email = skullydazed@gmail.com
+description = A program to help users work with QMK Firmware.
+long_description = file: README.md
+long_description_content_type = text/markdown
+license = MIT License
+project_urls =
+	Bug Tracker = https://github.com/qmk/qmk_cli/issues
+	Documentation = https://docs.qmk.fm/#/cli
+	Homepage = https://qmk.fm/
+	Source = https://github.com/qmk/qmk_cli/
+classifiers =
+	Development Status :: 3 - Alpha
+	Environment :: Console
+	Intended Audience :: Developers
+	Intended Audience :: System Administrators
+	Intended Audience :: End Users/Desktop
+	License :: OSI Approved :: MIT License
+	Natural Language :: English
+	Programming Language :: Python :: 3 :: Only
+	Topic :: Scientific/Engineering
+	Topic :: Software Development
+	Topic :: Utilities
 
-[entry_points]
-qmk = qmk_cli.script_qmk:main
+[options]
+install_requires =
+	hid
+	milc>=1.6.8
+	pyusb
+	setuptools>=45
+	# qmk_firmware packages
+	dotty-dict
+	hjson
+	jsonschema>=4
+	pillow
+	pygments
+	pyserial
+packages = find:
+python_requires = >=3.7
+
+[options.entry_points]
+console_scripts =
+	qmk = qmk_cli.script_qmk:main
 
 [yapf]
 align_closing_bracket_with_visual_indent = True
@@ -46,13 +72,13 @@ continuation_indent_width = 4
 dedent_closing_brackets = True
 disable_ending_comma_heuristic = False
 each_dict_entry_on_separate_line = True
-i18n_comment = 
-i18n_function_call = 
+i18n_comment =
+i18n_function_call =
 indent_blank_lines = False
 indent_dictionary_value = True
 indent_width = 4
 join_multiple_lines = False
-no_spaces_around_selected_binary_operators = 
+no_spaces_around_selected_binary_operators =
 spaces_around_default_or_named_assign = False
 spaces_around_power_operator = False
 spaces_before_comment = 2

setup.py

@@ -1,54 +0,0 @@
-from configparser import ConfigParser
-from setuptools import setup, find_packages
-
-setup_cfg = ConfigParser()
-setup_cfg.read('setup.cfg')
-metadata = setup_cfg['metadata']
-
-if __name__ == "__main__":
-    with open('README.md', encoding='utf-8') as readme_file:
-        long_description = readme_file.read()
-    setup(
-        name=metadata['dist-name'],
-        description='A program to help users work with QMK Firmware.',
-        entry_points={
-            'console_scripts': ['%s = %s' % i for i in setup_cfg['entry_points'].items()],
-        },
-        license='MIT License',
-        url=metadata['home-page'],
-        version=setup_cfg['bumpversion']['current_version'],
-        author=metadata['author'],
-        author_email=metadata['author-email'],
-        maintainer=metadata['author'],
-        maintainer_email=metadata['author-email'],
-        long_description=long_description,
-        long_description_content_type="text/markdown",
-        packages=find_packages(),
-        py_modules=['qmk_cli'],
-        classifiers=[
-            'Development Status :: 3 - Alpha',
-            'Environment :: Console',
-            'Intended Audience :: Developers',
-            'Intended Audience :: System Administrators',
-            'Intended Audience :: End Users/Desktop',
-            'License :: OSI Approved :: MIT License',
-            'Natural Language :: English',
-            'Programming Language :: Python :: 3 :: Only',
-            'Topic :: Scientific/Engineering',
-            'Topic :: Software Development',
-            'Topic :: Utilities',
-        ],
-        python_requires=metadata['requires-python'],
-        install_requires=[
-            "dotty-dict",
-            "flake8",
-            "hid",
-            "hjson",
-            "jsonschema>=3",
-            "milc>=1.4.0",
-            "nose2",
-            "pygments",
-            "pyusb",
-            "yapf"
-        ]
-    )

trigger_packages

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""Trigger workflows that build QMK packages.
+"""
+from os import environ
+from pathlib import Path
+
+import requests
+
+# Pull in environment vars
+gh_user = environ.get('GH_USERNAME', 'qmk-bot')
+gh_pat = environ.get('QMK_BOT_TOKEN', '')
+gh_repo_owner = environ.get('REPO_OWNER', 'qmk')
+gh_repo_name = environ.get('REPO_NAME', 'qmk_fpm')
+gh_ref = environ.get('BRANCH_NAME', 'main')
+gh_workflow_ids = environ.get('WORKFLOW_IDS', 'all-trigger.yml').split(',')
+gh_api_url = environ.get('GITHUB_API_URL', 'https://api.github.com')
+gh_workflow_args = {'ref': gh_ref}
+gh_workflow_headers = {'Accept': 'application/vnd.github.v3+json'}
+
+for gh_workflow_id in gh_workflow_ids:
+    gh_workflow_endpoint = f'{gh_api_url}/repos/{gh_repo_owner}/{gh_repo_name}/actions/workflows/{gh_workflow_id}/dispatches'
+    print(f'Triggering {gh_workflow_id} workflow at: {gh_workflow_endpoint}')
+    workflow_dispatch = requests.post(gh_workflow_endpoint, headers=gh_workflow_headers, json=gh_workflow_args, auth=(gh_user, gh_pat))
+    if workflow_dispatch.status_code != 204:
+        print(f'Error from GitHub API, status_code {workflow_dispatch.status_code}!')
+        print(workflow_dispatch.text)
+        exit(1)
+
+exit(0)