Add userspace-setup subcommand

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v2...v3)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.5
+current_version = 1.1.6
 commit = True
 tag = True
 tag_name = {new_version}

.github/workflows/cli_setup.yml

@@ -32,7 +32,10 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+        include:
+          - os: macos-13
+            python-version: '3.9'
 
     steps:
     - uses: actions/checkout@v4
@@ -59,18 +62,19 @@ jobs:
       uses: msys2/setup-msys2@v2
       with:
         update: true
-        install: git mingw-w64-x86_64-toolchain mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-pillow mingw-w64-x86_64-rust
+        install: git mingw-w64-x86_64-toolchain mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-build mingw-w64-x86_64-python-pillow mingw-w64-x86_64-rust
 
+    # Upgrade pip due to msys packaging + pypa/build/pull/736 issues
     - name: (MSYS2) Install Python dependencies
       shell: msys2 {0}
       run: |
-        python3 -m pip install -U setuptools wheel
+        python3 -m pip install --upgrade pip
+
     - name: (MSYS2) Install QMK CLI from source
       shell: msys2 {0}
       run: |
-        python3 -m pip install -U build
         python3 -m build
-        python3 -m pip install dist/qmk-*.tar.gz
+        python3 -m pip install --break-system-packages dist/qmk-*.tar.gz
     - name: (MSYS2) Run qmk setup -y
       shell: msys2 {0}
       run: qmk setup -y

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/docker-republish.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.7'
+        python-version: '3.9'
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
@@ -46,7 +46,7 @@ jobs:
         password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
     - name: Build and Push to Docker Hub
-      uses: docker/build-push-action@v5.1.0
+      uses: docker/build-push-action@v6.9.0
       with:
         context: .
         push: true

.github/workflows/python-publish.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.7'
+        python-version: '3.9'
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
@@ -74,7 +74,7 @@ jobs:
         password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
     - name: Build and Push to Docker Hub
-      uses: docker/build-push-action@v5.1.0
+      uses: docker/build-push-action@v6.9.0
       with:
         context: .
         push: true

README.md

@@ -1,5 +1,5 @@
 # QMK CLI
-[![CLI Setup](https://github.com/qmk/qmk_cli/workflows/CLI%20Setup/badge.svg)](https://github.com/qmk/qmk_cli/actions?query=workflow%3A%22CLI+Setup%22)  
+[![CLI Setup](https://github.com/qmk/qmk_cli/actions/workflows/cli_setup.yml/badge.svg)](https://github.com/qmk/qmk_cli/actions/workflows/cli_setup.yml)
 
 A program to help users work with [QMK Firmware](https://qmk.fm/).
 

qmk_cli/__init__.py

@@ -1,3 +1,3 @@
 """A program to help you work with qmk_firmware."""
 
-__version__ = '1.1.5'
+__version__ = '1.1.6'

qmk_cli/helpers.py

@@ -1,8 +1,9 @@
 """Useful helper functions.
 """
 import os
+import json
+from platformdirs import user_data_dir
 from functools import lru_cache
-from importlib.util import find_spec
 from pathlib import Path
 
 from milc import cli
@@ -44,6 +45,10 @@ def find_qmk_firmware():
             return path.resolve()
         return path
 
+    hidden_home = Path(user_data_dir('qmk_cli', 'QMK')) / 'qmk_firmware'
+    if hidden_home.exists():
+        return hidden_home
+
     return Path.home() / 'qmk_firmware'
 
 
@@ -58,3 +63,48 @@ def in_qmk_firmware():
         # Move up a directory before the next iteration
         cur_dir = cur_dir / '..'
         cur_dir = cur_dir.resolve()
+
+
+def is_qmk_userspace(qmk_userspace):
+    """Returns True if the given Path() is a qmk_userspace clone.
+    """
+    path = qmk_userspace / 'qmk.json'
+    if not path.exists():
+        return False
+
+    try:
+        return 'userspace_version' in json.loads(path.read_text(encoding="UTF-8"))
+    except json.decoder.JSONDecodeError as e:
+        return False
+
+
+@lru_cache(maxsize=2)
+def find_qmk_userspace():
+    """Look for qmk_userspace in the usual places.
+    """
+    if in_qmk_userspace():
+        return in_qmk_userspace()
+
+    if cli.config.user.overlay_dir:
+        return Path(cli.config.user.overlay_dir).expanduser().resolve()
+
+    if 'QMK_USERSPACE' in os.environ:
+        path = Path(os.environ['QMK_USERSPACE']).expanduser()
+        if path.exists():
+            return path.resolve()
+        return path
+
+    return Path.home() / 'qmk_userspace'
+
+
+def in_qmk_userspace():
+    """Returns the path to the qmk_userspace we are currently in, or None if we are not inside qmk_userspace.
+    """
+    cur_dir = Path.cwd()
+    while len(cur_dir.parents) > 0:
+        if is_qmk_userspace(cur_dir):
+            return cur_dir
+
+        # Move up a directory before the next iteration
+        cur_dir = cur_dir / '..'
+        cur_dir = cur_dir.resolve()

qmk_cli/script_qmk.py

@@ -10,19 +10,16 @@ import sys
 from platform import platform
 from traceback import print_exc
 
-from . import switch_to_venv # noqa, intentionally unused
-
 import milc
 
 from . import __version__
-from .helpers import find_qmk_firmware, is_qmk_firmware
+from .helpers import find_qmk_firmware, is_qmk_firmware, find_qmk_userspace, is_qmk_userspace
 
-milc.set_metadata(version=__version__)
+milc.cli.milc_options(version=__version__)
 milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}Ψ{style_reset_all}'
 
-# These must happen after the milc.set_metadata() call
-import milc.subcommand.config  # noqa, must come after milc.set_metadata()
-from milc.questions import yesno
+# These must happen after the milc.milc_options() call
+import milc.subcommand.config  # noqa, must come after milc.milc_options()
 
 
 @milc.cli.entrypoint('CLI wrapper for running QMK commands.')
@@ -48,9 +45,9 @@ def main():
     """Setup the environment before dispatching to the entrypoint.
     """
     # Warn if they use an outdated python version
-    if sys.version_info < (3, 7):
+    if sys.version_info < (3, 9):
         print('Warning: Your Python version is out of date! Some subcommands may not work!')
-        print('Please upgrade to Python 3.7 or later.')
+        print('Please upgrade to Python 3.9 or later.')
 
     if 'windows' in platform().lower():
         msystem = os.environ.get('MSYSTEM', '')
@@ -62,7 +59,9 @@ def main():
             exit(1)
 
     # Environment setup
+    qmk_userspace = find_qmk_userspace()
     qmk_firmware = find_qmk_firmware()
+    os.environ['QMK_USERSPACE'] = str(qmk_userspace)
     os.environ['QMK_HOME'] = str(qmk_firmware)
     os.environ['ORIG_CWD'] = os.getcwd()
 

qmk_cli/subcommands/__init__.py

@@ -6,4 +6,4 @@ from . import clone  # noqa
 from . import console  # noqa
 from . import env  # noqa
 from . import setup  # noqa
-from . import pip # noqa
+from . import userspace  # noqa

qmk_cli/subcommands/console.py

@@ -184,7 +184,7 @@ class FindDevices(object):
                         except Exception as e:
                             device['e'] = e
                             device['e_name'] = e.__class__.__name__
-                            cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s:%(vendor_id)04X:%(product_id)04X:%(index)d): %(e_name)s: %(e)s", device)
+                            cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all}): %(e_name)s: %(e)s", device)
                             if cli.config.general.verbose:
                                 cli.log.exception(e)
                             del live_devices[device['path']]

qmk_cli/subcommands/env.py

@@ -4,16 +4,18 @@ import os
 from pathlib import Path
 
 from milc import cli
-from qmk_cli.helpers import is_qmk_firmware
+from qmk_cli.helpers import is_qmk_firmware, is_qmk_userspace
 
 
 @cli.argument('var', arg_only=True, default=None, nargs='?', help='Optional variable to query')
 @cli.subcommand('Prints environment information.')
 def env(cli):
     home = os.environ.get('QMK_HOME', "")
+    userspace = os.environ.get('QMK_USERSPACE', "")
     data = {
         'QMK_HOME': home,
-        'QMK_FIRMWARE': home if is_qmk_firmware(Path(home)) else ""
+        'QMK_FIRMWARE': home if is_qmk_firmware(Path(home)) else "",
+        'QMK_USERSPACE': userspace if is_qmk_userspace(Path(userspace)) else ""
     }
 
     # Now munge the current cli config

qmk_cli/subcommands/pip.py

@@ -1,72 +0,0 @@
-"""Installs python dependencies.
-"""
-import shlex
-import shutil
-import subprocess
-from milc import cli
-
-from qmk_cli.helpers import find_qmk_firmware, is_qmk_firmware
-
-
-def _run_pip(pip_args, dry_run=False, upgrade=False):
-    pip_exe = ['pip']
-    if cli.config.user.pip_exe:
-        pip_exe = shlex.split(cli.config.user.pip_exe)
-    pip_command = [*pip_exe, *pip_args]
-    pip_command[0] = shutil.which(pip_command[0])
-    if shutil.which(pip_command[0]) is None:
-        pip_command[0] = 'pip' # Hope falling back to unqualified path works
-
-    if dry_run:
-        print(" ".join(shlex.quote(str(s)) for s in pip_command))
-        return True
-
-    try:
-        with subprocess.Popen(pip_command, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, encoding='utf-8') as p:
-            for line in p.stdout:
-                print(line, end='')
-    except Exception as e:
-        pip_cmd = ' '.join([s.replace(' ', r'\ ') for s in pip_command])
-        cli.log.error("Could not run '%s': %s: %s", pip_cmd, e.__class__.__name__, e)
-        return False
-
-    if p.returncode == 0:
-        return True
-    return False
-
-
-def install_qmk_python_deps(upgrade=False, developer=False, dry_run=False):
-    qmk_firmware_dir = find_qmk_firmware()
-    if not is_qmk_firmware(qmk_firmware_dir):
-        cli.log.error('Could not find {fg_cyan}qmk_firmware{fg_reset}!')
-        return False
-
-    reqs_path = qmk_firmware_dir / 'requirements.txt' if not developer else qmk_firmware_dir / 'requirements-dev.txt'
-
-    pip_args = ['install', '-r', reqs_path]
-    if upgrade:
-        pip_args.append('--upgrade')
-
-    if _run_pip(pip_args, dry_run=dry_run):
-        if not dry_run:
-            cli.log.info('Successfully installed python dependencies!')
-        return True
-    else:
-        if not dry_run:
-            cli.log.error('Failed to install python dependencies!')
-        return False
-
-
-@cli.argument('-u', '--upgrade', arg_only=True, action='store_true', help='Upgrades existing packages.')
-@cli.argument('-d', '--developer', arg_only=True, action='store_true', help='Installs QMK Firmware developer dependencies, too.')
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help='Prints the commands to execute, rather than executing them.')
-@cli.subcommand('Installs python dependencies.')
-def install_deps(cli):
-    return install_qmk_python_deps(upgrade=cli.args.upgrade, developer=cli.args.developer, dry_run=cli.args.dry_run)
-
-
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help='Prints the commands to execute, rather than executing them.')
-@cli.argument('arguments', arg_only=True, default=[], nargs='*', help='The arguments to pass to pip.')
-@cli.subcommand('Runs pip inside the qmk venv.')
-def pip(cli):
-    return _run_pip(cli.args.arguments, dry_run=cli.args.dry_run)

qmk_cli/subcommands/setup.py

@@ -5,9 +5,10 @@ import shlex
 import subprocess
 import sys
 from pathlib import Path
+from shutil import rmtree
 
 from milc import cli
-from milc.questions import yesno
+from milc.questions import choice, question, yesno
 from qmk_cli.git import git_clone
 from qmk_cli.helpers import is_qmk_firmware
 
@@ -43,6 +44,17 @@ def git_upstream(destination):
         return False
 
 
+def git_clone_fork(fork, branch, force=False):
+    if force:
+        rmtree(cli.args.home)
+
+    git_url = '/'.join((cli.args.baseurl, fork))
+    if git_clone(git_url, cli.args.home, branch):
+        git_upstream(cli.args.home)
+    else:
+        exit(1)
+
+
 @cli.argument('-n', '--no', arg_only=True, action='store_true', help='Answer no to all questions')
 @cli.argument('-y', '--yes', arg_only=True, action='store_true', help='Answer yes to all questions')
 @cli.argument('--baseurl', arg_only=True, default=default_base, help='The URL all git operations start from. Default: %s' % default_base)
@@ -61,9 +73,34 @@ def setup(cli):
         cli.log.error("Can't use both --yes and --no at the same time.")
         exit(1)
 
-    # Check on qmk_firmware, and if it doesn't exist offer to check it out.
+    # Check on qmk_firmware
+    # If it exists, ask the user what to do with it
+    # If it doesn't exist, offer to check it out
     if is_qmk_firmware(cli.args.home):
         cli.log.info('Found qmk_firmware at %s.', str(cli.args.home))
+        found_prompt = "What do you want to do?"
+        found_options = [
+            f"Delete and reclone {cli.args.fork}",
+            "Delete and clone a different fork",
+            "Keep it and continue"
+        ]
+        delete_confirm = "WARNING: This will delete your current qmk_firmware directory. Proceed?"
+
+        found_action = choice(found_prompt, options=found_options, default=2)
+        if found_action == f"Delete and reclone {cli.args.fork}":
+            if not yesno(delete_confirm, default=False):
+                exit(1)
+
+            git_clone_fork(cli.args.fork, cli.args.branch, force=True)
+
+        elif found_action == "Delete and clone a different fork":
+            fork_name = question("Enter the name of the fork:", default=cli.args.fork)
+            branch_name = question("Enter the branch name to clone:", default=cli.args.branch)
+
+            if not yesno(delete_confirm, default=False):
+                exit(1)
+
+            git_clone_fork(fork_name, branch_name, force=True)
 
     # Exists (but not an empty dir)
     elif cli.args.home.exists() and any(cli.args.home.iterdir()):
@@ -78,12 +115,7 @@ def setup(cli):
     else:
         cli.log.error('Could not find qmk_firmware!')
         if yesno(clone_prompt):
-            git_url = '/'.join((cli.args.baseurl, cli.args.fork))
-
-            if git_clone(git_url, cli.args.home, cli.args.branch):
-                git_upstream(cli.args.home)
-            else:
-                exit(1)
+            git_clone_fork(cli.args.fork, cli.args.branch)
         else:
             cli.log.warning('Not cloning qmk_firmware due to user input or --no flag.')
 

qmk_cli/subcommands/userspace.py

@@ -0,0 +1,70 @@
+"""Setup qmk_userspace on your computer.
+"""
+import os
+import sys
+from pathlib import Path
+from platformdirs import user_data_dir
+
+from milc import cli
+
+from milc.questions import yesno
+from qmk_cli.git import git_clone
+from qmk_cli.helpers import find_qmk_firmware, is_qmk_firmware, is_qmk_userspace
+
+default_base = 'https://github.com'
+default_fork = 'qmk/qmk_userspace'
+default_branch = 'main'
+
+
+@cli.argument('-n', '--no', arg_only=True, action='store_true', help='Answer no to all questions')
+@cli.argument('-y', '--yes', arg_only=True, action='store_true', help='Answer yes to all questions')
+@cli.argument('--baseurl', arg_only=True, default=default_base, help='The URL all git operations start from. Default: %s' % default_base)
+@cli.argument('-b', '--branch', arg_only=True, default=default_branch, help='The branch to clone. Default: %s' % default_branch)
+@cli.argument('-H', '--home', arg_only=True, default=Path(os.environ['QMK_USERSPACE']), type=Path, help='The location for qmk_userspace. Default: %s' % os.environ['QMK_HOME'])
+@cli.argument('fork', arg_only=True, default=default_fork, nargs='?', help='The qmk_userspace fork to clone. Default: %s' % default_fork)
+@cli.subcommand('Setup your computer for qmk_userspace.')
+def userspace_setup(cli):
+    """Guide the user through setting up their QMK environment.
+    """
+    # Sanity checks
+    if cli.args.yes and cli.args.no:
+        cli.log.error("Can't use both --yes and --no at the same time.")
+        exit(1)
+
+    if is_qmk_userspace(cli.args.home):
+        cli.log.info(f'Found qmk_userspace at {cli.args.home}.')
+    else: 
+        cli.log.error('Could not find qmk_userspace!')
+        if yesno('Would you like to clone qmk_userspace?'):
+            git_url = f'{cli.args.baseurl}/{cli.args.fork}'
+            if not git_clone(git_url, cli.args.home, cli.args.branch):
+                exit(1)
+        else:
+            cli.log.warning('Not cloning qmk_userspace due to user input or --no flag.')
+
+    qmk_firmware = find_qmk_firmware()
+    if is_qmk_firmware(qmk_firmware):
+        cli.log.info(f'Found qmk_firmware at {qmk_firmware}.')
+    else: 
+        cli.log.error('Could not find qmk_firmware!')
+        if yesno('Would you like to clone qmk_firmware?'):
+            git_url = f'{cli.args.baseurl}/qmk/qmk_firmware'
+            hidden_home = Path(user_data_dir('qmk_cli', 'QMK')) / 'qmk_firmware'
+            if not git_clone(git_url, hidden_home, 'master'):
+                exit(1)
+        else:
+            cli.log.warning('Not cloning qmk_firmware due to user input or --no flag.')
+
+    # Run `qmk doctor` to check the rest of the environment out
+    if cli.args.home.exists():
+        color = '--color' if cli.config.general.color else '--no-color'
+        unicode = '--unicode' if cli.config.general.unicode else '--no-unicode'
+        doctor_command = [Path(sys.argv[0]).as_posix(), color, unicode, 'doctor']
+
+        if cli.args.no:
+            doctor_command.append('-n')
+
+        if cli.args.yes:
+            doctor_command.append('-y')
+
+        cli.run(doctor_command, stdin=None, capture_output=False, cwd=cli.args.home)

qmk_cli/switch_to_venv.py

@@ -1,37 +0,0 @@
-""" Activates a virtualenv if not already present
-"""
-
-import os
-import sys
-
-
-if not os.environ.get('VIRTUAL_ENV') or True:
-    import site
-    from pathlib import Path
-
-    # Build a venv if required
-    INTENDED_VENV_PATH = Path(os.environ['HOME']) / ".local/qmk_venv"
-    if not INTENDED_VENV_PATH.exists():
-        from venv import EnvBuilder
-        builder = EnvBuilder(with_pip=True)
-        builder.create(str(INTENDED_VENV_PATH))
-
-    # Determine the venv's bin directory
-    bin_dir = INTENDED_VENV_PATH / "bin"
-
-    # Update the environment to point to the QMK venv (effectively matches `source path_to_venv/bin/activate`)
-    os.environ["PATH"] = os.pathsep.join([str(bin_dir), *os.environ.get("PATH", "").split(os.pathsep)])
-    os.environ["VIRTUAL_ENV"] = str(INTENDED_VENV_PATH)
-    if 'PYTHONHOME' in os.environ:
-        os.environ.pop('PYTHONHOME')
-
-    # Prepend the venv's library paths to the python import mechanism
-    pyver = sys.version_info[:2]
-    lib_dir = INTENDED_VENV_PATH / f'lib/python{pyver[0]}.{pyver[1]}/site-packages'
-    for d in [lib_dir, bin_dir]:
-        site.addsitedir(str(d))
-        sys.path.insert(0, str(d))
-
-    # Update the python prefixes
-    sys.base_prefix = sys.prefix
-    sys.prefix = str(INTENDED_VENV_PATH)

setup.cfg

@@ -9,7 +9,7 @@ ignore = E501,E226
 
 [metadata]
 name = qmk
-version = 1.1.5
+version = 1.1.6
 author = skullydazed
 author_email = skullydazed@gmail.com
 description = A program to help users work with QMK Firmware.
@@ -37,7 +37,7 @@ classifiers =
 [options]
 install_requires =
 	hid
-	milc>=1.6.8
+	milc>=1.9.0
 	pyusb
 	setuptools>=45
 	# qmk_firmware packages
@@ -48,7 +48,7 @@ install_requires =
 	pygments
 	pyserial
 packages = find:
-python_requires = >=3.7
+python_requires = >=3.9
 
 [options.entry_points]
 console_scripts =