Add E2E tests for CLI app

.github/workflows/test.yml

@@ -0,0 +1,55 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  test-e2e:
+    name: E2E Tests (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.13"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Build binary
+        run: |
+          pyinstaller --onefile main.py --name gitmastery
+
+      - name: Set binary path (Unix)
+        if: runner.os != 'Windows'
+        run: echo "GITMASTERY_BINARY=${{ github.workspace }}/dist/gitmastery" >> $GITHUB_ENV
+
+      - name: Set binary path (Windows)
+        if: runner.os == 'Windows'
+        run: echo "GITMASTERY_BINARY=${{ github.workspace }}/dist/gitmastery.exe" >> $env:GITHUB_ENV
+
+      - name: Configure Git
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Run E2E tests
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+        run: |
+          python -m pytest tests/e2e/ -v

requirements.txt

@@ -1,4 +1,5 @@
 click
+pytest
 mypy
 pyinstaller
 requests

scripts/build_test_e2e.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Build and run E2E tests for gitmastery
+
+set -e
+FILENAME="gitmastery"
+
+echo "Building gitmastery binary..."
+pyinstaller --onefile main.py --name $FILENAME
+
+echo "Running E2E tests..."
+pytest tests/e2e -v
+
+echo "All E2E tests passed!"

tests/e2e/commands/test_check.py

@@ -0,0 +1,15 @@
+from ..runner import BinaryRunner
+
+
+def test_check_git(runner: BinaryRunner) -> None:
+    """Test the check git command output."""
+    res = runner.run(["check", "git"])
+    res.assert_success()
+    res.assert_stdout_contains("Git is installed")
+
+
+def test_check_github(runner: BinaryRunner) -> None:
+    """Test the check gh command output."""
+    res = runner.run(["check", "github"])
+    res.assert_success()
+    res.assert_stdout_contains("Github CLI is installed")

tests/e2e/commands/test_download.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from ..constants import EXERCISE_NAME, HANDS_ON_NAME
+from ..runner import BinaryRunner
+
+
+def test_download_exercise(runner: BinaryRunner, exercises_dir: Path) -> None:
+    """Test the download command output successfully performs the download for exercise."""
+    res = runner.run(["download", EXERCISE_NAME], cwd=exercises_dir)
+    res.assert_success()
+
+    exercise_folder = exercises_dir / EXERCISE_NAME
+    assert exercise_folder.is_dir()
+
+    exercise_config = exercise_folder / ".gitmastery-exercise.json"
+    assert exercise_config.is_file()
+
+    exercise_readme = exercise_folder / "README.md"
+    assert exercise_readme.is_file()
+
+
+def test_download_hands_on(runner: BinaryRunner, exercises_dir: Path) -> None:
+    """Test the download command output successfully performs the download for hands-on."""
+    res = runner.run(["download", HANDS_ON_NAME], cwd=exercises_dir)
+    res.assert_success()
+
+    hands_on_folder = exercises_dir / HANDS_ON_NAME
+    assert hands_on_folder.is_dir()

tests/e2e/commands/test_progress.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from ..constants import EXERCISE_NAME
+from ..runner import BinaryRunner
+
+
+def test_progress_show(runner: BinaryRunner, exercises_dir: Path) -> None:
+    """Test that progress show displays progress."""
+    res = runner.run(["progress", "show"], cwd=exercises_dir)
+    res.assert_success()
+    res.assert_stdout_contains("Your Git-Mastery progress:")
+
+
+def test_progress_sync_on_then_off(runner: BinaryRunner, exercises_dir: Path) -> None:
+    """Test that progress sync on followed by sync off works correctly."""
+    # Enable sync
+    res_on = runner.run(["progress", "sync", "on"], cwd=exercises_dir)
+    res_on.assert_success()
+    res_on.assert_stdout_contains(
+        "You have setup the progress tracker for Git-Mastery!"
+    )
+
+    # Disable sync (send 'y' to confirm)
+    res_off = runner.run(
+        ["progress", "sync", "off"], cwd=exercises_dir, stdin_text="y\n"
+    )
+    res_off.assert_success()
+    res_off.assert_stdout_contains("Successfully removed your remote sync")
+
+
+def test_progress_reset(runner: BinaryRunner, exercises_dir: Path) -> None:
+    """Test that progress reset works correctly after verify has run."""
+    exercise_dir = exercises_dir / EXERCISE_NAME
+    res = runner.run(["progress", "reset"], cwd=exercise_dir)
+    # TODO: verify that the progress has actually been reset
+    res.assert_success()

tests/e2e/commands/test_setup.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+
+def test_setup(exercises_dir: Path) -> None:
+    """
+    Test that setup creates the progress directory, progress.json, .gitmastery.json and .gitmastery.log
+    Setup command already called in conftest.py for test setup
+    """
+    progress_dir = exercises_dir / "progress"
+    assert progress_dir.is_dir(), f"Expected {progress_dir} to exist"
+
+    progress_file = progress_dir / "progress.json"
+    assert progress_file.is_file(), f"Expected {progress_file} to exist"
+
+    config_file = exercises_dir / ".gitmastery.json"
+    assert config_file.is_file(), f"Expected {config_file} to exist"
+
+    log_file = exercises_dir / ".gitmastery.log"
+    assert log_file.is_file(), f"Expected {log_file} to exist"

tests/e2e/commands/test_verify.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+from ..constants import EXERCISE_NAME
+from ..runner import BinaryRunner
+
+
+def test_verify_exercise(runner: BinaryRunner, exercises_dir: Path) -> None:
+    """Test that verify runs on a downloaded exercise."""
+    exercise_dir = exercises_dir / EXERCISE_NAME
+    res = runner.run(["verify"], cwd=exercise_dir)
+    res.assert_success()
+    # TODO: check that the correct tests have been run
+    res.assert_stdout_contains("Starting verification of")
+    res.assert_stdout_contains("Verification completed.")

tests/e2e/conftest.py

@@ -0,0 +1,43 @@
+from collections.abc import Generator
+from pathlib import Path
+
+import pytest
+
+from .utils import rmtree
+from .runner import BinaryRunner
+
+
+@pytest.fixture(scope="session")
+def runner() -> BinaryRunner:
+    """
+    Return a BinaryRunner instance for the gitmastery binary.
+    """
+    return BinaryRunner.from_env()
+
+
+@pytest.fixture(scope="session")
+def exercises_dir(
+    runner: BinaryRunner, tmp_path_factory: pytest.TempPathFactory
+) -> Generator[Path, None, None]:
+    """
+    Run setup once and return the path to the exercises directory.
+    Tears down by deleting the entire working directory after all tests complete.
+    """
+    work_dir = tmp_path_factory.mktemp("e2e-tests-tmp")
+
+    # Send newline to accept the default directory name prompt
+    res = runner.run(["setup"], cwd=work_dir, stdin_text="\n")
+    assert res.returncode == 0, (
+        f"Setup failed with exit code {res.returncode}\n"
+        f"stdout:\n{res.stdout}\nstderr:\n{res.stderr}"
+    )
+
+    exercises_path = work_dir / "gitmastery-exercises"
+    assert exercises_path.is_dir(), (
+        f"Expected directory {exercises_path} to exist after setup"
+    )
+
+    try:
+        yield exercises_path
+    finally:
+        rmtree(work_dir)  # ensure cleanup even if tests fail

tests/e2e/constants.py

@@ -0,0 +1,2 @@
+EXERCISE_NAME = "under-control"
+HANDS_ON_NAME = "hp-first-commit"

tests/e2e/result.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass
+from typing import List, Self
+import re
+
+
+@dataclass(frozen=True)
+class RunResult:
+    """Represents the result of running a command-line process."""
+
+    stdout: str
+    stderr: str
+    returncode: int
+    command: List[str]
+
+    def assert_success(self) -> Self:
+        """Assert the command exited with code 0."""
+        ERROR_MSG = (
+            f"Expected exit code 0, got {self.returncode}\n"
+            f"Command: {' '.join(self.command)}\n"
+            f"stdout:\n{self.stdout}\n"
+            f"stderr:\n{self.stderr}"
+        )
+        assert self.returncode == 0, ERROR_MSG
+        return self
+
+    def assert_stdout_contains(self, text: str) -> Self:
+        """Assert stdout contains the given text."""
+        ERROR_MSG = (
+            f"Expected stdout to contain {text!r}\nActual stdout:\n{self.stdout}"
+        )
+        assert text in self.stdout, ERROR_MSG
+        return self
+
+    def assert_stdout_matches(self, pattern: str, flags: int = 0) -> Self:
+        """Assert stdout matches a regex pattern."""
+        ERROR_MSG = (
+            f"Expected stdout to match pattern {pattern!r}\n"
+            f"Actual stdout:\n{self.stdout}"
+        )
+        assert re.search(pattern, self.stdout, flags), ERROR_MSG
+        return self

tests/e2e/runner.py

@@ -0,0 +1,73 @@
+import os
+import platform
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional, Sequence, Self
+from .result import RunResult
+
+
+@dataclass
+class BinaryRunner:
+    """Cross-platform runner for the gitmastery binary."""
+
+    binary_path: str
+    project_root: Path
+
+    @classmethod
+    def from_env(
+        cls,
+        env_var: str = "GITMASTERY_BINARY",
+        project_root: Optional[Path] = None,
+    ) -> Self:
+        """Build a runner from an environment variable."""
+        if project_root is None:
+            project_root = Path(__file__).resolve().parents[2]
+
+        raw = os.environ.get(env_var, "").strip()
+        if raw:
+            binary_path = raw
+        else:
+            system = platform.system().lower()
+            if system == "windows":
+                binary_path = str(project_root / "dist" / "gitmastery.exe")
+            else:
+                binary_path = str(project_root / "dist" / "gitmastery")
+
+        return cls(binary_path=binary_path, project_root=project_root)
+
+    def run(
+        self,
+        args: Sequence[str] = (),
+        *,
+        cwd: Optional[Path] = None,
+        env: Optional[Dict[str, str]] = None,
+        timeout: int = 30,
+        stdin_text: Optional[str] = None,
+    ) -> RunResult:
+        """Execute the binary with args and return a RunResult."""
+        cmd = [self.binary_path] + list(args)
+        run_env = os.environ.copy()
+        if env:
+            run_env.update(env)
+
+        # Disable color output and set encoding to utf-8 for consistent output across OS
+        run_env.setdefault("NO_COLOR", "1")
+        run_env.setdefault("PYTHONIOENCODING", "utf-8")
+
+        proc = subprocess.run(
+            cmd,
+            cwd=str(cwd) if cwd else str(self.project_root),
+            env=run_env,
+            capture_output=True,
+            text=True,
+            timeout=timeout if timeout > 0 else None,
+            input=stdin_text,
+        )
+
+        return RunResult(
+            stdout=proc.stdout,
+            stderr=proc.stderr,
+            returncode=proc.returncode,
+            command=cmd,
+        )

tests/e2e/test_version.py

@@ -0,0 +1,9 @@
+from .runner import BinaryRunner
+
+
+def test_version(runner: BinaryRunner) -> None:
+    """Test the version command output."""
+    res = runner.run(["version"])
+    res.assert_success()
+    res.assert_stdout_contains("Git-Mastery app is")
+    res.assert_stdout_matches(r"v\d+\.\d+\.\d+")