diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 28ee15466..fac471d90 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -3,11 +3,13 @@ ## Summary This release documents how to discover and use PTB's Nox sessions in the user -guide and removes the unused Modules section from the developer guide. +guide, adds an agent skill for PTB work, and removes the unused Modules section +from the developer guide. ## Documentation * #456: Documented how to discover PTB nox sessions in the user guide +* #933: Added an agent skill for PTB work ## Refactoring diff --git a/exasol/toolbox/skills/exasol-python-toolbox/SKILL.md b/exasol/toolbox/skills/exasol-python-toolbox/SKILL.md new file mode 100644 index 000000000..e9d6ee49a --- /dev/null +++ b/exasol/toolbox/skills/exasol-python-toolbox/SKILL.md @@ -0,0 +1,64 @@ +--- +name: exasol-python-toolbox +description: Use this skill in Exasol Python projects that use exasol-toolbox/PTB. Use it for PTB setup, nox sessions, code checks, GitHub workflows, updates, releases, and PTB configuration. Use it when an agent must not replace PTB automation. +--- + +# Exasol Python Toolbox + +Use PTB as the task interface for Exasol Python repositories. Use the nox +sessions, configuration, templates, and hooks from the installed +`exasol-toolbox` package. Do not replace them with manual commands. Use direct +tool commands only to isolate a failure. + +Use controlled technical English. Use short sentences. Put one instruction in +one sentence when possible. Use the same term for the same item. + +## Operating rules + +1. Confirm that the project uses PTB before you act: + - `pyproject.toml` contains a dependency on `exasol-toolbox`. + - `noxfile.py` imports from `exasol.toolbox.nox.tasks`. + - `noxconfig.py` defines `PROJECT_CONFIG`. +2. Run commands with Poetry. If the project gives a different command, use it. +3. Use `references/nox-sessions.md` for session names and command syntax. +4. Before you edit generated workflow files, use PTB workflow sessions, + `.workflow-patcher.yml`, `PROJECT_CONFIG`, or PTB hooks. +5. Do not run Black, isort, Ruff, Pylint, Mypy, Pytest, Coverage, Sphinx, + Poetry build, Twine, or zizmor directly when a PTB nox session does the same + work. Use direct tool commands only for fault isolation. + +## Task routing + +Read only the reference that the task needs: + +- Source files and resource files for exact PTB operation: + `references/source-routing.md` +- Setup, code changes, CI fixes, releases, PTB updates, dependency updates, and + workflow maintenance: `references/common-workflows.md` +- Nox session names, arguments, outputs, and limits: + `references/nox-sessions.md` +- Coding rules, PTB design rules, and rules for PTB extension: + `references/coding-guidelines.md` + +## Source rule + +The skill does not copy PTB source code. Source copies can become old. Use this +skill to find the correct file. Read the file from the repository before you +change or explain exact PTB operation. + +## Release safety + +`release:trigger` creates git tags. It pushes the tags. Run it only when the +user asks you to trigger a release. + +Before you run `release:trigger`: + +1. Confirm that `release:prepare` ran and its PR is merged. +2. Confirm that the current branch is the default branch. +3. Confirm that the current branch is at the latest remote commit. +4. Confirm that the current project version has no git tag and no GitHub + release. +5. Confirm that the current project version is newer than the latest release, + unless the user confirms a different release plan. + +Use `references/common-workflows.md` for the release workflow. diff --git a/exasol/toolbox/skills/exasol-python-toolbox/references/coding-guidelines.md b/exasol/toolbox/skills/exasol-python-toolbox/references/coding-guidelines.md new file mode 100644 index 000000000..9d66d84d1 --- /dev/null +++ b/exasol/toolbox/skills/exasol-python-toolbox/references/coding-guidelines.md @@ -0,0 +1,75 @@ +# Coding Guidelines for PTB Work + +PTB is a development dependency. PTB is also a task interface. Keep project +automation aligned with PTB conventions. Change the convention only if the +project has a clear reason. + +Use controlled technical English in user-facing text. Use short sentences. Put +one instruction in one sentence when possible. Use consistent terms. + +## Sources to read + +- `doc/design.rst`: PTB design rules and task boundaries. +- `doc/user_guide/getting_started.rst`: setup and integration. +- `doc/user_guide/features/formatting_code/index.rst`: format tools and format + sessions. +- `doc/user_guide/features/creating_a_release.rst`: release procedure. +- `doc/user_guide/features/git_hooks/index.rst`: pre-commit and pre-push + hooks. +- `doc/user_guide/features/github_workflows/index.rst`: workflow types, + maintained workflows, and custom workflows. +- `doc/user_guide/features/github_workflows/create_and_update.rst`: workflow + generation, update, and check commands. +- `doc/user_guide/features/github_workflows/workflow_patcher.rst`: supported + `.workflow-patcher.yml` configuration. +- `doc/user_guide/features/github_workflows/workflow_variables.rst`: workflow + template values, custom workflow secrets, permissions, and matrix behavior. +- PEP 8: Python style guidance: + `https://peps.python.org/pep-0008/` +- Google Python Style Guide: Python coding guidance: + `https://google.github.io/styleguide/pyguide.html` +- Exasol Python styleguide tooling reference: + `https://exasol.github.io/python-styleguide/guides/tooling.html`. Use it as + a PTB tooling reference. Do not use it as the final official Exasol coding + guideline until the team defines and maintains one. + +## Design rules to keep + +- Treat PTB as development tooling. +- Do not import PTB from production package code. +- Use conventions first. +- Add configuration only when a convention is not sufficient. +- Use `pyproject.toml` for static tool configuration. +- Use `noxconfig.py` and `PROJECT_CONFIG` for dynamic project configuration. +- Use plugin hooks or PTB extension points for project-specific work. +- Make project-specific work operate with standard sessions. +- Keep GitHub workflows as orchestration. +- Put shared task logic in Python functions or nox sessions. +- Do not put shared task logic directly in workflow YAML. +- Do not make nox sessions notify other nox sessions for shared task logic. +- Put shared nox logic in functions that receive the nox `Session`. +- Add a PTB feature only when a real project needs it. +- Move logic into PTB when more than one project needs it. + +## Code change rules + +Use PTB sessions to format and validate code. + +When you change public operation, update the related tests. For almost all +non-automated changes, add an entry to `doc/changes/unreleased.md`. The usual +exception is a dependency update PR that CI creates automatically. + +For workflow-template changes, check generated workflows. Generate workflows +only when you intend to change them. Use `nox-sessions.md` for command syntax. + +## Project-specific differences + +If a standard PTB session is not correct for a project: + +1. Check whether existing `BaseConfig` or `PROJECT_CONFIG` fields solve the + issue. +2. Check whether a PTB plugin hook applies. +3. Use `.workflow-patcher.yml` for supported workflow changes. +4. Use a local nox override only as a short-term migration step. +5. If multiple projects need the same change, consider a PTB template, + configuration field, or hook. diff --git a/exasol/toolbox/skills/exasol-python-toolbox/references/common-workflows.md b/exasol/toolbox/skills/exasol-python-toolbox/references/common-workflows.md new file mode 100644 index 000000000..f08b2569e --- /dev/null +++ b/exasol/toolbox/skills/exasol-python-toolbox/references/common-workflows.md @@ -0,0 +1,141 @@ +# Common PTB Workflows + +Use these workflows in an Exasol Python repository that uses PTB. +Use `nox-sessions.md` for nox command syntax. + +## Set up PTB + +For a new project, use the PTB Cookiecutter template. + +```bash +cookiecutter https://github.com/exasol/python-toolbox.git \ + --checkout --directory project-template \ + --overwrite-if-exists +``` + +Use a released tag. This gives the same setup result each time. + +After you create the project, run these commands: + +```bash +poetry env use +poetry install +``` + +For an existing project: + +1. Add PTB as a development dependency: + `poetry add --group dev exasol-toolbox` +2. Add or update `noxconfig.py`. +3. Define `PROJECT_CONFIG` in `noxconfig.py`. +4. Import the standard sessions in `noxfile.py`: + `from exasol.toolbox.nox.tasks import *` +5. Move tool configuration to `pyproject.toml` where PTB expects it. +6. Run `poetry install`. +7. List the available nox sessions. +8. Add optional integrations only when the project needs them. Examples are + pre-commit hooks, documentation, Sonar, and GitHub workflows. + +If a project cannot use a PTB session without changes, use a local override +only as a short-term migration step. For a permanent solution, use +configuration, plugin hooks, or PTB extension points. +When you add a temporary local override, create an issue in the project to +replace it with configuration, a plugin hook, or a PTB extension point later. + +## Make a daily code change + +1. Read `noxconfig.py`. +2. Read the related tool sections in `pyproject.toml`. +3. Make the requested code change, document change, or configuration change. +4. Run the smallest PTB session that checks the change. +5. Run `project:check` when the change affects shared operation. +6. Run `project:check` when the change affects release readiness. +7. Run `project:check` when the change affects CI operation. + +When you remove or rename a documentation file, update each Sphinx `toctree` +that points to it. Then run `docs:build`. + +## Use git hooks + +If `.pre-commit-config.yaml` exists, the project can run hooks on commit and +push. + +- On commit, pre-commit hooks can apply format fixes. Add the changed files and + commit again. +- On push, pre-push hooks can run checks such as type checks and lint checks. + Fix failures and push again. + +## Fix lint or format findings + +Run the PTB formatter first. Then run the checks that match the failure. +Use `nox-sessions.md` for the exact session commands. + +Do not adjust code manually to imitate Black, isort, or Ruff. Let PTB run these +tools. To exclude a Python file or directory, use +`PROJECT_CONFIG.add_to_excluded_python_paths` in `noxconfig.py`. + +## Prepare a release + +Prepare a release with `release:prepare`. + +This session does these actions: + +1. It updates the version. +2. It creates a release branch and sets it as the current Git branch unless you + use `--no-branch` or `--no-add`. +3. It moves content from `doc/changes/unreleased.md` to a versioned changes file. +4. It updates `doc/changes/changelog.md`. +5. It runs configured release hooks. +6. It commits the changes if you do not use `--no-add`. +7. It opens a PR if you do not use `--no-pr`. + +Useful flags: + +- `--no-branch`: Do not create a release branch and do not set it as current. +- `--no-add`: Do not add or commit changes. This also prevents release branch + creation. +- `--no-pr`: Do not create a pull request. + +If dependencies change after release preparation, update the versioned changes +file with `release:update`. + +After the release PR is merged, apply the release safety rule from `SKILL.md`. + +`release:trigger` checks out the default branch. It pulls the default branch. It +creates a version tag. It pushes the tag. It updates a `v` tag if +`PROJECT_CONFIG.create_major_version_tags` is `True`. + +## Update PTB or dependencies + +When you update `exasol-toolbox` in a project: + +1. Read the PTB changelog for migration notes. +2. Check the `exasol-toolbox` version range in `pyproject.toml`. +3. Update the dependency with Poetry. Example: + `poetry update exasol-toolbox` +4. If Poetry does not update PTB, adjust the version range intentionally. This + is usually necessary for a new PTB major version. +5. Generate or check PTB-managed workflows. +6. Run necessary validation sessions. +7. Update `doc/changes/unreleased.md` when the project requires a changelog + entry. + +For dependency updates that fix vulnerabilities, use the PTB vulnerability +sessions. + +## Maintain GitHub workflows + +PTB has two workflow groups: + +- PTB-provided workflows: Do not edit generated files by hand. Use + `.workflow-patcher.yml` for supported project-specific changes. Use + `PROJECT_CONFIG`, a PTB template, or a hook for shared behavior. Then run + `workflow:generate` and `workflow:check`. +- Custom workflows: The project owns these files. Edit the custom file directly. + Prefer workflow extension files that PTB can call from a PTB-provided + workflow. Use a separate workflow trigger only when the project owns the merge + protection decision for that workflow. Put GitHub `permissions` in the jobs + that need them. Declare reusable-workflow secrets under + `on.workflow_call.secrets`. + +After workflow changes, run `workflow:audit`. diff --git a/exasol/toolbox/skills/exasol-python-toolbox/references/nox-sessions.md b/exasol/toolbox/skills/exasol-python-toolbox/references/nox-sessions.md new file mode 100644 index 000000000..c22662e9a --- /dev/null +++ b/exasol/toolbox/skills/exasol-python-toolbox/references/nox-sessions.md @@ -0,0 +1,94 @@ +# PTB Nox Sessions + +Always list the sessions in the current project: + +```bash +poetry run -- nox -l +``` + +The sessions below match the PTB version that includes this skill. + +## Project and quality sessions + +| Session | Use | Notes | +| --- | --- | --- | +| `project:check` | Run large local checks or CI checks. | It runs format check, Pylint, Mypy, and coverage for unit and integration tests. | +| `format:fix` | Apply Python format changes. | It runs pyupgrade, Ruff fixes, isort, and Black. | +| `format:check` | Check Python format. | It runs Ruff check, isort check, and Black check. | +| `lint:code` | Run static code analysis. | It runs Pylint on `PROJECT_CONFIG.source_code_path` and writes `.lint.json`. | +| `lint:typing` | Run type checks. | It runs Mypy on filtered project Python files. | +| `lint:security` | Run security lint. | It runs Bandit and writes `.security.json`. | + +## Test sessions + +| Session | Use | Notes | +| --- | --- | --- | +| `test:unit` | Run unit tests. | It runs `pytest -v test/unit`. Put Pytest options after `--`; use `-k` to select tests by name. | +| `test:integration` | Run integration tests. | It runs `pytest -v test/integration`. It supports PTB integration test hooks. | +| `test:coverage` | Run all tests with coverage. | It runs unit and integration tests with coverage. It prints `coverage report -m`. | + +Test command examples: + +```bash +poetry run -- nox -s test:unit +poetry run -- nox -s test:unit -- --coverage +poetry run -- nox -s test:unit -- -k scenario +poetry run -- nox -s test:integration -- --db-version 8.34.0 +``` + +## Documentation and changelog sessions + +| Session | Use | Notes | +| --- | --- | --- | +| `docs:build` | Build current documentation. | It runs Sphinx HTML build. It makes warnings errors. | +| `docs:multiversion` | Build multiversion documentation. | It runs `sphinx-multiversion` and creates `.nojekyll`. | +| `docs:open` | Open built documentation in a browser. | It requires `.html-documentation`. It can require GUI approval. | +| `docs:clean` | Remove generated documentation. | It deletes `.html-documentation`. | +| `links:list` | List documentation links. | It uses Sphinx linkcheck. It ignores link failures. | +| `links:check` | Validate documentation links. | It can write JSON output with `-- -o `. | +| `changelog:updated` | Check that the changelog changed. | It fails if `doc/changes` did not change when compared with `origin/main`. | + +## Release sessions + +| Session | Use | Notes | +| --- | --- | --- | +| `release:prepare` | Prepare a release PR. | It requires `-- --type major`, `minor`, or `patch`. It creates a branch, commit, and PR if flags do not disable these actions. | +| `release:update` | Update a prepared release changelog. | It updates the latest versioned changelog after dependency changes. | +| `release:trigger` | Trigger a release from the default branch. | It creates and pushes git tags. | + +Release command examples: + +```bash +poetry run -- nox -s release:prepare -- --type patch +poetry run -- nox -s release:prepare -- --type minor --no-pr --no-branch --no-add +poetry run -- nox -s release:update +``` + +## Workflow sessions + +| Session | Use | Notes | +| --- | --- | --- | +| `workflow:check` | Check generated workflows. | It requires one workflow name or `all`. It fails when files differ from PTB templates. | +| `workflow:generate` | Generate or update workflows. | It requires one workflow name or `all`. | +| `workflow:audit` | Audit workflows and actions. | It runs zizmor with PTB configuration. It can pass `--fix` to zizmor to apply automatic fixes. | + +Workflow command examples: + +```bash +poetry run -- nox -s workflow:check -- all +poetry run -- nox -s workflow:generate -- checks +poetry run -- nox -s workflow:audit -- --fix=safe +``` + +## Dependency, package, and matrix sessions + +| Session | Use | Notes | +| --- | --- | --- | +| `dependency:licenses` | Report dependency licenses. | It prints a Markdown license report. | +| `dependency:audit` | Report vulnerabilities. | It prints known vulnerabilities as JSON. | +| `vulnerabilities:update` | Update vulnerable dependencies. | It can write a JSON report file inside the repository. | +| `vulnerabilities:resolved` | Report vulnerabilities that are resolved since the latest tag. | Use it during release preparation. | +| `dependency:sbom` | Generate SBOM files. | It writes `bom.cdx.json` and `bom.spdx.json`. | +| `sonar:check` | Upload artifacts to Sonar. | It uses `SONAR_TOKEN` and prepares `ci-coverage.xml`. | +| `package:check` | Validate the package long description. | It runs `poetry build` and `twine check`. | +| `matrix:generate` | Print selected config values as JSON arrays. | Use this session for matrix output. | diff --git a/exasol/toolbox/skills/exasol-python-toolbox/references/source-routing.md b/exasol/toolbox/skills/exasol-python-toolbox/references/source-routing.md new file mode 100644 index 000000000..1c9559cf1 --- /dev/null +++ b/exasol/toolbox/skills/exasol-python-toolbox/references/source-routing.md @@ -0,0 +1,96 @@ +# Source Routing + +Use this reference to find the source file for exact PTB operation. +Read the source file before you change code. +Read the source file before you explain a detailed rule. + +## Project entry points + +- `pyproject.toml`: package data, dependencies, tool configuration, and + packaged resources. +- `noxfile.py`: local nox entry point. +- `noxconfig.py`: local `PROJECT_CONFIG` and release hooks. +- `README.rst`: project overview. +- `doc/design.rst`: PTB design rules. + +## PTB configuration + +- `exasol/toolbox/config.py`: `BaseConfig`, computed configuration fields, + supported versions, paths, workflow variables, and plugin validation. +- `exasol/toolbox/nox/plugin.py`: nox plugin hooks. + +## PTB nox sessions + +- `exasol/toolbox/nox/_artifacts.py`: artifact validation, artifact copy, and + Sonar upload sessions. +- `exasol/toolbox/nox/_dependencies.py`: dependency audit, license, + vulnerability, and SBOM sessions. +- `exasol/toolbox/nox/_documentation.py`: documentation, link check, and + changelog sessions. +- `exasol/toolbox/nox/_format.py`: `format:fix` and `format:check`. +- `exasol/toolbox/nox/_lint.py`: `lint:code`, `lint:typing`, and + `lint:security`. +- `exasol/toolbox/nox/_matrix.py`: matrix output sessions for CI usage. +- `exasol/toolbox/nox/_package.py`: package validation. +- `exasol/toolbox/nox/_release.py`: release preparation, release update, and + release trigger sessions. +- `exasol/toolbox/nox/_test.py`: `test:unit`, `test:integration`, and + `test:coverage`. +- `exasol/toolbox/nox/_workflow.py`: workflow check, generation, and audit + sessions. + +## PTB nox support files + +- `exasol/toolbox/nox/_shared.py`: old shared nox helpers. Prefer + `exasol/toolbox/util/` for new code that more than one session uses. +- `exasol/toolbox/nox/tasks.py`: exported nox session list. Import new nox + session modules here so projects that use PTB can import them with + `from exasol.toolbox.nox.tasks import *`. Do not add new session code here. + +## PTB tools + +Do not add new functions under `exasol/toolbox/tools/`. Put reusable code in +`exasol/toolbox/util/`. Prefer a nox session for new user-facing operations. + +- `exasol/toolbox/tools/issue.py`: issue template CLI. +- `exasol/toolbox/tools/replace_version.py`: version replacement helpers. +- `exasol/toolbox/tools/security.py`: security issue conversion and creation. +- `exasol/toolbox/tools/tbx.py`: CLI root. +- `exasol/toolbox/tools/template.py`: template list, show, diff, install, and + update helpers. + +## Workflow templates and helpers + +- `exasol/toolbox/templates/github/workflows/`: packaged GitHub workflow + templates. +- `exasol/toolbox/templates/github/`: packaged GitHub templates and zizmor + configuration. +- `exasol/toolbox/util/workflows/`: workflow rendering, patching, validation, + and custom workflow extraction. +- `.workflow-patcher.yml`: PTB workflow patch configuration for this repository. +- `.github/workflows/`: generated repository workflows. +- `.github/actions/`: repository GitHub actions. + +## Release and dependency helpers + +- `exasol/toolbox/util/dependencies/`: dependency audit, license report, + dependency change, vulnerability tracking, and dependency update helpers. +- `exasol/toolbox/util/git.py`: git helper functions. +- `exasol/toolbox/util/release/`: changelog, release notes, and Cookiecutter + version helpers. +- `exasol/toolbox/util/version.py`: version parsing and version upgrade logic. + +## Documentation and project template + +- `doc/user_guide/`: user tasks and feature documentation. +- `doc/developer_guide/`: developer and plugin documentation. +- `doc/api/`: API documentation entry points. +- `doc/changes/`: changelog files. +- `project-template/`: Cookiecutter template for new PTB projects. + +## Tests + +- `test/unit/`: unit tests for PTB operation. +- `test/integration/`: integration tests. +- `test/integration/project-template/`: tests for the Cookiecutter project + template. diff --git a/exasol/toolbox/util/skills.py b/exasol/toolbox/util/skills.py new file mode 100644 index 000000000..e18c83b87 --- /dev/null +++ b/exasol/toolbox/util/skills.py @@ -0,0 +1,29 @@ +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import importlib_resources as resources + +SKILLS_DIRECTORY: Final = "exasol.toolbox.skills" +PTB_SKILL_NAME: Final = "exasol-python-toolbox" + + +def get_skill_path(skill_name: str = PTB_SKILL_NAME) -> Path: + """ + Return the path to a packaged skill. + """ + return Path(str(resources.files(SKILLS_DIRECTORY) / skill_name)) + + +def get_skill_files(skill_name: str = PTB_SKILL_NAME) -> Mapping[str, Path]: + """ + Return packaged skill files. + + The keys are paths relative to the skill root. + """ + skill_path = get_skill_path(skill_name) + return { + str(path.relative_to(skill_path)): path + for path in skill_path.rglob("*") + if path.is_file() + } diff --git a/pyproject.toml b/pyproject.toml index e5d1a28cb..f5a92a47a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,8 +82,7 @@ packages = [ include = [ "README.rst", "doc/changes/changelog.md", - "LICENSE", - "exasol/toolbox/templates/**/*" + "LICENSE" ] [project.urls] diff --git a/test/resources/skills/exasol-python-toolbox/eval_cases.yml b/test/resources/skills/exasol-python-toolbox/eval_cases.yml new file mode 100644 index 000000000..42b2d0e68 --- /dev/null +++ b/test/resources/skills/exasol-python-toolbox/eval_cases.yml @@ -0,0 +1,112 @@ +version: 1 +skill: "exasol-python-toolbox" +cases: + - id: "setup-existing-project" + category: "setup" + prompt: "An existing Python project must start to use PTB. Tell me the correct setup steps." + expected: + must_include: + - "poetry add --group dev exasol-toolbox" + - "noxconfig.py" + - "PROJECT_CONFIG" + - "noxfile.py" + - "from exasol.toolbox.nox.tasks import *" + must_not_include: + - "copy PTB source code" + - "replace PTB automation" + + - id: "fix-format-failure" + category: "quality" + prompt: "A PTB project has format check failures. Tell me the correct first action." + expected: + must_include: + - "format:fix" + - "nox-sessions.md" + must_not_include: + - "run Black directly" + - "run isort directly" + + - id: "run-local-quality-check" + category: "quality" + prompt: "I changed code in a PTB project. Tell me the local checks before commit and push." + expected: + must_include: + - ".pre-commit-config.yaml" + - "pre-commit" + - "pre-push" + - "project:check" + - "nox-sessions.md" + must_not_include: + - "run Pytest directly" + - "test/unit/path_test.py" + + - id: "prepare-release" + category: "release" + prompt: "Prepare a patch release for a PTB project. Tell me the safe workflow." + expected: + must_include: + - "release:prepare" + - "--type patch" + - "review" + - "changelog" + must_not_include: + - "release:trigger" + - "push tags" + + - id: "trigger-release-safety" + category: "release" + prompt: "The release PR is merged. Tell me the safety checks before I trigger the release." + expected: + must_include: + - "release:trigger" + - "default branch" + - "latest remote commit" + - "no git tag" + - "no GitHub release" + - "latest release" + - "user asks" + must_not_include: + - "run without approval" + - "skip repository checks" + + - id: "workflow-template-update" + category: "workflow" + prompt: "A generated or patched GitHub workflow changed in a PTB project. Tell me what to do." + expected: + must_include: + - "PTB-provided workflows" + - "Custom workflows" + - "workflow:check" + - "workflow:generate" + - ".workflow-patcher.yml" + - "permissions" + - "on.workflow_call.secrets" + must_not_include: + - "edit generated workflow files first" + - "manual workflow edit first" + + - id: "update-ptb-dependency" + category: "update" + prompt: "Update exasol-toolbox in a PTB project. Tell me the validation steps." + expected: + must_include: + - "pyproject.toml" + - "poetry update exasol-toolbox" + - "workflow:generate" + - "workflow:check" + - "doc/changes/unreleased.md" + must_not_include: + - "ignore migration notes" + - "skip validation" + + - id: "exact-source-behavior" + category: "source-routing" + prompt: "I need exact behavior for release preparation. Tell me which source file to read." + expected: + must_include: + - "source-routing.md" + - "exasol/toolbox/nox/_release.py" + - "read the source file" + must_not_include: + - "guess" + - "copy PTB source code" diff --git a/test/unit/skills_test.py b/test/unit/skills_test.py new file mode 100644 index 000000000..56c4980b5 --- /dev/null +++ b/test/unit/skills_test.py @@ -0,0 +1,180 @@ +from pathlib import Path +from subprocess import run +from zipfile import ZipFile + +from ruamel.yaml import YAML + +from exasol.toolbox.util.skills import ( + PTB_SKILL_NAME, + get_skill_files, + get_skill_path, +) + +PROJECT_ROOT = Path(__file__).parents[2] +SKILL = get_skill_path(PTB_SKILL_NAME) +SKILL_FILES = [ + "SKILL.md", + "references/coding-guidelines.md", + "references/common-workflows.md", + "references/nox-sessions.md", + "references/source-routing.md", +] +EVAL_CASES = ( + PROJECT_ROOT + / "test" + / "resources" + / "skills" + / "exasol-python-toolbox" + / "eval_cases.yml" +) + + +def _load_eval_cases() -> dict: + return YAML(typ="safe").load(EVAL_CASES) + + +def test_ptb_skill_resources_are_available(): + skill_files = get_skill_files(PTB_SKILL_NAME) + + for expected in SKILL_FILES: + assert expected in skill_files + assert skill_files[expected].is_file() + + +def test_ptb_skill_resources_are_packaged(tmp_path): + build_output = tmp_path / "dist" + result = run( + [ + "poetry", + "build", + "--project", + str(PROJECT_ROOT), + "--format", + "wheel", + "--output", + str(build_output), + ], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + + wheels = list(build_output.glob("*.whl")) + assert len(wheels) == 1 + + with ZipFile(wheels[0]) as wheel: + wheel_files = set(wheel.namelist()) + + expected_files = { + f"exasol/toolbox/skills/{PTB_SKILL_NAME}/{path}" for path in SKILL_FILES + } + assert expected_files <= wheel_files + + +def test_ptb_skill_frontmatter_is_complete(): + content = (SKILL / "SKILL.md").read_text(encoding="utf-8") + frontmatter = content.split("---", maxsplit=2)[1] + + assert "name: exasol-python-toolbox" in frontmatter + assert "description: Use this skill in Exasol Python projects" in frontmatter + assert "[TODO" not in content + + +def test_ptb_skill_has_no_main_branch_metadata(): + forbidden = [ + "main-branch", + "main branch", + "master-branch", + "master branch", + "inventory", + "source-map", + ] + content = "\n".join( + path.read_text(encoding="utf-8") for path in SKILL.rglob("*") if path.is_file() + ).lower() + + for term in forbidden: + assert term not in content + + +def test_ptb_skill_has_no_duplicate_markdown_lines(): + for path in SKILL.rglob("*.md"): + seen = {} + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), 1 + ): + normalized = line.strip().lower() + if ( + not normalized + or normalized in {"---", "```bash", "```"} + or normalized.startswith("|") + ): + continue + assert normalized not in seen, ( + f"{path} duplicates line {seen[normalized]} at line {line_number}: " + f"{line}" + ) + seen[normalized] = line_number + + +def test_nox_command_syntax_is_only_in_nox_session_reference(): + nox_reference = SKILL / "references" / "nox-sessions.md" + for path in SKILL.rglob("*"): + if not path.is_file() or path == nox_reference: + continue + + content = path.read_text(encoding="utf-8") + assert "poetry run -- nox -s" not in content + assert "poetry run -- nox -l" not in content + + +def test_ptb_skill_eval_cases_are_valid(): + eval_cases = _load_eval_cases() + + assert eval_cases["version"] == 1 + assert eval_cases["skill"] == "exasol-python-toolbox" + # Keep enough cases to cover the ticket scope, but not so many that the + # deterministic eval file becomes hard to review. + assert 6 <= len(eval_cases["cases"]) <= 8 + + ids = [case["id"] for case in eval_cases["cases"]] + assert len(ids) == len(set(ids)) + + for case in eval_cases["cases"]: + assert case["id"] + assert case["category"] + assert case["prompt"] + assert case["expected"]["must_include"] + assert case["expected"]["must_not_include"] + + +def test_ptb_skill_eval_cases_cover_ticket_scope(): + eval_cases = _load_eval_cases() + categories = {case["category"] for case in eval_cases["cases"]} + + assert { + "setup", + "quality", + "release", + "update", + "workflow", + "source-routing", + }.issubset(categories) + + +def test_ptb_skill_eval_cases_do_not_define_llm_ci_execution(): + content = EVAL_CASES.read_text(encoding="utf-8").lower() + + forbidden = [ + "model:", + "api_key", + "openai", + "chatgpt", + "codex", + "temperature", + ] + + for term in forbidden: + assert term not in content