diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..d66839a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,69 @@ +name: Publish to PyPI + +on: + push: + tags: + - 'v[0-9]*' + +concurrency: + group: pypi-publish-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.12' + + - name: Install build tooling + run: python -m pip install --upgrade build twine + + - name: Verify tag matches pyproject version + run: | + if [[ ! "$GITHUB_REF_NAME" =~ ^v[0-9] ]]; then + echo "Release tag '$GITHUB_REF_NAME' must start with 'v' followed by a digit (e.g. v1.0.0)" >&2 + exit 1 + fi + tag="${GITHUB_REF_NAME#v}" + pkg_version=$(python -c "import tomllib,pathlib; print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])") + if [ "$tag" != "$pkg_version" ]; then + echo "Release tag ($tag) does not match pyproject.toml version ($pkg_version)" >&2 + exit 1 + fi + + - name: Build sdist and wheel + run: python -m build + + - name: Check distribution metadata + run: python -m twine check --strict dist/* + + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/hotdata-materialized + permissions: + id-token: write + steps: + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 + with: + name: dist + path: dist/ + + - name: Publish via Trusted Publishing + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..55d5d86 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +name: GitHub Release + +on: + push: + tags: + - 'v[0-9]*' + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to create or update a GitHub Release for (e.g. v0.2.2)' + required: true + type: string + +# Deny all permissions by default; grant only what each job needs. +permissions: {} + +jobs: + release: + name: Create GitHub Release + runs-on: ubuntu-latest + permissions: + contents: write # create/update the GitHub Release and read the tagged ref + env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.tag || github.ref_name }} + steps: + - name: Validate release tag format + if: github.event_name == 'workflow_dispatch' + env: + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + set -euo pipefail + if [[ ! "$INPUT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "tag must look like vX.Y.Z, got: $INPUT_TAG" >&2 + exit 1 + fi + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ env.RELEASE_TAG }} + fetch-depth: 0 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.12' + + - name: Read package metadata + id: meta + run: | + pkg_name=$(python -c "import tomllib,pathlib; print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['name'])") + pkg_version="${RELEASE_TAG#v}" + echo "name=${pkg_name}" >> "$GITHUB_OUTPUT" + echo "version=${pkg_version}" >> "$GITHUB_OUTPUT" + + - name: Extract changelog notes + id: notes + run: | + set -euo pipefail + version="${RELEASE_TAG#v}" + if [[ -f CHANGELOG.md ]]; then + body="$(python scripts/extract-changelog.py "$version")" + else + body="Release ${version}." + fi + delimiter="EOF_${RANDOM}_${RANDOM}" + { + echo "body<<${delimiter}" + echo "$body" + echo "${delimiter}" + } >> "$GITHUB_OUTPUT" + + - name: Create GitHub Release + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0 + with: + tag_name: ${{ env.RELEASE_TAG }} + name: ${{ steps.meta.outputs.name }} ${{ steps.meta.outputs.version }} + body: ${{ steps.notes.outputs.body }} + generate_release_notes: false + make_latest: true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bc88b1c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +- `@materialize` decorator and `MaterializedFrame`: write-behind caching of + expensive Django query results into Hotdata; hits return Arrow-backed + frames (`.arrow()/.df()/.to_pylist()`), `.sql()` runs server-side against + the cached entry +- BM25 and vector search on cached entries: `@materialize(index=BM25(...))` / + `Vector(..., provider=...)` declarations, `frame.search()` and + `frame.vector_search()` +- Content-addressed fingerprinting for querysets and function calls +- Remote registry (no migrations or local state in the host app) and + one-managed-database-per-entry storage with TTL expiry backstop +- TPC-H demo project benchmarking direct Postgres vs the cache diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..2430f7f --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,44 @@ +# Releasing + +Every release uses `./scripts/release.sh`. Do not bump versions, tag, or create GitHub Releases manually. + +## One-time setup + +- Install [GitHub CLI](https://cli.github.com/) (`gh`) and authenticate. +- Ensure PyPI [trusted publishing](https://docs.pypi.org/trusted-publishers/) is configured for this repo (`publish.yml` uses the `pypi` GitHub environment). + +## Release steps + +1. Add user-facing notes under `## [Unreleased]` in `CHANGELOG.md`. +2. Prepare the release PR: + + ```bash + ./scripts/release.sh prepare patch # or minor | major | X.Y.Z + ``` + + This bumps the version in `pyproject.toml` and + `hotdata_materialized/__init__.py`, rolls the changelog, and opens the PR. + +3. Merge the PR after CI passes. +4. Publish from a clean `main` checkout: + + ```bash + git checkout main + git pull + ./scripts/release.sh publish + ``` + +## What happens automatically + +Pushing a `vX.Y.Z` tag triggers two workflows: + +| Workflow | Purpose | +|----------|---------| +| `publish.yml` | Build wheel/sdist, verify tag↔version, publish to PyPI via trusted publishing | +| `release.yml` | Create the GitHub Release with notes from `CHANGELOG.md` | + +## Recover a missing GitHub Release + +If PyPI publish succeeded but the GitHub Release workflow failed, rerun +`release.yml` from the Actions tab via `workflow_dispatch` with the existing +tag — no retagging needed. diff --git a/scripts/extract-changelog.py b/scripts/extract-changelog.py new file mode 100755 index 0000000..c2caef1 --- /dev/null +++ b/scripts/extract-changelog.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Print the Keep a Changelog section for a release version.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +def extract(changelog: str, version: str) -> str: + pattern = rf"^## \[{re.escape(version)}\].*$" + match = re.search(pattern, changelog, re.M) + if not match: + raise SystemExit(f"no changelog section for {version}") + + start = match.start() + rest = changelog[match.end() :] + next_heading = re.search(r"^## \[", rest, re.M) + end = match.end() + (next_heading.start() if next_heading else len(rest)) + section = changelog[start:end].strip() + title, _, body = section.partition("\n") + return body.strip() or f"Release {version}." + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: extract-changelog.py VERSION") + + version = sys.argv[1] + changelog = Path("CHANGELOG.md").read_text() + print(extract(changelog, version)) + + +if __name__ == "__main__": + main() diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..5275db5 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +die() { echo "error: $*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "$1 is required"; } + +usage() { + cat <<'EOF' +Usage: + ./scripts/release.sh prepare [patch|minor|major|X.Y.Z] + ./scripts/release.sh publish + +Workflow: + 1. Move notes from [Unreleased] in CHANGELOG.md (or add them there). + 2. ./scripts/release.sh prepare patch + 3. Merge the release PR. + 4. ./scripts/release.sh publish + +Tag push triggers PyPI publish and GitHub Release creation in CI. +EOF +} + +get_version() { + python3 - <<'PY' +import tomllib +from pathlib import Path +print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]) +PY +} + +get_pkg_name() { + python3 - <<'PY' +import tomllib +from pathlib import Path +print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["name"]) +PY +} + +set_version() { + local ver="$1" + python3 - "$ver" <<'PY' +import re, sys +from pathlib import Path +ver = sys.argv[1] +path = Path("pyproject.toml") +text = path.read_text() +new, n = re.subn(r'(?m)^version = "[^"]+"', f'version = "{ver}"', text, count=1) +if n != 1: + raise SystemExit("could not update version in pyproject.toml") +path.write_text(new) + +# __version__ is hardcoded (and checked against pyproject by the test suite) +init = Path("hotdata_materialized/__init__.py") +text = init.read_text() +new, n = re.subn(r'(?m)^__version__ = "[^"]+"', f'__version__ = "{ver}"', text, count=1) +if n != 1: + raise SystemExit("could not update __version__ in hotdata_materialized/__init__.py") +init.write_text(new) +PY +} + +bump_version() { + local kind="$1" current="$2" + python3 - "$kind" "$current" <<'PY' +import re, sys +kind, current = sys.argv[1], sys.argv[2] +match = re.match(r"^(\d+)\.(\d+)\.(\d+)(.*)$", current) +if not match: + raise SystemExit(f"unsupported version: {current}") +major, minor, patch, suffix = int(match[1]), int(match[2]), int(match[3]), match[4] +if suffix: + raise SystemExit("pre-release versions must be set explicitly as X.Y.Z") +if kind == "patch": + patch += 1 +elif kind == "minor": + minor += 1 + patch = 0 +elif kind == "major": + major += 1 + minor = 0 + patch = 0 +else: + raise SystemExit(f"unknown bump kind: {kind}") +print(f"{major}.{minor}.{patch}") +PY +} + +default_branch() { + local remote="${1:-origin}" + git symbolic-ref --quiet "refs/remotes/${remote}/HEAD" 2>/dev/null | sed "s|refs/remotes/${remote}/||" \ + || { git branch -r | sed -n "s|^ ${remote}/\\(main\\|master\\)$|\\1|p" | head -1; } \ + || echo main +} + +ensure_clean() { + [[ -z "$(git status --porcelain)" ]] || die "working tree is not clean" +} + +update_changelog() { + local ver="$1" + local date + date="$(date +%Y-%m-%d)" + python3 scripts/update_changelog.py "$ver" "$date" +} + +cmd_prepare() { + local bump="${1:-}" + [[ -n "$bump" ]] || { usage; die "missing bump kind or explicit version"; } + need gh + need python3 + ensure_clean + + local current new base branch pkg + current="$(get_version)" + if [[ "$bump" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + new="$bump" + else + new="$(bump_version "$bump" "$current")" + fi + [[ "$new" != "$current" ]] || die "new version ($new) equals current ($current)" + + base="$(default_branch)" + git fetch origin "$base" + git checkout "$base" + git pull --ff-only origin "$base" + ensure_clean + + set_version "$new" + update_changelog "$new" + if command -v uv >/dev/null 2>&1 && [[ -f uv.lock ]]; then + uv lock + fi + + branch="release/v${new}" + git checkout -b "$branch" + git add pyproject.toml CHANGELOG.md hotdata_materialized/__init__.py + [[ -f uv.lock ]] && git add uv.lock + git commit -m "chore: release v${new}" + + pkg="$(get_pkg_name)" + git push -u origin "$branch" + gh pr create --base "$base" --head "$branch" \ + --title "chore: release ${pkg} v${new}" \ + --body "## Summary +Release **${pkg} v${new}**. + +## Checklist +- [x] Version bumped in \`pyproject.toml\` +- [x] \`CHANGELOG.md\` updated +- [ ] CI green + +After merge, run \`./scripts/release.sh publish\` from a clean \`${base}\` checkout." + + echo "Prepared ${pkg} v${new}. Merge the PR, then run: ./scripts/release.sh publish" +} + +cmd_publish() { + need gh + need python3 + ensure_clean + + local base ver tag + base="$(default_branch)" + git fetch origin "$base" + git checkout "$base" + git pull --ff-only origin "$base" + ensure_clean + + ver="$(get_version)" + tag="v${ver}" + + git rev-parse "$tag" >/dev/null 2>&1 && die "tag $tag already exists" + [[ -f CHANGELOG.md ]] || die "CHANGELOG.md is required" + python3 - "$ver" <<'PY' +import re, sys +from pathlib import Path +ver = sys.argv[1] +text = Path("CHANGELOG.md").read_text() +if not re.search(rf"^## \[{re.escape(ver)}\]", text, re.M): + raise SystemExit(f"CHANGELOG.md missing section for {ver}") +PY + + git tag "$tag" + git push origin "$tag" + + pkg="$(get_pkg_name)" + echo "Pushed ${tag} for ${pkg}." + echo "CI will publish to PyPI and create the GitHub Release." +} + +case "${1:-}" in + prepare) shift; cmd_prepare "${1:-}" ;; + publish) cmd_publish ;; + -h|--help|help|"") usage ;; + *) usage; die "unknown command: $1" ;; +esac diff --git a/scripts/update_changelog.py b/scripts/update_changelog.py new file mode 100755 index 0000000..b419242 --- /dev/null +++ b/scripts/update_changelog.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Update CHANGELOG.md for a new release version.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +def update_changelog_text(text: str, ver: str, date: str) -> str: + if re.search(rf"^## \[{re.escape(ver)}\]", text, re.M): + return text + + unreleased = re.search(r"^## \[Unreleased\]\s*\n(.*?)(?=^## \[|\Z)", text, re.M | re.S) + if unreleased: + body = unreleased.group(1).strip() + if body: + section = f"## [{ver}] - {date}\n\n{body}\n\n" + else: + section = ( + f"## [{ver}] - {date}\n\n" + "### Changed\n\n" + f"- Release {ver}\n\n" + ) + return re.sub( + r"^## \[Unreleased\]\s*\n.*?(?=^## \[|\Z)", + lambda _match: f"## [Unreleased]\n\n{section}", + text, + count=1, + flags=re.M | re.S, + ) + + section = ( + f"## [Unreleased]\n\n" + f"## [{ver}] - {date}\n\n" + "### Changed\n\n" + f"- Release {ver}\n\n" + ) + first_heading = re.search(r"^## \[", text, re.M) + if first_heading: + pos = first_heading.start() + return text[:pos] + section + text[pos:] + return text.rstrip() + "\n\n" + section + + +def update_changelog_file(path: Path, ver: str, date: str) -> None: + if not path.exists(): + path.write_text( + "# Changelog\n\n" + "All notable changes to this project will be documented in this file.\n\n" + "The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),\n" + "and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n\n" + f"## [Unreleased]\n\n" + f"## [{ver}] - {date}\n\n" + "### Changed\n\n" + f"- Release {ver}\n" + ) + return + + path.write_text(update_changelog_text(path.read_text(), ver, date)) + + +def main() -> None: + if len(sys.argv) != 3: + raise SystemExit("usage: update_changelog.py VERSION YYYY-MM-DD") + update_changelog_file(Path("CHANGELOG.md"), sys.argv[1], sys.argv[2]) + + +if __name__ == "__main__": + main()