From a085e26c5f6234f1621e85bba5e4f4b184540932 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Thu, 30 Jul 2026 18:17:56 -0400 Subject: [PATCH 01/14] Handle editable uv path dependencies --- licensecheck/packageinforesolver.py | 64 ++++++++++- tests/test_packageinfo.py | 160 ++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 2 deletions(-) diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index 37c4f39..ab42ce9 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -30,6 +30,63 @@ HTTP_OK = 200 +def _source_path_name(path: object) -> str | None: + if not isinstance(path, str): + return None + + path_name = Path(path).name + return path_name if re.fullmatch(r"(?!-)[A-Za-z0-9_.-]+", path_name) else None + + +def _source_options(source: object) -> list[object]: + return list(source) if isinstance(source, list) else [source] + + +def _nested_pyproject(requirements_path: Path, path: object) -> Path | None: + if not isinstance(path, str): + return None + + source_path = (requirements_path.parent / path).resolve() + return source_path if source_path.name == "pyproject.toml" else source_path / "pyproject.toml" + + +def _uv_sources(requirements_path: Path) -> dict[str, object]: + try: + pyproject = tomli.loads(requirements_path.read_text(encoding="utf-8")) + except (OSError, tomli.TOMLDecodeError): + return {} + + sources = pyproject.get("tool", {}).get("uv", {}).get("sources", {}) + return sources if isinstance(sources, dict) else {} + + +def _editable_uv_sources(requirements_path: Path, visited: set[Path] | None = None) -> set[str]: + requirements_path = requirements_path.resolve() + if visited is None: + visited = set() + if requirements_path.name != "pyproject.toml" or requirements_path in visited: + return set() + visited.add(requirements_path) + + editable_sources: set[str] = set() + + for name, source in _uv_sources(requirements_path).items(): + for source_option in _source_options(source): + if not isinstance(source_option, dict): + continue + + path = source_option.get("path") + if source_option.get("editable"): + editable_sources.add(name) + if path_name := _source_path_name(path): + editable_sources.add(path_name) + + if nested_pyproject := _nested_pyproject(requirements_path, path): + editable_sources.update(_editable_uv_sources(nested_pyproject, visited)) + + return editable_sources + + class PackageInfoManager: """Manages retrieval of local and remote package information.""" @@ -51,12 +108,15 @@ def resolve_requirements( skip_dependencies: set[str], ) -> None: for requirements_path in requirements_paths: + requirements_path_obj = Path(requirements_path) + # DepGather cannot parse the `-e path` lines uv emits for editable sources. self.reqs.update( gather( - skipDependencies=skip_dependencies, + skipDependencies=skip_dependencies + | _editable_uv_sources(requirements_path_obj), groups=groups, extras=extras, - requirementsPath=Path(requirements_path), + requirementsPath=requirements_path_obj, base_index_url=self.base_pypi_url, ) ) diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 2bc454a..9026eab 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -128,3 +128,163 @@ def test_unpinned_requirement_does_not_crash(package_info_manager: PackageInfoMa assert package.name == "sample" assert package.errorCode == 0 + + +def test_resolve_requirements_handles_nested_editable_uv_sources( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + transitive_path = tmp_path / "transitive_dependency" + transitive_path.mkdir() + (transitive_path / "pyproject.toml").write_text( + """ +[project] +name = "transitive-dependency" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + nested_path = tmp_path / "nested_workspace_member" + nested_path.mkdir() + (nested_path / "pyproject.toml").write_text( + """ +[project] +name = "nested-dependency" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + dependency_path = tmp_path / "workspace_member" + dependency_path.mkdir() + (dependency_path / "pyproject.toml").write_text( + f""" +[project] +name = "local-dependency" +version = "1.0.0" +dependencies = [ + "nested-dependency", + "transitive-dependency @ {transitive_path.as_uri()}", +] + +[tool.uv.sources] +nested-dependency = {{ path = "../nested_workspace_member", editable = true }} +""".strip(), + encoding="utf-8", + ) + + project_path = tmp_path / "project" + project_path.mkdir() + pyproject_path = project_path / "pyproject.toml" + pyproject_path.write_text( + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../workspace_member", editable = true } +""".strip(), + encoding="utf-8", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "transitive-dependency" + } + + +def test_resolve_requirements_handles_editable_uv_sources_in_monorepo( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + repository_path = tmp_path / "repository" + libraries_path = repository_path / "libs" + + published_path = repository_path / "published_dependency" + published_path.mkdir(parents=True) + (published_path / "pyproject.toml").write_text( + """ +[project] +name = "published-dependency" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + sdk_path = libraries_path / "sdk_extensions" + sdk_path.mkdir(parents=True) + (sdk_path / "pyproject.toml").write_text( + f""" +[project] +name = "sdk-extensions" +version = "1.0.0" +dependencies = ["published-dependency @ {published_path.as_uri()}"] +""".strip(), + encoding="utf-8", + ) + + anomaly_path = libraries_path / "anomaly_detection" + anomaly_path.mkdir() + (anomaly_path / "pyproject.toml").write_text( + """ +[project] +name = "anomaly-detection" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + common_path = libraries_path / "service_common" + common_path.mkdir() + (common_path / "pyproject.toml").write_text( + """ +[project] +name = "service-common" +version = "1.0.0" +dependencies = ["sdk-extensions", "anomaly-detection"] + +[tool.uv.sources] +sdk-extensions = { path = "../sdk_extensions", editable = true } +anomaly-detection = { path = "../anomaly_detection", editable = true } +""".strip(), + encoding="utf-8", + ) + + function_path = repository_path / "functions" / "reconciliation" + function_path.mkdir(parents=True) + pyproject_path = function_path / "pyproject.toml" + pyproject_path.write_text( + """ +[project] +name = "reconciliation" +version = "1.0.0" +dependencies = ["sdk-extensions", "service-common", "anomaly-detection"] + +[tool.uv.sources] +sdk-extensions = { path = "../../libs/sdk_extensions", editable = true } +service-common = { path = "../../libs/service_common", editable = true } +anomaly-detection = { path = "../../libs/anomaly_detection", editable = true } + +[tool.uv] +package = false +""".strip(), + encoding="utf-8", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "published-dependency" + } From 53bcaed994bbbd39a4cdb4885b26ac99db1e4060 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Thu, 30 Jul 2026 19:47:52 -0400 Subject: [PATCH 02/14] Filters actual editable lines instead of guessing package names from configuration. --- licensecheck/packageinforesolver.py | 122 ++++++++------- pyproject.toml | 1 + tests/test_packageinfo.py | 224 ++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 57 deletions(-) diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index ab42ce9..9da981b 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -5,6 +5,7 @@ import configparser import contextlib import re +import subprocess from concurrent.futures import ThreadPoolExecutor from email.message import Message from importlib import metadata @@ -14,6 +15,7 @@ import license_expression import requests +import requirements import tomli from boolean.boolean import Expression from depgather.models.pypijson import ProjectResponse @@ -30,61 +32,60 @@ HTTP_OK = 200 -def _source_path_name(path: object) -> str | None: - if not isinstance(path, str): - return None - - path_name = Path(path).name - return path_name if re.fullmatch(r"(?!-)[A-Za-z0-9_.-]+", path_name) else None - +def _parse_uv_requirements(raw_requirements: str, skip_dependencies: set[str]) -> set[Requirement]: + skip_names = {canonicalize_name(name) for name in skip_dependencies} + parsed_requirements: set[Requirement] = set() + + for parsed in requirements.parse(raw_requirements): + if parsed.editable: + continue + if not parsed.name or canonicalize_name(parsed.name) in skip_names: + continue + + requirement = Requirement(parsed.line) + requirement.name = canonicalize_name(requirement.name) + parsed_requirements.add(requirement) + + return parsed_requirements + + +def _gather_uv_requirements( + requirements_path: Path, + groups: set[str], + extras: set[str], + skip_dependencies: set[str], + base_index_url: str, +) -> set[Requirement]: + command = [ + "uv", + "pip", + "compile", + "--color", + "never", + "--index", + base_index_url, + requirements_path.as_posix(), + ] + for group in groups: + command.extend(["--group", group]) + for extra in extras: + command.extend(["--extra", extra]) -def _source_options(source: object) -> list[object]: - return list(source) if isinstance(source, list) else [source] - - -def _nested_pyproject(requirements_path: Path, path: object) -> Path | None: - if not isinstance(path, str): - return None - - source_path = (requirements_path.parent / path).resolve() - return source_path if source_path.name == "pyproject.toml" else source_path / "pyproject.toml" - - -def _uv_sources(requirements_path: Path) -> dict[str, object]: try: - pyproject = tomli.loads(requirements_path.read_text(encoding="utf-8")) - except (OSError, tomli.TOMLDecodeError): - return {} - - sources = pyproject.get("tool", {}).get("uv", {}).get("sources", {}) - return sources if isinstance(sources, dict) else {} - - -def _editable_uv_sources(requirements_path: Path, visited: set[Path] | None = None) -> set[str]: - requirements_path = requirements_path.resolve() - if visited is None: - visited = set() - if requirements_path.name != "pyproject.toml" or requirements_path in visited: - return set() - visited.add(requirements_path) - - editable_sources: set[str] = set() - - for name, source in _uv_sources(requirements_path).items(): - for source_option in _source_options(source): - if not isinstance(source_option, dict): - continue - - path = source_option.get("path") - if source_option.get("editable"): - editable_sources.add(name) - if path_name := _source_path_name(path): - editable_sources.add(path_name) + result = subprocess.run( # noqa: S603 + command, + capture_output=True, + text=True, + check=False, + ) + except OSError as error: + raise RuntimeError from error - if nested_pyproject := _nested_pyproject(requirements_path, path): - editable_sources.update(_editable_uv_sources(nested_pyproject, visited)) + if result.returncode != 0: + message = f"Non-zero returncode: {result.stderr}, {result.stdout}" + raise RuntimeError(message) - return editable_sources + return _parse_uv_requirements(result.stdout, skip_dependencies) class PackageInfoManager: @@ -109,17 +110,24 @@ def resolve_requirements( ) -> None: for requirements_path in requirements_paths: requirements_path_obj = Path(requirements_path) - # DepGather cannot parse the `-e path` lines uv emits for editable sources. - self.reqs.update( - gather( - skipDependencies=skip_dependencies - | _editable_uv_sources(requirements_path_obj), + try: + resolved_requirements = _gather_uv_requirements( + requirements_path=requirements_path_obj, + groups=groups, + extras=extras, + skip_dependencies=skip_dependencies, + base_index_url=self.base_pypi_url, + ) + except RuntimeError: + resolved_requirements = gather( + skipDependencies=skip_dependencies, groups=groups, extras=extras, requirementsPath=requirements_path_obj, base_index_url=self.base_pypi_url, ) - ) + + self.reqs.update(resolved_requirements) def getPackages(self) -> set[PackageInfo]: """ diff --git a/pyproject.toml b/pyproject.toml index 61efbe7..df4b73b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic>=2.13.4", "requests>=2.32.5", "requests-cache>=1.3.2", + "requirements-parser>=0.13.1", "rich>=15.0.0", "tomli>=2.2.1", ] diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 9026eab..504f266 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from subprocess import CompletedProcess import pytest from packaging.requirements import Requirement @@ -38,6 +39,13 @@ def aux_packageinfo(package_name: str) -> PackageInfo: return PackageInfo(name=package_name) +def write_pyproject(directory: Path, contents: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + pyproject_path = directory / "pyproject.toml" + pyproject_path.write_text(contents.strip(), encoding="utf-8") + return pyproject_path + + requests_package = aux_packageinfo("requests") @@ -288,3 +296,219 @@ def test_resolve_requirements_handles_editable_uv_sources_in_monorepo( assert {requirement.name for requirement in package_info_manager.reqs} == { "published-dependency" } + + +def test_resolve_requirements_does_not_skip_inactive_editable_source( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["idna==3.10"] + +[tool.uv.sources] +idna = { + path = "../local-idna", + editable = true, + marker = "python_version < '0'", +} +""", + ) + + def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=["uv", "pip", "compile"], + returncode=0, + stdout="idna==3.10\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {str(requirement) for requirement in package_info_manager.reqs} == {"idna==3.10"} + + +def test_resolve_requirements_keeps_package_sharing_editable_directory_name( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + published_path = tmp_path / "published_idna" + write_pyproject( + published_path, + """ +[project] +name = "idna" +version = "3.10" +""", + ) + + editable_path = tmp_path / "idna" + write_pyproject( + editable_path, + """ +[project] +name = "internal-helper" +version = "1.0.0" +""", + ) + + pyproject_path = write_pyproject( + tmp_path / "project", + f""" +[project] +name = "project" +version = "1.0.0" +dependencies = [ + "internal-helper", + "idna @ {published_path.as_uri()}", +] + +[tool.uv.sources] +internal-helper = {{ path = "../idna", editable = true }} +""", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == {"idna"} + + +def test_resolve_requirements_handles_editable_uv_workspace_source( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + published_path = tmp_path / "published_dependency" + write_pyproject( + published_path, + """ +[project] +name = "published-dependency" +version = "1.0.0" +""", + ) + + workspace_path = tmp_path / "workspace" + write_pyproject( + workspace_path / "packages" / "local_dependency", + f""" +[project] +name = "local-dependency" +version = "1.0.0" +dependencies = ["published-dependency @ {published_path.as_uri()}"] +""", + ) + pyproject_path = write_pyproject( + workspace_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { workspace = true } + +[tool.uv.workspace] +members = ["packages/local_dependency"] +""", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "published-dependency" + } + + +def test_resolve_requirements_handles_editable_path_with_spaces( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + published_path = tmp_path / "published_dependency" + write_pyproject( + published_path, + """ +[project] +name = "published-dependency" +version = "1.0.0" +""", + ) + + write_pyproject( + tmp_path / "local dependency", + f""" +[project] +name = "local-dependency" +version = "1.0.0" +dependencies = ["published-dependency @ {published_path.as_uri()}"] +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../local dependency", editable = true } +""", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "published-dependency" + } + + +def test_resolve_requirements_falls_back_for_uv_lock( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + lock_path = tmp_path / "uv.lock" + lock_path.write_text( + """ +version = 1 + +[[package]] +name = "fallback-package" +version = "1.2.3" +""".strip(), + encoding="utf-8", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(lock_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {str(requirement) for requirement in package_info_manager.reqs} == { + "fallback-package==1.2.3" + } From 7ea5294bd16a3c03988ff53e624f2b708f19743c Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Fri, 31 Jul 2026 07:26:29 -0400 Subject: [PATCH 03/14] Add support for private package repositories --- README.md | 8 +- documentation/user/README.md | 26 +- licensecheck/checker.py | 16 +- licensecheck/io/cli.py | 10 +- licensecheck/license_matrix/__init__.py | 3 + licensecheck/packageinforesolver.py | 251 +++++++++++++++++-- tests/test_checker.py | 34 +++ tests/test_license_matrix.py | 8 + tests/test_packageinfo.py | 305 ++++++++++++++++++++++++ 9 files changed, 617 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 7addf99..fb70461 100644 --- a/README.md +++ b/README.md @@ -169,9 +169,9 @@ options: Select extras from supported files --file FILE, -o FILE Filename to write output to (omit this for stdout) --ignore-packages IGNORE_PACKAGES [IGNORE_PACKAGES ...] - List of packages/dependencies to ignore (compat=True), globs are supported + List of packages/dependencies to ignore (compat=True); names, name==version, and globs are supported --fail-packages FAIL_PACKAGES [FAIL_PACKAGES ...] - List of packages/dependencies to fail (compat=False), globs are supported + List of packages/dependencies to fail (compat=False); names, name==version, and globs are supported --ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...] List of licenses to ignore (skipped, compat may still be False) --fail-licenses FAIL_LICENSES [FAIL_LICENSES ...] @@ -210,8 +210,8 @@ requirements_paths = [] # List of filenames to read from groups = [] # List of selected groups extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) -ignore_packages = [] # Packages/dependencies to ignore -fail_packages = [] # Packages/dependencies that cause failure +ignore_packages = [] # Names, name==version entries, or globs to ignore +fail_packages = [] # Names, name==version entries, or globs that fail ignore_licenses = [] # Licenses to ignore fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) diff --git a/documentation/user/README.md b/documentation/user/README.md index 128b3e4..be21259 100644 --- a/documentation/user/README.md +++ b/documentation/user/README.md @@ -91,9 +91,10 @@ uv run licensecheck --only-licenses mit apache --show-only-failing -g dev ## Supported tools/ standards Licensecheck supports a broad range of different tools and workflows. Though please note that -for some of these tools, behaviour may differ from what is expected. We use `uv` for the dependency -resolution due to the good performance across projects, with a fallback to a native parser in case of -an error, which will be logged +for some of these tools, behaviour may differ from what is expected. For a `pyproject.toml` with an +adjacent `uv.lock`, Licensecheck exports the locked dependency graph. Otherwise it uses `uv` to +resolve dependencies. Resolution errors for a `pyproject.toml` are reported directly instead of +falling back to a less accurate dependency graph. Note that `uv` supports requirements.in files. If a pyproject.toml, setup.py, or setup.cfg file is provided, `uv` will extract the requirements for the relevant project. In testing this seems to have @@ -158,19 +159,20 @@ classifiers = [ Previous versions of the licensecheck tool implemented a custom resolver to discover packages. Current versions look to move away from this for a number of reasons, such as correctness and reducing the maintenance burden. Over time many contributors helped out with the custom -resolver which is very much appreciated. Now, we use `uv` to attempt to resolve deps before -falling back to the legacy approach, which is needed in certain cases where uv fails +resolver which is very much appreciated. Now, we use `uv` to export an adjacent lockfile or resolve +dependencies from the supplied project file. The legacy parser remains for formats that `uv pip +compile` does not accept directly. Q: Why doesn't this use packages from my lockfile? -A: The answer to this somewhat depends on what resolver licensecheck ends up using to -find all of the packages in use by your project. Ideally, `uv` is used which has pretty -good support for pyproject.toml and some other standard requirements formats and will discover -packages. The legacy resolver is deprecated and may result in funky output in some cases +A: When a `uv.lock` is adjacent to a supplied `pyproject.toml`, Licensecheck uses `uv export +--locked` and audits those exact versions. Without an adjacent lockfile it resolves the supplied +requirements with `uv`. Q: The license for my dep has changed in v >1.0, so I'm using v < 1.0, why doesn't licensecheck report the correct license version? -A: In some cases it will, for example if licensecheck can find the dep metadata via importlib. Otherwise we reach out to pypi.org for this metadata. There are no plans at present to resolve this - -[note to me: I 'might' look at this as we should have a copy of the package version ] +A: Licensecheck requests the exact resolved version from the PyPI JSON API. If the package is not +available from public PyPI, it asks `uv` to fetch the exact wheel from the indexes configured for +the current project and reads the wheel metadata. Index credentials and source selection remain +`uv`'s responsibility. ## License lookup format diff --git a/licensecheck/checker.py b/licensecheck/checker.py index 5f8f3c9..25355d7 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -11,6 +11,17 @@ from licensecheck.packageinforesolver import PackageInfoManager +def _package_matches(package: PackageInfo, patterns: set[str]) -> bool: + package_names = {package.name.upper()} + if package.version: + package_names.add(f"{package.name}=={package.version}".upper()) + return any( + fnmatch(package_name, pattern.upper()) + for package_name in package_names + for pattern in patterns + ) + + def check( requirements_paths: set[str], groups: set[str], @@ -53,10 +64,9 @@ def check( for package in packages: # Deal with --ignore-packages and --fail-packages package.licenseCompat = False - packageName = package.name.upper() - if any(fnmatch(packageName, pattern.upper()) for pattern in ignore_packages): + if _package_matches(package, ignore_packages): package.licenseCompat = True - elif any(fnmatch(packageName, pattern.upper()) for pattern in fail_packages): + elif _package_matches(package, fail_packages): pass # package.licenseCompat = False # Else get compat with myLice else: diff --git a/licensecheck/io/cli.py b/licensecheck/io/cli.py index e8a4467..cc59ba2 100644 --- a/licensecheck/io/cli.py +++ b/licensecheck/io/cli.py @@ -66,12 +66,18 @@ def cli() -> None: # pragma: no cover ) parser.add_argument( "--ignore-packages", - help="set of packages/dependencies to ignore (compat=True), globs are supported", + help=( + "set of packages/dependencies to ignore (compat=True); names, " + "name==version, and globs are supported" + ), nargs="+", ) parser.add_argument( "--fail-packages", - help="set of packages/dependencies to fail (compat=False), globs are supported", + help=( + "set of packages/dependencies to fail (compat=False); names, " + "name==version, and globs are supported" + ), nargs="+", ) parser.add_argument( diff --git a/licensecheck/license_matrix/__init__.py b/licensecheck/license_matrix/__init__.py index c957595..5b589f6 100644 --- a/licensecheck/license_matrix/__init__.py +++ b/licensecheck/license_matrix/__init__.py @@ -183,6 +183,9 @@ def depCompatWMyLice( ignoreLicenses = ignoreLicenses or set() onlyLicenses = onlyLicenses or set() + if depLice & failLicenses: + return False + return any( liceCompat( myLicense, diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index 9da981b..e090db9 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -6,7 +6,9 @@ import contextlib import re import subprocess +import tempfile from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace from email.message import Message from importlib import metadata from importlib.metadata._meta import PackageMetadata @@ -30,6 +32,7 @@ RAW_JOINS = " AND " HTTP_OK = 200 +HTTP_NOT_FOUND = 404 def _parse_uv_requirements(raw_requirements: str, skip_dependencies: set[str]) -> set[Requirement]: @@ -56,21 +59,46 @@ def _gather_uv_requirements( skip_dependencies: set[str], base_index_url: str, ) -> set[Requirement]: - command = [ - "uv", - "pip", - "compile", - "--color", - "never", - "--index", - base_index_url, - requirements_path.as_posix(), - ] + lock_path = requirements_path.with_name("uv.lock") + use_lock = requirements_path.name == "pyproject.toml" and lock_path.is_file() + if use_lock: + command = [ + "uv", + "export", + "--project", + requirements_path.parent.as_posix(), + "--format", + "requirements.txt", + "--locked", + "--no-hashes", + "--no-header", + "--no-default-groups", + "--no-emit-project", + "--color", + "never", + ] + else: + command = [ + "uv", + "pip", + "compile", + "--color", + "never", + "--index", + base_index_url, + requirements_path.as_posix(), + ] for group in groups: command.extend(["--group", group]) for extra in extras: command.extend(["--extra", extra]) + if not use_lock and requirements_path.name == "pyproject.toml": + pyproject = tomli.loads(requirements_path.read_text(encoding="utf-8")) + prerelease = pyproject.get("tool", {}).get("uv", {}).get("prerelease") + if prerelease: + command.extend(["--prerelease", prerelease]) + try: result = subprocess.run( # noqa: S603 command, @@ -100,6 +128,7 @@ def __init__(self, base_pypi_url: str = "https://pypi.org") -> None: """ self.base_pypi_url = base_pypi_url self.reqs: set[Requirement] = set() + self.local_projects: dict[str, PackageInfo] = {} def resolve_requirements( self, @@ -110,6 +139,7 @@ def resolve_requirements( ) -> None: for requirements_path in requirements_paths: requirements_path_obj = Path(requirements_path) + self._register_local_sources(requirements_path_obj) try: resolved_requirements = _gather_uv_requirements( requirements_path=requirements_path_obj, @@ -119,6 +149,8 @@ def resolve_requirements( base_index_url=self.base_pypi_url, ) except RuntimeError: + if requirements_path_obj.name == "pyproject.toml": + raise resolved_requirements = gather( skipDependencies=skip_dependencies, groups=groups, @@ -129,6 +161,77 @@ def resolve_requirements( self.reqs.update(resolved_requirements) + def _register_local_sources( + self, + pyproject_path: Path, + seen: set[Path] | None = None, + ) -> None: + if pyproject_path.name != "pyproject.toml" or not pyproject_path.is_file(): + return + + resolved_path = pyproject_path.resolve() + seen = seen or set() + if resolved_path in seen: + return + seen.add(resolved_path) + + pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) + uv_config = pyproject.get("tool", {}).get("uv", {}) + source_paths: set[Path] = set() + + for source in uv_config.get("sources", {}).values(): + source_options = source if isinstance(source, list) else [source] + for source_option in source_options: + if not isinstance(source_option, dict) or "path" not in source_option: + continue + source_path = pyproject_path.parent / source_option["path"] + source_paths.add( + source_path + if source_path.name == "pyproject.toml" + else source_path / "pyproject.toml" + ) + + for member_pattern in uv_config.get("workspace", {}).get("members", []): + for member_path in pyproject_path.parent.glob(member_pattern): + source_paths.add(member_path / "pyproject.toml") + + for source_path in source_paths: + source_package = self._read_project_package(source_path) + if source_package is not None: + self.local_projects[source_package.name] = source_package + self._register_local_sources(source_path, seen) + + @staticmethod + def _read_project_package(pyproject_path: Path) -> PackageInfo | None: + if not pyproject_path.is_file(): + return None + + pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) + project = pyproject.get("project", {}) + name = project.get("name") + if not name: + return None + + license_value = project.get("license", UNKNOWN) + if isinstance(license_value, dict): + license_value = license_value.get("text", UNKNOWN) + + authors = project.get("authors", []) + author_names = [ + author.get("name", "") if isinstance(author, dict) else str(author) + for author in authors + ] + project_urls = project.get("urls", {}) + + return PackageInfo( + name=canonicalize_name(name), + version=project.get("version"), + homePage=project_urls.get("Homepage") or project_urls.get("homepage"), + author=", ".join(filter(None, author_names)), + license=str(license_value), + errorCode=0, + ) + def getPackages(self) -> set[PackageInfo]: """ Retrieve package information from local installation or PyPI. @@ -149,9 +252,16 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: versions: set[str | None] = {None} package.name = canonicalize_name(package.name) + if local_project := self.local_projects.get(package.name): + return replace(local_project) + specifier = getattr(package, "specifier", None) if specifier is not None: - parsed_versions = {item.version for item in specifier} + parsed_versions = { + item.version + for item in specifier + if item.operator in {"==", "==="} and "*" not in item.version + } if parsed_versions: versions = parsed_versions @@ -163,15 +273,32 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: lpi = LocalPackageInfo(package=base_pkg_info) rpi = RemotePackageInfo(pypi_api=self.base_pypi_url, package=base_pkg_info) + rpi.lazy_fetch() + + ipi = IndexPackageInfo(package=base_pkg_info) if rpi.http_code == HTTP_NOT_FOUND else None + index_name = ipi.get_name() if ipi is not None else None pkg_info = PackageInfo( name=package.name, - version=lpi.get_version() or rpi.get_version(), - size=lpi.get_size() or rpi.get_size(), - homePage=lpi.get_homePage() or rpi.get_homePage(), - author=lpi.get_author() or rpi.get_author(), - license=str(lpi.get_license() or rpi.get_license()), - errorCode=rpi.http_code if rpi.http_code != HTTP_OK else 0, + version=base_pkg_info.version + or lpi.get_version() + or (ipi.get_version() if ipi is not None else None) + or rpi.get_version(), + size=lpi.get_size() or (ipi.get_size() if ipi is not None else None) or rpi.get_size(), + homePage=lpi.get_homePage() + or (ipi.get_homePage() if ipi is not None else None) + or rpi.get_homePage(), + author=lpi.get_author() + or (ipi.get_author() if ipi is not None else None) + or rpi.get_author(), + license=str( + lpi.get_license() + or (ipi.get_license() if ipi is not None else None) + or rpi.get_license() + ), + errorCode=( + 0 if rpi.http_code == HTTP_OK or lpi.get_name() or index_name else rpi.http_code + ), ) # normailzing the license @@ -237,6 +364,85 @@ def get_size(self) -> int | None: return None # Package not found +class IndexPackageInfo: + """Handles package metadata from indexes configured for uv.""" + + def __init__(self, package: PackageInfo) -> None: + self.package = package + self.meta: PackageMetadata = Message() + self.fetched = False + + def lazy_fetch(self) -> None: + if self.fetched: + return + self.fetched = True + + requirement = self.package.name + if self.package.version: + requirement = f"{requirement}=={self.package.version}" + + with tempfile.TemporaryDirectory(prefix="licensecheck-") as target: + command = [ + "uv", + "pip", + "install", + "--color", + "never", + "--no-progress", + "--no-deps", + "--only-binary", + ":all:", + "--target", + target, + requirement, + ] + try: + result = subprocess.run( # noqa: S603 + command, + capture_output=True, + text=True, + check=False, + ) + except OSError: + return + + if result.returncode != 0: + return + + for distribution in metadata.distributions(path=[target]): + name = distribution.metadata.get("Name") + if name and canonicalize_name(name) == self.package.name: + self.meta = distribution.metadata + return + + def get_license(self) -> str | None: + self.lazy_fetch() + return ( + self.meta.get("License-Expression") + or from_classifiers(self.meta.get_all("Classifier")) + or self.meta.get("License") + ) + + def get_name(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Name") + + def get_version(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Version") + + def get_homePage(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Home-page") + + def get_author(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Author") + + def get_size(self) -> None: + return None + + class RemotePackageInfo: """Handles retrieval of package info from PyPI.""" @@ -249,12 +455,11 @@ def __init__(self, pypi_api: str, package: PackageInfo) -> None: def lazy_fetch(self) -> None: if self.resp is None: - # Attempt to get versioned info first - rc, raw_resp = self.make_req( - url=f"{self.pypi_api_pypi}{self.package.name}/{self.package.version}/json" - ) - # Otherwise just get the latest - if rc != HTTP_OK: + if self.package.version: + rc, raw_resp = self.make_req( + url=f"{self.pypi_api_pypi}/{self.package.name}/{self.package.version}/json" + ) + else: rc, raw_resp = self.make_req(url=f"{self.pypi_api_pypi}/{self.package.name}/json") self.http_code = rc diff --git a/tests/test_checker.py b/tests/test_checker.py index 736c2bc..a2f41f6 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -62,3 +62,37 @@ def test_check( ) assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("ignore_packages", "expected_incompatible"), + [ + ({"private-package==1.2.3"}, False), + ({"private-package==1.2.4"}, True), + ({"private-package==1.*"}, False), + ], +) +def test_ignore_packages_can_match_versions( + mock_package_info_manager: PackageInfoManager, + ignore_packages: set[str], + *, + expected_incompatible: bool, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo( + name="private-package", + version="1.2.3", + license="PROPRIETARY", + ) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.MIT, + package_info_manager=mock_package_info_manager, + ignore_packages=ignore_packages, + ) + + assert incompatible == expected_incompatible, packages diff --git a/tests/test_license_matrix.py b/tests/test_license_matrix.py index 86d7670..5a751f3 100644 --- a/tests/test_license_matrix.py +++ b/tests/test_license_matrix.py @@ -90,6 +90,14 @@ def test_dualLicenseCompat() -> None: assert license_matrix.depCompatWMyLice(L.MIT, {L.GPL_2, L.MIT}) +def test_failed_license_cannot_be_masked_by_compatible_license() -> None: + assert not license_matrix.depCompatWMyLice( + L.MIT, + {L.GPL_2, L.MIT}, + failLicenses={L.GPL_2}, + ) + + def test_whitelistedLicenseCompat() -> None: assert license_matrix.depCompatWMyLice(L.MIT, {L.MIT}, onlyLicenses={L.MIT}) assert license_matrix.depCompatWMyLice(L.MPL, {L.MIT}, onlyLicenses={L.MIT}) diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 504f266..9f81e57 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -9,6 +9,7 @@ from licensecheck.models.constants import UNKNOWN from licensecheck.models.packageinfo import PackageInfo from licensecheck.packageinforesolver import ( + IndexPackageInfo, LocalPackageInfo, PackageInfoManager, RemotePackageInfo, @@ -65,6 +66,118 @@ def test_getPackageInfoPypi(remote_package_info: RemotePackageInfo) -> None: assert pkg.get_license() == "Apache Software License" +def test_remote_package_info_uses_versioned_pypi_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pkg = RemotePackageInfo( + "https://packages.example", + PackageInfo(name="sample", version="1.2.3"), + ) + requested_urls: list[str] = [] + + def fake_make_req( + url: str, headers: dict[str, str] | None = None + ) -> tuple[int, dict[str, object]]: + del headers + requested_urls.append(url) + return 200, { + "info": { + "name": "sample", + "version": "1.2.3", + "license_expression": "MIT", + } + } + + monkeypatch.setattr(pkg, "make_req", fake_make_req) + + assert pkg.get_license() == "MIT" + assert requested_urls == ["https://packages.example/pypi/sample/1.2.3/json"] + + +def test_index_package_info_uses_uv_configured_indexes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pkg = IndexPackageInfo(PackageInfo(name="private-package", version="1.2.3")) + commands: list[list[str]] = [] + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: LicenseRef-Example-Proprietary +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + assert pkg.get_name() == "private-package" + assert pkg.get_version() == "1.2.3" + assert pkg.get_license() == "LicenseRef-Example-Proprietary" + assert commands == [ + [ + "uv", + "pip", + "install", + "--color", + "never", + "--no-progress", + "--no-deps", + "--only-binary", + ":all:", + "--target", + commands[0][commands[0].index("--target") + 1], + "private-package==1.2.3", + ] + ] + + +def test_package_manager_uses_private_index_when_pypi_is_missing( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 404, {} + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: MIT +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + package_info_manager.reqs = {Requirement("private-package==1.2.3")} + + package = package_info_manager.getPackages().pop() + + assert package.name == "private-package" + assert package.version == "1.2.3" + assert package.license == "MIT" + assert package.errorCode == 0 + + def test_getPackageInfoLocalNotFound() -> None: pkg = LocalPackageInfo(aux_packageinfo("this_package_does_not_exist")) assert pkg.get_size() is None @@ -138,6 +251,32 @@ def test_unpinned_requirement_does_not_crash(package_info_manager: PackageInfoMa assert package.errorCode == 0 +def test_resolved_requirement_version_is_preserved( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "sample", + "version": "1.0.0", + "license_expression": "MIT", + } + } + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + package_info_manager.reqs = {Requirement("sample==1.0.0.0")} + + package = package_info_manager.getPackages().pop() + + assert package.version == "1.0.0.0" + + def test_resolve_requirements_handles_nested_editable_uv_sources( package_info_manager: PackageInfoManager, tmp_path: Path ) -> None: @@ -340,6 +479,86 @@ def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: assert {str(requirement) for requirement in package_info_manager.reqs} == {"idna==3.10"} +def test_resolve_requirements_uses_uv_prerelease_setting( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["prerelease-package"] + +[tool.uv] +prerelease = "allow" +""", + ) + commands: list[list[str]] = [] + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + return CompletedProcess( + args=command, + returncode=0, + stdout="prerelease-package==1.0.0.dev1\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert commands[0][-2:] == ["--prerelease", "allow"] + + +def test_resolve_requirements_prefers_adjacent_uv_lock( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["dependency>=1"] +""", + ) + (tmp_path / "uv.lock").write_text("version = 1", encoding="utf-8") + commands: list[list[str]] = [] + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + return CompletedProcess( + args=command, + returncode=0, + stdout="dependency==1.2.3\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert commands[0][:4] == ["uv", "export", "--project", tmp_path.as_posix()] + assert "--locked" in commands[0] + assert {str(requirement) for requirement in package_info_manager.reqs} == {"dependency==1.2.3"} + + def test_resolve_requirements_keeps_package_sharing_editable_directory_name( package_info_manager: PackageInfoManager, tmp_path: Path ) -> None: @@ -487,6 +706,58 @@ def test_resolve_requirements_handles_editable_path_with_spaces( } +def test_resolve_requirements_uses_local_project_license_metadata( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + local_path = tmp_path / "local_dependency" + write_pyproject( + local_path, + """ +[project] +name = "local-dependency" +version = "1.2.3" +license = "LicenseRef-Example-Proprietary" +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../local_dependency" } +""", + ) + + def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=["uv", "pip", "compile"], + returncode=0, + stdout=f"local-dependency @ {local_path.as_uri()}\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + package = package_info_manager.getPackages().pop() + + assert package.name == "local-dependency" + assert package.version == "1.2.3" + assert package.license == "LicenseRef-Example-Proprietary" + assert package.errorCode == 0 + + def test_resolve_requirements_falls_back_for_uv_lock( package_info_manager: PackageInfoManager, tmp_path: Path ) -> None: @@ -512,3 +783,37 @@ def test_resolve_requirements_falls_back_for_uv_lock( assert {str(requirement) for requirement in package_info_manager.reqs} == { "fallback-package==1.2.3" } + + +def test_resolve_requirements_preserves_pyproject_resolution_error( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["dependency"] +""", + ) + + def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=["uv", "pip", "compile"], + returncode=1, + stdout="", + stderr="index unavailable", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + with pytest.raises(RuntimeError, match="index unavailable"): + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) From d5b003078a40711f206432421ec5d2da5ff59eea Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Fri, 31 Jul 2026 07:56:04 -0400 Subject: [PATCH 04/14] Resolve licenses from exact package artifacts --- documentation/user/README.md | 6 +- licensecheck/license_matrix/__init__.py | 8 +- licensecheck/packageinforesolver.py | 56 ++++++++++--- tests/test_license_matrix.py | 16 ++++ tests/test_packageinfo.py | 102 ++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 18 deletions(-) diff --git a/documentation/user/README.md b/documentation/user/README.md index be21259..b4175a5 100644 --- a/documentation/user/README.md +++ b/documentation/user/README.md @@ -170,9 +170,9 @@ requirements with `uv`. Q: The license for my dep has changed in v >1.0, so I'm using v < 1.0, why doesn't licensecheck report the correct license version? A: Licensecheck requests the exact resolved version from the PyPI JSON API. If the package is not -available from public PyPI, it asks `uv` to fetch the exact wheel from the indexes configured for -the current project and reads the wheel metadata. Index credentials and source selection remain -`uv`'s responsibility. +available from public PyPI or that response lacks usable license metadata, it asks `uv` to fetch +the exact wheel from the indexes configured for the current project and reads the wheel metadata. +Index credentials and source selection remain `uv`'s responsibility. ## License lookup format diff --git a/licensecheck/license_matrix/__init__.py b/licensecheck/license_matrix/__init__.py index 5b589f6..6696c66 100644 --- a/licensecheck/license_matrix/__init__.py +++ b/licensecheck/license_matrix/__init__.py @@ -221,11 +221,13 @@ def liceCompat( return True if len(onlyLicenses) > 0 and (lice not in onlyLicenses): return False - licenses = list(L) - row, col = licenses.index(myLicense) + 1, licenses.index(lice) + 1 + if myLicense in {L.UNKNOWN, L.NO_LICENSE} or lice in {L.UNKNOWN, L.NO_LICENSE}: + return False try: + licenses = list(L) + row, col = licenses.index(myLicense) + 1, licenses.index(lice) + 1 return LICENSE_MATRIX[row][col] == "1" - except KeyError: + except (IndexError, ValueError): logger.warning(f"Licenses {myLicense} and {lice} cannot be compared") return False diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index e090db9..b253d16 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -25,6 +25,7 @@ from license_expression import Licensing from packaging.requirements import Requirement from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version from licensecheck.models.constants import JOINS, UNKNOWN from licensecheck.models.packageinfo import PackageInfo @@ -32,7 +33,26 @@ RAW_JOINS = " AND " HTTP_OK = 200 -HTTP_NOT_FOUND = 404 + + +def _has_usable_license(license_value: str | None) -> bool: + return bool( + license_value + and license_value.strip() + and license_value.strip().upper() not in {UNKNOWN, "NONE"} + ) + + +def _versions_match(expected: str | None, actual: str | None) -> bool: + if expected is None: + return True + if actual is None: + return False + + try: + return Version(expected) == Version(actual) + except InvalidVersion: + return expected == actual def _parse_uv_requirements(raw_requirements: str, skip_dependencies: set[str]) -> set[Requirement]: @@ -275,29 +295,41 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: rpi = RemotePackageInfo(pypi_api=self.base_pypi_url, package=base_pkg_info) rpi.lazy_fetch() - ipi = IndexPackageInfo(package=base_pkg_info) if rpi.http_code == HTTP_NOT_FOUND else None + local_matches = _versions_match(base_pkg_info.version, lpi.get_version()) + local_name = lpi.get_name() if local_matches else None + local_license = lpi.get_license() if local_matches else None + remote_license = rpi.get_license() + + needs_index = (not local_name and rpi.http_code != HTTP_OK) or not any( + _has_usable_license(value) for value in (local_license, remote_license) + ) + ipi = IndexPackageInfo(package=base_pkg_info) if needs_index else None index_name = ipi.get_name() if ipi is not None else None + index_license = ipi.get_license() if ipi is not None else None + license_candidates = (local_license, index_license, remote_license) + license_value = next( + (value for value in license_candidates if _has_usable_license(value)), + next((value for value in license_candidates if value), None), + ) pkg_info = PackageInfo( name=package.name, version=base_pkg_info.version - or lpi.get_version() + or (lpi.get_version() if local_matches else None) or (ipi.get_version() if ipi is not None else None) or rpi.get_version(), - size=lpi.get_size() or (ipi.get_size() if ipi is not None else None) or rpi.get_size(), - homePage=lpi.get_homePage() + size=(lpi.get_size() if local_matches else None) + or (ipi.get_size() if ipi is not None else None) + or rpi.get_size(), + homePage=(lpi.get_homePage() if local_matches else None) or (ipi.get_homePage() if ipi is not None else None) or rpi.get_homePage(), - author=lpi.get_author() + author=(lpi.get_author() if local_matches else None) or (ipi.get_author() if ipi is not None else None) or rpi.get_author(), - license=str( - lpi.get_license() - or (ipi.get_license() if ipi is not None else None) - or rpi.get_license() - ), + license=license_value, errorCode=( - 0 if rpi.http_code == HTTP_OK or lpi.get_name() or index_name else rpi.http_code + 0 if rpi.http_code == HTTP_OK or local_name or index_name else rpi.http_code ), ) diff --git a/tests/test_license_matrix.py b/tests/test_license_matrix.py index 5a751f3..02c83a5 100644 --- a/tests/test_license_matrix.py +++ b/tests/test_license_matrix.py @@ -98,6 +98,22 @@ def test_failed_license_cannot_be_masked_by_compatible_license() -> None: ) +@pytest.mark.parametrize( + ("project_license", "dependency_license"), + [ + (L.UNKNOWN, L.MIT), + (L.NO_LICENSE, L.MIT), + (L.MIT, L.UNKNOWN), + (L.MIT, L.NO_LICENSE), + ], +) +def test_unknown_and_missing_licenses_fail_closed( + project_license: L, + dependency_license: L, +) -> None: + assert not license_matrix.depCompatWMyLice(project_license, {dependency_license}) + + def test_whitelistedLicenseCompat() -> None: assert license_matrix.depCompatWMyLice(L.MIT, {L.MIT}, onlyLicenses={L.MIT}) assert license_matrix.depCompatWMyLice(L.MPL, {L.MIT}, onlyLicenses={L.MIT}) diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 9f81e57..5cbb9b0 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -1,5 +1,6 @@ from __future__ import annotations +from email.message import Message from pathlib import Path from subprocess import CompletedProcess @@ -178,6 +179,103 @@ def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: assert package.errorCode == 0 +def test_package_manager_uses_exact_artifact_when_pypi_license_is_missing( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + commands: list[list[str]] = [] + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "artifact-package", + "version": "1.2.3", + } + } + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "artifact_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: artifact-package +Version: 1.2.3 +License-Expression: MIT +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + package_info_manager.reqs = {Requirement("artifact-package==1.2.3")} + + package = package_info_manager.getPackages().pop() + + assert package.name == "artifact-package" + assert package.version == "1.2.3" + assert package.license == "MIT" + assert commands[0][-1] == "artifact-package==1.2.3" + + +def test_package_manager_ignores_installed_metadata_from_another_version( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + installed_metadata = Message() + installed_metadata["Name"] = "artifact-package" + installed_metadata["Version"] = "9.9.9" + installed_metadata["License-Expression"] = "GPL-3.0-only" + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "artifact-package", + "version": "1.2.3", + } + } + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "artifact_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: artifact-package +Version: 1.2.3 +License-Expression: MIT +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr( + "licensecheck.packageinforesolver.metadata.metadata", lambda _name: installed_metadata + ) + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + package_info_manager.reqs = {Requirement("artifact-package==1.2.3")} + + package = package_info_manager.getPackages().pop() + + assert package.version == "1.2.3" + assert package.license == "MIT" + + def test_getPackageInfoLocalNotFound() -> None: pkg = LocalPackageInfo(aux_packageinfo("this_package_does_not_exist")) assert pkg.get_size() is None @@ -269,7 +367,11 @@ def fake_make_req( } } + def fail_artifact_fetch(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + pytest.fail("artifact metadata should not be fetched when PyPI declares a license") + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fail_artifact_fetch) package_info_manager.reqs = {Requirement("sample==1.0.0.0")} package = package_info_manager.getPackages().pop() From bc2240575cf50b10e34090d49607b168877114de Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Fri, 31 Jul 2026 08:11:33 -0400 Subject: [PATCH 05/14] Audit editable local projects Resolve active editable paths to their local pyproject metadata and add them to the audited dependency set instead of dropping them. Preserve filtering for inactive sources and cover nested, workspace, mismatched directory, and spaced-path cases. --- documentation/user/README.md | 3 +- licensecheck/packageinforesolver.py | 74 +++++++++++++++++++++++++++-- tests/test_packageinfo.py | 74 +++++++++++++++++++++++++++-- 3 files changed, 140 insertions(+), 11 deletions(-) diff --git a/documentation/user/README.md b/documentation/user/README.md index b4175a5..54fa614 100644 --- a/documentation/user/README.md +++ b/documentation/user/README.md @@ -94,7 +94,8 @@ Licensecheck supports a broad range of different tools and workflows. Though ple for some of these tools, behaviour may differ from what is expected. For a `pyproject.toml` with an adjacent `uv.lock`, Licensecheck exports the locked dependency graph. Otherwise it uses `uv` to resolve dependencies. Resolution errors for a `pyproject.toml` are reported directly instead of -falling back to a less accurate dependency graph. +falling back to a less accurate dependency graph. Editable local projects emitted by `uv` are +audited using the project metadata from their own `pyproject.toml` files. Note that `uv` supports requirements.in files. If a pyproject.toml, setup.py, or setup.cfg file is provided, `uv` will extract the requirements for the relevant project. In testing this seems to have diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index b253d16..3941eca 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -14,6 +14,7 @@ from importlib.metadata._meta import PackageMetadata from pathlib import Path from typing import Any +from urllib.parse import unquote, urlparse import license_expression import requests @@ -55,12 +56,38 @@ def _versions_match(expected: str | None, actual: str | None) -> bool: return expected == actual -def _parse_uv_requirements(raw_requirements: str, skip_dependencies: set[str]) -> set[Requirement]: +def _editable_project_path(line: str, base_path: Path) -> Path | None: + stripped_line = line.strip() + for prefix in ("-e ", "--editable "): + if stripped_line.startswith(prefix): + target = stripped_line.removeprefix(prefix).strip() + break + else: + return None + + parsed_url = urlparse(target) + if parsed_url.scheme and parsed_url.scheme != "file": + return None + + path = Path(unquote(parsed_url.path if parsed_url.scheme == "file" else target)) + if not path.is_absolute(): + path = base_path / path + return path.resolve() + + +def _parse_uv_requirements( + raw_requirements: str, + skip_dependencies: set[str], + base_path: Path, +) -> tuple[set[Requirement], set[Path]]: skip_names = {canonicalize_name(name) for name in skip_dependencies} parsed_requirements: set[Requirement] = set() + editable_paths: set[Path] = set() for parsed in requirements.parse(raw_requirements): if parsed.editable: + if editable_path := _editable_project_path(parsed.line, base_path): + editable_paths.add(editable_path) continue if not parsed.name or canonicalize_name(parsed.name) in skip_names: continue @@ -69,7 +96,7 @@ def _parse_uv_requirements(raw_requirements: str, skip_dependencies: set[str]) - requirement.name = canonicalize_name(requirement.name) parsed_requirements.add(requirement) - return parsed_requirements + return parsed_requirements, editable_paths def _gather_uv_requirements( @@ -78,7 +105,7 @@ def _gather_uv_requirements( extras: set[str], skip_dependencies: set[str], base_index_url: str, -) -> set[Requirement]: +) -> tuple[set[Requirement], set[Path]]: lock_path = requirements_path.with_name("uv.lock") use_lock = requirements_path.name == "pyproject.toml" and lock_path.is_file() if use_lock: @@ -133,7 +160,11 @@ def _gather_uv_requirements( message = f"Non-zero returncode: {result.stderr}, {result.stdout}" raise RuntimeError(message) - return _parse_uv_requirements(result.stdout, skip_dependencies) + return _parse_uv_requirements( + result.stdout, + skip_dependencies, + requirements_path.parent.resolve(), + ) class PackageInfoManager: @@ -161,7 +192,7 @@ def resolve_requirements( requirements_path_obj = Path(requirements_path) self._register_local_sources(requirements_path_obj) try: - resolved_requirements = _gather_uv_requirements( + resolved_requirements, editable_paths = _gather_uv_requirements( requirements_path=requirements_path_obj, groups=groups, extras=extras, @@ -178,9 +209,42 @@ def resolve_requirements( requirementsPath=requirements_path_obj, base_index_url=self.base_pypi_url, ) + editable_paths = set() + self._register_editable_projects( + resolved_requirements, + editable_paths, + skip_dependencies, + ) self.reqs.update(resolved_requirements) + def _register_editable_projects( + self, + resolved_requirements: set[Requirement], + editable_paths: set[Path], + skip_dependencies: set[str], + ) -> None: + skip_names = {canonicalize_name(name) for name in skip_dependencies} + for editable_path in editable_paths: + pyproject_path = ( + editable_path + if editable_path.name == "pyproject.toml" + else editable_path / "pyproject.toml" + ) + package = self._read_project_package(pyproject_path) + if package is None: + continue + + self.local_projects[package.name] = package + self._register_local_sources(pyproject_path) + if package.name in skip_names: + continue + + requirement = package.name + if package.version: + requirement = f"{requirement}=={package.version}" + resolved_requirements.add(Requirement(requirement)) + def _register_local_sources( self, pyproject_path: Path, diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 5cbb9b0..6eece05 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -349,6 +349,60 @@ def test_unpinned_requirement_does_not_crash(package_info_manager: PackageInfoMa assert package.errorCode == 0 +def test_resolve_requirements_audits_editable_project( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_pyproject( + tmp_path / "local_dependency", + """ +[project] +name = "local-dependency" +version = "1.2.3" +license = "LicenseRef-Example-Proprietary" +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../local_dependency", editable = true } +""", + ) + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=command, + returncode=0, + stdout="-e ../local_dependency\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + package = package_info_manager.getPackages().pop() + + assert {str(requirement) for requirement in package_info_manager.reqs} == { + "local-dependency==1.2.3" + } + assert package.name == "local-dependency" + assert package.version == "1.2.3" + assert package.license == "LicenseRef-Example-Proprietary" + assert package.errorCode == 0 + + def test_resolved_requirement_version_is_preserved( package_info_manager: PackageInfoManager, monkeypatch: pytest.MonkeyPatch, @@ -446,7 +500,9 @@ def test_resolve_requirements_handles_nested_editable_uv_sources( ) assert {requirement.name for requirement in package_info_manager.reqs} == { - "transitive-dependency" + "local-dependency", + "nested-dependency", + "transitive-dependency", } @@ -535,7 +591,10 @@ def test_resolve_requirements_handles_editable_uv_sources_in_monorepo( ) assert {requirement.name for requirement in package_info_manager.reqs} == { - "published-dependency" + "anomaly-detection", + "published-dependency", + "sdk-extensions", + "service-common", } @@ -707,7 +766,10 @@ def test_resolve_requirements_keeps_package_sharing_editable_directory_name( skip_dependencies=set(), ) - assert {requirement.name for requirement in package_info_manager.reqs} == {"idna"} + assert {requirement.name for requirement in package_info_manager.reqs} == { + "idna", + "internal-helper", + } def test_resolve_requirements_handles_editable_uv_workspace_source( @@ -757,7 +819,8 @@ def test_resolve_requirements_handles_editable_uv_workspace_source( ) assert {requirement.name for requirement in package_info_manager.reqs} == { - "published-dependency" + "local-dependency", + "published-dependency", } @@ -804,7 +867,8 @@ def test_resolve_requirements_handles_editable_path_with_spaces( ) assert {requirement.name for requirement in package_info_manager.reqs} == { - "published-dependency" + "local-dependency", + "published-dependency", } From c884376acef78abb33130de9993c60e8f1d50942 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Fri, 31 Jul 2026 19:06:22 -0400 Subject: [PATCH 06/14] Fix source-aware dependency metadata resolution --- licensecheck/packageinforesolver.py | 638 +++++++++++++++++++++++----- tests/test_packageinfo.py | 368 +++++++++++++++- 2 files changed, 891 insertions(+), 115 deletions(-) diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index 3941eca..4f2205d 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -4,17 +4,19 @@ import configparser import contextlib +import os import re import subprocess import tempfile from concurrent.futures import ThreadPoolExecutor -from dataclasses import replace +from dataclasses import dataclass, replace from email.message import Message from importlib import metadata from importlib.metadata._meta import PackageMetadata from pathlib import Path from typing import Any from urllib.parse import unquote, urlparse +from urllib.request import url2pathname import license_expression import requests @@ -24,7 +26,7 @@ from depgather.models.pypijson import ProjectResponse from depgather.parse import gather from license_expression import Licensing -from packaging.requirements import Requirement +from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name from packaging.version import InvalidVersion, Version @@ -36,6 +38,28 @@ HTTP_OK = 200 +class UvUnavailableError(RuntimeError): + """Raised when the optional uv executable is unavailable.""" + + +@dataclass(frozen=True) +class _UvIndex: + name: str | None + url: str + format: str | None = None + + +@dataclass(frozen=True) +class _UvResolutionContext: + directory: Path + base_index_url: str + source_url: str | None = None + index_args: tuple[str, ...] = () + index_environment: tuple[tuple[str, str], ...] = () + prefer_artifact: bool = False + remote_matches_source: bool = False + + def _has_usable_license(license_value: str | None) -> bool: return bool( license_value @@ -56,6 +80,237 @@ def _versions_match(expected: str | None, actual: str | None) -> bool: return expected == actual +def _exact_requirement_version(requirement: Requirement | PackageInfo) -> str | None: + if not isinstance(requirement, Requirement): + return requirement.version + versions = { + item.version + for item in requirement.specifier + if item.operator in {"==", "==="} and "*" not in item.version + } + return next(iter(versions)) if len(versions) == 1 else None + + +def _requirement_key( + requirement: Requirement | PackageInfo, +) -> tuple[str, str | None, str | None]: + return ( + canonicalize_name(requirement.name), + _exact_requirement_version(requirement), + getattr(requirement, "url", None), + ) + + +def _normalized_index_url(url: str) -> str: + parsed = urlparse(url) + if not parsed.scheme and Path(url).is_absolute(): + return Path(url).resolve().as_uri().rstrip("/") + if parsed.scheme == "file": + return Path(url2pathname(unquote(parsed.path))).resolve().as_uri().rstrip("/") + normalized = url.rstrip("/").removesuffix("/simple") + parsed = urlparse(normalized) + if parsed.hostname: + host = parsed.hostname.lower() + if parsed.port: + host = f"{host}:{parsed.port}" + normalized = parsed._replace(netloc=host, query="", fragment="").geturl() + return normalized + + +def _is_public_pypi(url: str) -> bool: + parsed = urlparse(_normalized_index_url(url)) + return parsed.hostname in {"pypi.org", "www.pypi.org"} and parsed.path in {"", "/"} + + +def _same_index(left: str, right: str) -> bool: + return _normalized_index_url(left) == _normalized_index_url(right) + + +def _read_uv_configuration(directory: Path) -> tuple[dict[str, Any], Path]: + for candidate_directory in (directory, *directory.parents): + uv_toml = candidate_directory / "uv.toml" + if uv_toml.is_file(): + return tomli.loads(uv_toml.read_text(encoding="utf-8")), candidate_directory + + pyproject_path = candidate_directory / "pyproject.toml" + if pyproject_path.is_file(): + pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) + uv_config = pyproject.get("tool", {}).get("uv") + if isinstance(uv_config, dict): + return uv_config, candidate_directory + + return {}, directory + + +def _resolved_uv_index_url(raw_url: object, config_directory: Path) -> str: + url = str(raw_url) + if Path(url).is_absolute() or not urlparse(url).scheme: + return (config_directory / url).resolve().as_uri() + return url + + +def _legacy_uv_indexes( + config: dict[str, Any], + config_directory: Path, +) -> list[_UvIndex]: + indexes: list[_UvIndex] = [] + default_index = config.get("index-url") + if default_index: + indexes.append( + _UvIndex( + name=None, + url=_resolved_uv_index_url(default_index, config_directory), + ) + ) + extra_indexes = config.get("extra-index-url", []) + if isinstance(extra_indexes, str): + extra_indexes = [extra_indexes] + indexes.extend( + _UvIndex(name=None, url=_resolved_uv_index_url(url, config_directory)) + for url in extra_indexes + ) + find_links = config.get("find-links", []) + if isinstance(find_links, str): + find_links = [find_links] + indexes.extend( + _UvIndex( + name=None, + url=_resolved_uv_index_url(url, config_directory), + format="flat", + ) + for url in find_links + ) + return indexes + + +def _configured_uv_indexes( + uv_config: dict[str, Any], + config_directory: Path, +) -> list[_UvIndex]: + + raw_indexes = uv_config.get("index", []) + if isinstance(raw_indexes, dict): + raw_indexes = [raw_indexes] + + indexes: list[_UvIndex] = [] + for index in raw_indexes: + if not isinstance(index, dict) or not index.get("url"): + continue + indexes.append( + _UvIndex( + name=index.get("name"), + url=_resolved_uv_index_url(index["url"], config_directory), + format=index.get("format"), + ) + ) + + indexes.extend(_legacy_uv_indexes(uv_config, config_directory)) + pip_config = uv_config.get("pip", {}) + if isinstance(pip_config, dict): + indexes.extend(_legacy_uv_indexes(pip_config, config_directory)) + + return indexes + + +def _environment_uv_indexes() -> list[_UvIndex]: + indexes: list[_UvIndex] = [] + for variable in ("UV_INDEX", "UV_EXTRA_INDEX_URL"): + for value in os.environ.get(variable, "").split(): + if urlparse(value).scheme: + indexes.append(_UvIndex(name=None, url=value)) + continue + name, separator, url = value.partition("=") + indexes.append(_UvIndex(name=name if separator else None, url=url or name)) + indexes.extend( + _UvIndex(name=None, url=value) + for variable in ("UV_DEFAULT_INDEX", "UV_INDEX_URL") + if (value := os.environ.get(variable)) + ) + indexes.extend( + _UvIndex(name=None, url=value, format="flat") + for value in os.environ.get("UV_FIND_LINKS", "").split() + ) + return indexes + + +def _index_invocation_for_source( + source_url: str, + indexes: list[_UvIndex], +) -> tuple[tuple[str, ...], tuple[tuple[str, str], ...]]: + matching_index = next( + (index for index in indexes if _same_index(index.url, source_url)), + None, + ) + if matching_index is not None and matching_index.format == "flat": + parsed_index = urlparse(matching_index.url) + if parsed_index.username or parsed_index.password or parsed_index.query: + return ("--no-index",), (("UV_FIND_LINKS", matching_index.url),) + return ("--no-index", "--find-links", matching_index.url), () + + if matching_index is not None and matching_index.name: + parsed_index = urlparse(matching_index.url) + if parsed_index.username or parsed_index.password or parsed_index.query: + index_value = f"{matching_index.name}={matching_index.url}" + return (), (("UV_INDEX", index_value),) + return ( + ("--index", f"{matching_index.name}={source_url}"), + (), + ) + + if matching_index is not None and ( + urlparse(matching_index.url).username + or urlparse(matching_index.url).password + or urlparse(matching_index.url).query + ): + return (), (("UV_INDEX", matching_index.url),) + if urlparse(source_url).username or urlparse(source_url).password or urlparse(source_url).query: + return (), (("UV_INDEX", source_url),) + + return ("--index", source_url), () + + +def _annotated_requirement_sources( + raw_requirements: str, +) -> dict[tuple[str, str | None, str | None], str]: + sources: dict[tuple[str, str | None, str | None], str] = {} + current_requirement: Requirement | None = None + for raw_line in raw_requirements.splitlines(): + line = raw_line.strip() + if line.startswith("# from ") and current_requirement is not None: + sources[_requirement_key(current_requirement)] = line.removeprefix("# from ").strip() + continue + if not line or line.startswith("#"): + continue + if line.startswith(("-e ", "--editable ", "--")): + current_requirement = None + continue + try: + current_requirement = Requirement(line) + except InvalidRequirement: + current_requirement = None + return sources + + +def _locked_requirement_sources( + lock_path: Path, +) -> dict[tuple[str, str | None, str | None], str]: + if not lock_path.is_file(): + return {} + + lock = tomli.loads(lock_path.read_text(encoding="utf-8")) + sources: dict[tuple[str, str | None, str | None], str] = {} + for package in lock.get("package", []): + if not isinstance(package, dict): + continue + name = package.get("name") + version = package.get("version") + source = package.get("source", {}) + registry = source.get("registry") if isinstance(source, dict) else None + if name and registry: + sources[(canonicalize_name(name), version, None)] = str(registry) + return sources + + def _editable_project_path(line: str, base_path: Path) -> Path | None: stripped_line = line.strip() for prefix in ("-e ", "--editable "): @@ -69,7 +324,21 @@ def _editable_project_path(line: str, base_path: Path) -> Path | None: if parsed_url.scheme and parsed_url.scheme != "file": return None - path = Path(unquote(parsed_url.path if parsed_url.scheme == "file" else target)) + path_value = parsed_url.path if parsed_url.scheme == "file" else target + path = Path(url2pathname(unquote(path_value))) + if not path.is_absolute(): + path = base_path / path + return path.resolve() + + +def _requirement_project_path(requirement: Requirement, base_path: Path) -> Path | None: + if not requirement.url: + return None + parsed_url = urlparse(requirement.url) + if parsed_url.scheme != "file": + return None + + path = Path(url2pathname(unquote(parsed_url.path))) if not path.is_absolute(): path = base_path / path return path.resolve() @@ -79,7 +348,11 @@ def _parse_uv_requirements( raw_requirements: str, skip_dependencies: set[str], base_path: Path, -) -> tuple[set[Requirement], set[Path]]: +) -> tuple[ + set[Requirement], + set[Path], + dict[tuple[str, str | None, str | None], str], +]: skip_names = {canonicalize_name(name) for name in skip_dependencies} parsed_requirements: set[Requirement] = set() editable_paths: set[Path] = set() @@ -96,20 +369,17 @@ def _parse_uv_requirements( requirement.name = canonicalize_name(requirement.name) parsed_requirements.add(requirement) - return parsed_requirements, editable_paths + return parsed_requirements, editable_paths, _annotated_requirement_sources(raw_requirements) -def _gather_uv_requirements( +def _uv_requirement_command( requirements_path: Path, - groups: set[str], - extras: set[str], - skip_dependencies: set[str], + *, + use_lock: bool, base_index_url: str, -) -> tuple[set[Requirement], set[Path]]: - lock_path = requirements_path.with_name("uv.lock") - use_lock = requirements_path.name == "pyproject.toml" and lock_path.is_file() +) -> list[str]: if use_lock: - command = [ + return [ "uv", "export", "--project", @@ -124,17 +394,40 @@ def _gather_uv_requirements( "--color", "never", ] - else: - command = [ - "uv", - "pip", - "compile", - "--color", - "never", - "--index", - base_index_url, - requirements_path.as_posix(), - ] + + command = [ + "uv", + "pip", + "compile", + "--color", + "never", + "--emit-index-annotation", + ] + if not _is_public_pypi(base_index_url): + command.extend(["--default-index", base_index_url]) + command.append(requirements_path.as_posix()) + return command + + +def _gather_uv_requirements( + requirements_path: Path, + groups: set[str], + extras: set[str], + skip_dependencies: set[str], + base_index_url: str, +) -> tuple[ + set[Requirement], + set[Path], + dict[tuple[str, str | None, str | None], str], +]: + requirements_path = requirements_path.resolve() + lock_path = requirements_path.with_name("uv.lock") + use_lock = requirements_path.name == "pyproject.toml" and lock_path.is_file() + command = _uv_requirement_command( + requirements_path, + use_lock=use_lock, + base_index_url=base_index_url, + ) for group in groups: command.extend(["--group", group]) for extra in extras: @@ -152,7 +445,10 @@ def _gather_uv_requirements( capture_output=True, text=True, check=False, + cwd=requirements_path.parent, ) + except FileNotFoundError as error: + raise UvUnavailableError from error except OSError as error: raise RuntimeError from error @@ -160,11 +456,51 @@ def _gather_uv_requirements( message = f"Non-zero returncode: {result.stderr}, {result.stdout}" raise RuntimeError(message) - return _parse_uv_requirements( + parsed_requirements, editable_paths, sources = _parse_uv_requirements( result.stdout, skip_dependencies, requirements_path.parent.resolve(), ) + if use_lock: + sources.update(_locked_requirement_sources(lock_path)) + return parsed_requirements, editable_paths, sources + + +def _resolution_context( + requirements_path: Path, + requirement: Requirement, + source_url: str | None, + base_index_url: str, +) -> _UvResolutionContext: + directory = requirements_path.parent.resolve() + uv_config, config_directory = _read_uv_configuration(directory) + indexes = [ + *_configured_uv_indexes(uv_config, config_directory), + *_environment_uv_indexes(), + ] + index_args, index_environment = ( + _index_invocation_for_source(source_url, indexes) if source_url else ((), ()) + ) + has_custom_index = any(not _is_public_pypi(index.url) for index in indexes) + prefer_artifact = bool( + requirement.url + or (source_url and not _is_public_pypi(source_url)) + or (source_url is None and has_custom_index) + or not _is_public_pypi(base_index_url) + ) + remote_matches_source = bool( + (source_url and _same_index(source_url, base_index_url)) + or (source_url is None and not requirement.url and not _is_public_pypi(base_index_url)) + ) + return _UvResolutionContext( + directory=directory, + base_index_url=base_index_url, + source_url=source_url, + index_args=index_args, + index_environment=index_environment, + prefer_artifact=prefer_artifact, + remote_matches_source=remote_matches_source, + ) class PackageInfoManager: @@ -180,6 +516,9 @@ def __init__(self, base_pypi_url: str = "https://pypi.org") -> None: self.base_pypi_url = base_pypi_url self.reqs: set[Requirement] = set() self.local_projects: dict[str, PackageInfo] = {} + self.resolution_contexts: dict[ + tuple[str, str | None, str | None], _UvResolutionContext + ] = {} def resolve_requirements( self, @@ -189,16 +528,25 @@ def resolve_requirements( skip_dependencies: set[str], ) -> None: for requirements_path in requirements_paths: - requirements_path_obj = Path(requirements_path) - self._register_local_sources(requirements_path_obj) + requirements_path_obj = Path(requirements_path).resolve() try: - resolved_requirements, editable_paths = _gather_uv_requirements( + resolved_requirements, editable_paths, source_urls = _gather_uv_requirements( requirements_path=requirements_path_obj, groups=groups, extras=extras, skip_dependencies=skip_dependencies, base_index_url=self.base_pypi_url, ) + except UvUnavailableError: + resolved_requirements = gather( + skipDependencies=skip_dependencies, + groups=groups, + extras=extras, + requirementsPath=requirements_path_obj, + base_index_url=self.base_pypi_url, + ) + editable_paths = set() + source_urls = {} except RuntimeError: if requirements_path_obj.name == "pyproject.toml": raise @@ -210,12 +558,25 @@ def resolve_requirements( base_index_url=self.base_pypi_url, ) editable_paths = set() + source_urls = {} self._register_editable_projects( resolved_requirements, editable_paths, skip_dependencies, ) + self._register_direct_local_projects( + resolved_requirements, + requirements_path_obj.parent, + ) + for requirement in resolved_requirements: + key = _requirement_key(requirement) + self.resolution_contexts[key] = _resolution_context( + requirements_path_obj, + requirement, + source_urls.get(key), + self.base_pypi_url, + ) self.reqs.update(resolved_requirements) def _register_editable_projects( @@ -236,7 +597,6 @@ def _register_editable_projects( continue self.local_projects[package.name] = package - self._register_local_sources(pyproject_path) if package.name in skip_names: continue @@ -245,45 +605,23 @@ def _register_editable_projects( requirement = f"{requirement}=={package.version}" resolved_requirements.add(Requirement(requirement)) - def _register_local_sources( + def _register_direct_local_projects( self, - pyproject_path: Path, - seen: set[Path] | None = None, + resolved_requirements: set[Requirement], + base_path: Path, ) -> None: - if pyproject_path.name != "pyproject.toml" or not pyproject_path.is_file(): - return - - resolved_path = pyproject_path.resolve() - seen = seen or set() - if resolved_path in seen: - return - seen.add(resolved_path) - - pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) - uv_config = pyproject.get("tool", {}).get("uv", {}) - source_paths: set[Path] = set() - - for source in uv_config.get("sources", {}).values(): - source_options = source if isinstance(source, list) else [source] - for source_option in source_options: - if not isinstance(source_option, dict) or "path" not in source_option: - continue - source_path = pyproject_path.parent / source_option["path"] - source_paths.add( - source_path - if source_path.name == "pyproject.toml" - else source_path / "pyproject.toml" - ) - - for member_pattern in uv_config.get("workspace", {}).get("members", []): - for member_path in pyproject_path.parent.glob(member_pattern): - source_paths.add(member_path / "pyproject.toml") - - for source_path in source_paths: - source_package = self._read_project_package(source_path) - if source_package is not None: - self.local_projects[source_package.name] = source_package - self._register_local_sources(source_path, seen) + for requirement in resolved_requirements: + project_path = _requirement_project_path(requirement, base_path) + if project_path is None: + continue + pyproject_path = ( + project_path + if project_path.name == "pyproject.toml" + else project_path / "pyproject.toml" + ) + package = self._read_project_package(pyproject_path) + if package is not None and package.name == canonicalize_name(requirement.name): + self.local_projects[package.name] = package @staticmethod def _read_project_package(pyproject_path: Path) -> PackageInfo | None: @@ -291,28 +629,48 @@ def _read_project_package(pyproject_path: Path) -> PackageInfo | None: return None pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) - project = pyproject.get("project", {}) - name = project.get("name") + tool = pyproject.get("tool", {}) + project = ( + pyproject.get("project") + or tool.get("poetry") + or tool.get("flit", {}).get("metadata", {}) + ) + if not isinstance(project, dict): + return None + + name = project.get("name") or project.get("dist-name") or project.get("module") if not name: return None license_value = project.get("license", UNKNOWN) if isinstance(license_value, dict): license_value = license_value.get("text", UNKNOWN) + if not _has_usable_license(str(license_value)): + license_value = from_classifiers(project.get("classifiers")) or UNKNOWN + license_value = normalize_license(str(license_value)) - authors = project.get("authors", []) + authors = project.get("authors", project.get("author", [])) + if isinstance(authors, str): + authors = [authors] author_names = [ author.get("name", "") if isinstance(author, dict) else str(author) for author in authors ] project_urls = project.get("urls", {}) + if not isinstance(project_urls, dict): + project_urls = {} return PackageInfo( name=canonicalize_name(name), version=project.get("version"), - homePage=project_urls.get("Homepage") or project_urls.get("homepage"), + homePage=( + project_urls.get("Homepage") + or project_urls.get("homepage") + or project.get("homepage") + or project.get("home-page") + ), author=", ".join(filter(None, author_names)), - license=str(license_value), + license=license_value, errorCode=0, ) @@ -333,41 +691,74 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: :param Requirement package: package info to unpack :return PackageInfo: Information about the package. """ - versions: set[str | None] = {None} + context = self.resolution_contexts.get(_requirement_key(package)) package.name = canonicalize_name(package.name) if local_project := self.local_projects.get(package.name): - return replace(local_project) - - specifier = getattr(package, "specifier", None) - if specifier is not None: - parsed_versions = { - item.version - for item in specifier - if item.operator in {"==", "==="} and "*" not in item.version - } - if parsed_versions: - versions = parsed_versions - - package.name = canonicalize_name(package.name) + local_package = replace(local_project) + if local_package.license: + local_package.license = normalize_license(local_package.license) + return local_package base_pkg_info: PackageInfo = PackageInfo( - name=package.name, version=versions.pop(), errorCode=1 + name=package.name, + version=_exact_requirement_version(package), + errorCode=1, ) lpi = LocalPackageInfo(package=base_pkg_info) - rpi = RemotePackageInfo(pypi_api=self.base_pypi_url, package=base_pkg_info) - rpi.lazy_fetch() - - local_matches = _versions_match(base_pkg_info.version, lpi.get_version()) + resolved_source = context is not None and ( + context.source_url is not None or context.prefer_artifact + ) + local_matches = not resolved_source and _versions_match( + base_pkg_info.version, + lpi.get_version(), + ) local_name = lpi.get_name() if local_matches else None local_license = lpi.get_license() if local_matches else None + + if context is not None and context.prefer_artifact: + preferred_index = IndexPackageInfo( + package=base_pkg_info, + requirement=package, + context=context, + ) + if preferred_index.get_name(): + pkg_info = PackageInfo( + name=package.name, + version=base_pkg_info.version or preferred_index.get_version(), + size=preferred_index.get_size(), + homePage=preferred_index.get_homePage(), + author=preferred_index.get_author(), + license=preferred_index.get_license(), + errorCode=0, + ) + if pkg_info.license: + pkg_info.license = normalize_license(pkg_info.license) + return pkg_info + if not context.remote_matches_source: + return PackageInfo( + name=package.name, + version=base_pkg_info.version, + errorCode=1, + ) + + rpi = RemotePackageInfo(pypi_api=self.base_pypi_url, package=base_pkg_info) + rpi.lazy_fetch() remote_license = rpi.get_license() needs_index = (not local_name and rpi.http_code != HTTP_OK) or not any( _has_usable_license(value) for value in (local_license, remote_license) ) - ipi = IndexPackageInfo(package=base_pkg_info) if needs_index else None + ipi = ( + IndexPackageInfo( + package=base_pkg_info, + requirement=package, + context=context, + ) + if needs_index + else None + ) index_name = ipi.get_name() if ipi is not None else None index_license = ipi.get_license() if ipi is not None else None license_candidates = (local_license, index_license, remote_license) @@ -423,7 +814,7 @@ def __init__(self, package: PackageInfo) -> None: self.package: PackageInfo = package # email message appears to mostly conform to the protocol # https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata - self.meta: PackageMetadata = Message() + self.meta: Message[str, str] | PackageMetadata = Message() with contextlib.suppress(metadata.PackageNotFoundError): self.meta = metadata.metadata(package.name) @@ -463,9 +854,16 @@ def get_size(self) -> int | None: class IndexPackageInfo: """Handles package metadata from indexes configured for uv.""" - def __init__(self, package: PackageInfo) -> None: + def __init__( + self, + package: PackageInfo, + requirement: Requirement | PackageInfo | None = None, + context: _UvResolutionContext | None = None, + ) -> None: self.package = package - self.meta: PackageMetadata = Message() + self.requirement = requirement + self.context = context + self.meta: Message[str, str] | PackageMetadata = Message() self.fetched = False def lazy_fetch(self) -> None: @@ -473,8 +871,9 @@ def lazy_fetch(self) -> None: return self.fetched = True - requirement = self.package.name - if self.package.version: + requirement_url = getattr(self.requirement, "url", None) + requirement = str(self.requirement) if requirement_url else self.package.name + if self.package.version and not requirement_url: requirement = f"{requirement}=={self.package.version}" with tempfile.TemporaryDirectory(prefix="licensecheck-") as target: @@ -490,14 +889,24 @@ def lazy_fetch(self) -> None: ":all:", "--target", target, - requirement, ] + if self.context is not None: + if not _is_public_pypi(self.context.base_index_url): + command.extend(["--default-index", self.context.base_index_url]) + command.extend(self.context.index_args) + command.append(requirement) + run_environment = None + if self.context is not None and self.context.index_environment: + run_environment = os.environ.copy() + run_environment.update(dict(self.context.index_environment)) try: result = subprocess.run( # noqa: S603 command, capture_output=True, text=True, check=False, + cwd=self.context.directory if self.context is not None else None, + env=run_environment, ) except OSError: return @@ -547,7 +956,7 @@ def __init__(self, pypi_api: str, package: PackageInfo) -> None: self.pypi_api_integrity = pypi_api + "/integrity" self.package = package self.http_code: int = 0 - self.resp: ProjectResponse = None + self.resp: ProjectResponse | None = None def lazy_fetch(self) -> None: if self.resp is None: @@ -561,6 +970,13 @@ def lazy_fetch(self) -> None: self.http_code = rc self.resp = ProjectResponse.model_validate(raw_resp) + def _response(self) -> ProjectResponse: + self.lazy_fetch() + if self.resp is None: + message = "Package metadata response was not initialized" + raise RuntimeError(message) + return self.resp + def make_req( self, url: str, headers: dict[str, str] | None = None ) -> tuple[int, dict[str, Any]]: @@ -575,33 +991,29 @@ def make_req( return -2, {} def get_name(self) -> str: - self.lazy_fetch() - return self.resp.info.name + return self._response().info.name def get_version(self) -> str: - self.lazy_fetch() - return self.resp.info.version + return self._response().info.version def get_homePage(self) -> str: - self.lazy_fetch() - return self.resp.info.home_page + return self._response().info.home_page def get_author(self) -> str: - self.lazy_fetch() - author_email = self.resp.info.author_email or "" - return self.resp.info.author or author_email.split("<")[0].strip() + response = self._response() + author_email = response.info.author_email or "" + return response.info.author or author_email.split("<")[0].strip() def get_license(self) -> str: - self.lazy_fetch() + response = self._response() return ( - self.resp.info.license_expression - or from_classifiers(self.resp.info.classifiers) - or self.resp.info.license + response.info.license_expression + or from_classifiers(response.info.classifiers) + or response.info.license ) def get_size(self) -> int | None: - self.lazy_fetch() - urls = self.resp.urls + urls = self._response().urls return urls[-1].size if len(urls) > 0 else None diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 6eece05..c73dce9 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -140,6 +140,224 @@ def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: ] +def test_resolve_requirements_uses_project_directory_and_default_index( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["private-package"] +""", + ) + calls: list[tuple[list[str], dict[str, object]]] = [] + + def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]: + calls.append((command, kwargs)) + return CompletedProcess(command, 0, "private-package==1.2.3\n", "") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + manager = PackageInfoManager("https://packages.example/simple") + manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + + command, kwargs = calls[0] + assert command[command.index("--default-index") + 1] == "https://packages.example/simple" + assert "--index" not in command + assert kwargs["cwd"] == pyproject_path.parent + + +def test_package_manager_reads_metadata_from_resolved_private_source( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["private-package==1.2.3"] + +[[tool.uv.index]] +name = "private" +url = "https://packages.example/simple" +explicit = true + +[tool.uv.sources] +private-package = { index = "private" } +""", + ) + commands: list[tuple[list[str], object]] = [] + + def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]: + commands.append((command, kwargs.get("cwd"))) + if command[:3] == ["uv", "pip", "compile"]: + return CompletedProcess( + command, + 0, + """private-package==1.2.3 + # via project + # from https://packages.example/simple +""", + "", + ) + + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: LicenseRef-Private-Proprietary +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(command, 0, "", "") + + def fail_public_lookup(*_args: object, **_kwargs: object) -> tuple[int, dict[str, object]]: + pytest.fail("public metadata must not replace the resolved private artifact") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + monkeypatch.setattr(RemotePackageInfo, "make_req", fail_public_lookup) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + package = package_info_manager.getPackages().pop() + + assert package.license == "LicenseRef-Private-Proprietary" + install_command, install_cwd = commands[1] + assert install_command[install_command.index("--index") + 1] == ( + "private=https://packages.example/simple" + ) + assert install_cwd == pyproject_path.parent + + +def test_private_index_credentials_are_not_exposed_in_command( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + private_index = "https://user:secret@packages.example/simple" + pyproject_path = write_pyproject( + tmp_path, + f""" +[project] +name = "project" +version = "1.0.0" +dependencies = ["private-package==1.2.3"] + +[tool.uv] +index-url = "{private_index}" +""", + ) + install_call: tuple[list[str], dict[str, object]] | None = None + + def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]: + nonlocal install_call + if command[:3] == ["uv", "pip", "compile"]: + return CompletedProcess( + command, + 0, + "private-package==1.2.3\n # from https://packages.example/simple\n", + "", + ) + + install_call = command, kwargs + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: LicenseRef-Private-Proprietary +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(command, 0, "", "") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + monkeypatch.setattr( + RemotePackageInfo, + "make_req", + lambda *_args, **_kwargs: pytest.fail("the resolved private artifact must be used"), + ) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + package = package_info_manager.getPackages().pop() + + assert package.license == "LicenseRef-Private-Proprietary" + assert install_call is not None + install_command, install_kwargs = install_call + assert "secret" not in " ".join(install_command) + install_environment = install_kwargs["env"] + assert isinstance(install_environment, dict) + assert install_environment["UV_INDEX"] == private_index + + +def test_resolved_public_source_ignores_installed_private_homonym( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["homonym==1.2.3"] +""", + ) + installed_metadata = Message() + installed_metadata["Name"] = "homonym" + installed_metadata["Version"] = "1.2.3" + installed_metadata["License-Expression"] = "LicenseRef-Private-Proprietary" + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + if command[:3] != ["uv", "pip", "compile"]: + pytest.fail("public metadata already declares a usable license") + return CompletedProcess( + command, + 0, + "homonym==1.2.3\n # from https://pypi.org/simple\n", + "", + ) + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "homonym", + "version": "1.2.3", + "license_expression": "MIT", + } + } + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + monkeypatch.setattr( + "licensecheck.packageinforesolver.metadata.metadata", + lambda _name: installed_metadata, + ) + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + package = package_info_manager.getPackages().pop() + + assert package.version == "1.2.3" + assert package.license == "MIT" + + def test_package_manager_uses_private_index_when_pypi_is_missing( package_info_manager: PackageInfoManager, monkeypatch: pytest.MonkeyPatch, @@ -287,7 +505,7 @@ def test_getPackagePypiLocalNotFound() -> None: def test_getPackages(package_info_manager: PackageInfoManager) -> None: - package_info_manager.reqs = {aux_packageinfo("requests")} + package_info_manager.reqs = {Requirement("requests")} packages = package_info_manager.getPackages() package = packages.pop() assert package.name == "requests" @@ -296,7 +514,7 @@ def test_getPackages(package_info_manager: PackageInfoManager) -> None: def test_getPackagesNotFound(package_info_manager: PackageInfoManager) -> None: - package_info_manager.reqs = {aux_packageinfo("this_package_does_not_exist")} + package_info_manager.reqs = {Requirement("this_package_does_not_exist")} packages = package_info_manager.getPackages() package = packages.pop() @@ -339,6 +557,88 @@ def test_normalize_license(lice: str, normalized: str) -> None: assert normalize_license(lice) == normalized +@pytest.mark.parametrize( + ("contents", "name", "author", "homepage"), + [ + ( + """ +[tool.poetry] +name = "poetry-local" +version = "1.2.3" +license = "MIT" +authors = ["Poetry Author "] +homepage = "https://poetry.example" +""", + "poetry-local", + "Poetry Author ", + "https://poetry.example", + ), + ( + """ +[tool.flit.metadata] +module = "flit_local" +dist-name = "flit-local" +version = "1.2.3" +license = "MIT" +author = "Flit Author" +home-page = "https://flit.example" +""", + "flit-local", + "Flit Author", + "https://flit.example", + ), + ], +) +def test_read_project_package_supports_legacy_metadata( + tmp_path: Path, + contents: str, + name: str, + author: str, + homepage: str, +) -> None: + pyproject_path = write_pyproject(tmp_path, contents) + + package = PackageInfoManager._read_project_package(pyproject_path) + + assert package is not None + assert package.name == name + assert package.version == "1.2.3" + assert package.license == "MIT" + assert package.author == author + assert package.homePage == homepage + + +@pytest.mark.parametrize( + ("license_metadata", "expected"), + [ + ('license = "MIT OR GPL-3.0-only"', "GPL-3.0-only;; MIT"), + ( + 'classifiers = ["License :: OSI Approved :: MIT License"]', + "MIT License", + ), + ], +) +def test_read_project_package_normalizes_license_metadata( + tmp_path: Path, + license_metadata: str, + expected: str, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + f""" +[project] +name = "local-package" +version = "1.2.3" +{license_metadata} +""", + ) + + package = PackageInfoManager._read_project_package(pyproject_path) + + assert package is not None + assert package.license == expected + + def test_unpinned_requirement_does_not_crash(package_info_manager: PackageInfoManager) -> None: package_info_manager.reqs = {Requirement("sample")} @@ -603,6 +903,15 @@ def test_resolve_requirements_does_not_skip_inactive_editable_source( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + write_pyproject( + tmp_path / "local-idna", + """ +[project] +name = "idna" +version = "999" +license = "LicenseRef-Local-Proprietary" +""", + ) pyproject_path = write_pyproject( tmp_path / "project", """ @@ -630,6 +939,22 @@ def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "idna", + "version": "3.10", + "license_expression": "BSD-3-Clause", + } + } + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + package_info_manager.resolve_requirements( requirements_paths={str(pyproject_path)}, groups=set(), @@ -638,6 +963,10 @@ def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: ) assert {str(requirement) for requirement in package_info_manager.reqs} == {"idna==3.10"} + package = package_info_manager.getPackages().pop() + assert package.version == "3.10" + assert package.license == "BSD-3-Clause" + assert "idna" not in package_info_manager.local_projects def test_resolve_requirements_uses_uv_prerelease_setting( @@ -983,3 +1312,38 @@ def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: extras=set(), skip_dependencies=set(), ) + + +def test_resolve_requirements_falls_back_when_uv_is_unavailable( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["dependency"] +""", + ) + fallback_calls: list[Path] = [] + + def missing_uv(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + message = "uv" + raise FileNotFoundError(message) + + def fake_gather(**kwargs: object) -> set[Requirement]: + requirements_path = kwargs["requirementsPath"] + assert isinstance(requirements_path, Path) + fallback_calls.append(requirements_path) + return {Requirement("dependency==1.2.3")} + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", missing_uv) + monkeypatch.setattr("licensecheck.packageinforesolver.gather", fake_gather) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + + assert fallback_calls == [pyproject_path] + assert {str(requirement) for requirement in package_info_manager.reqs} == {"dependency==1.2.3"} From 61ebaa9f4ac7d9e6074003d71ea51ea99f3752aa Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 09:00:23 -0400 Subject: [PATCH 07/14] Allow matching custom license references --- licensecheck/checker.py | 11 +++++++++++ licensecheck/io/cli.py | 1 + tests/test_checker.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/licensecheck/checker.py b/licensecheck/checker.py index 25355d7..565522e 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -22,12 +22,21 @@ def _package_matches(package: PackageInfo, patterns: set[str]) -> bool: ) +def _matches_custom_license(this_license_text: str | None, dependency_license: str) -> bool: + if not this_license_text: + return False + project_license = this_license_text.strip().casefold() + dependency_license = dependency_license.strip().casefold() + return project_license.startswith("licenseref-") and project_license == dependency_license + + def check( requirements_paths: set[str], groups: set[str], extras: set[str], this_license: License, package_info_manager: PackageInfoManager, + this_license_text: str | None = None, ignore_packages: set[str] | None = None, fail_packages: set[str] | None = None, ignore_licenses: set[str] | None = None, @@ -68,6 +77,8 @@ def check( package.licenseCompat = True elif _package_matches(package, fail_packages): pass # package.licenseCompat = False + elif _matches_custom_license(this_license_text, str(package.license)): + package.licenseCompat = True # Else get compat with myLice else: package.licenseCompat = license_matrix.depCompatWMyLice( diff --git a/licensecheck/io/cli.py b/licensecheck/io/cli.py index cc59ba2..1a8d28c 100644 --- a/licensecheck/io/cli.py +++ b/licensecheck/io/cli.py @@ -170,6 +170,7 @@ def main(licensecheckConf: LC_Config) -> ExitCode: groups=licensecheckConf.groups, extras=licensecheckConf.extras, this_license=this_license, + this_license_text=this_license_text, package_info_manager=package_info_manager, ignore_packages=licensecheckConf.ignore_packages, fail_packages=licensecheckConf.fail_packages, diff --git a/tests/test_checker.py b/tests/test_checker.py index a2f41f6..bb7dc8b 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -96,3 +96,34 @@ def test_ignore_packages_can_match_versions( ) assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("dependency_license", "expected_incompatible"), + [ + ("LicenseRef-CVector-Proprietary", False), + ("licenseref-cvector-proprietary", False), + ("LicenseRef-Other-Proprietary", True), + ("PROPRIETARY", True), + ], +) +def test_matching_custom_license_reference_is_compatible( + mock_package_info_manager: PackageInfoManager, + dependency_license: str, + *, + expected_incompatible: bool, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo(name="private-package", version="1.2.3", license=dependency_license) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.PROPRIETARY, + this_license_text="LicenseRef-CVector-Proprietary", + package_info_manager=mock_package_info_manager, + ) + + assert incompatible == expected_incompatible, packages From 96579a413cdf49dc99514cab7ef06d4746ae52e2 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 11:43:12 -0400 Subject: [PATCH 08/14] Allow exact custom license references --- README.md | 2 ++ licensecheck/checker.py | 13 +++++++++ licensecheck/io/cli.py | 6 ++++ licensecheck/models/config.py | 1 + pyproject.toml | 1 + tests/models/test_config.py | 2 ++ tests/test_checker.py | 55 +++++++++++++++++++++++++++++++++++ 7 files changed, 80 insertions(+) diff --git a/README.md b/README.md index fb70461..6f4e31a 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,7 @@ file = "" # Output file (leave empty for stdout) ignore_packages = [] # Names, name==version entries, or globs to ignore fail_packages = [] # Names, name==version entries, or globs that fail ignore_licenses = [] # Licenses to ignore +allowed_license_refs = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -237,6 +238,7 @@ zero = false # Return non-zero exit code for incompatible licen "groups": [], "hide_output_parameters": [], "ignore_licenses": [], + "allowed_license_refs": [], "ignore_packages": [], "license": "mit", "only_licenses": [], diff --git a/licensecheck/checker.py b/licensecheck/checker.py index 565522e..a92c43d 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -30,6 +30,13 @@ def _matches_custom_license(this_license_text: str | None, dependency_license: s return project_license.startswith("licenseref-") and project_license == dependency_license +def _matches_allowed_license_ref(allowed_license_refs: set[str], dependency_license: str) -> bool: + dependency_license = dependency_license.strip().casefold() + return dependency_license.startswith("licenseref-") and dependency_license in { + license_ref.strip().casefold() for license_ref in allowed_license_refs + } + + def check( requirements_paths: set[str], groups: set[str], @@ -40,6 +47,7 @@ def check( ignore_packages: set[str] | None = None, fail_packages: set[str] | None = None, ignore_licenses: set[str] | None = None, + allowed_license_refs: set[str] | None = None, fail_licenses: set[str] | None = None, only_licenses: set[str] | None = None, skip_dependencies: set[str] | None = None, @@ -48,6 +56,7 @@ def check( ignore_packages = ignore_packages or set() fail_packages = fail_packages or set() ignore_licenses = ignore_licenses or set() + allowed_license_refs = allowed_license_refs or set() fail_licenses = fail_licenses or set() only_licenses = only_licenses or set() skip_dependencies = skip_dependencies or set() @@ -77,6 +86,10 @@ def check( package.licenseCompat = True elif _package_matches(package, fail_packages): pass # package.licenseCompat = False + elif license_matrix.licenseType(str(package.license), ignore_licenses) & failLicensesType: + pass + elif _matches_allowed_license_ref(allowed_license_refs, str(package.license)): + package.licenseCompat = True elif _matches_custom_license(this_license_text, str(package.license)): package.licenseCompat = True # Else get compat with myLice diff --git a/licensecheck/io/cli.py b/licensecheck/io/cli.py index 1a8d28c..809b64a 100644 --- a/licensecheck/io/cli.py +++ b/licensecheck/io/cli.py @@ -85,6 +85,11 @@ def cli() -> None: # pragma: no cover help="set of licenses to ignore (skipped, compat may still be False)", nargs="+", ) + parser.add_argument( + "--allowed-license-refs", + help="set of exact LicenseRef-* identifiers to accept", + nargs="+", + ) parser.add_argument( "--fail-licenses", help="set of licenses to fail (compat=False)", @@ -175,6 +180,7 @@ def main(licensecheckConf: LC_Config) -> ExitCode: ignore_packages=licensecheckConf.ignore_packages, fail_packages=licensecheckConf.fail_packages, ignore_licenses=licensecheckConf.ignore_licenses, + allowed_license_refs=licensecheckConf.allowed_license_refs, fail_licenses=licensecheckConf.fail_licenses, only_licenses=licensecheckConf.only_licenses, skip_dependencies=licensecheckConf.skip_dependencies, diff --git a/licensecheck/models/config.py b/licensecheck/models/config.py index 79a859d..ab1ea4a 100644 --- a/licensecheck/models/config.py +++ b/licensecheck/models/config.py @@ -25,6 +25,7 @@ class LC_Config(DefaultOnNoneModel): ignore_packages: set[str] = field(default_factory=set) fail_packages: set[str] = field(default_factory=set) ignore_licenses: set[str] = field(default_factory=set) + allowed_license_refs: set[str] = field(default_factory=set) fail_licenses: set[str] = field(default_factory=set) only_licenses: set[str] = field(default_factory=set) skip_dependencies: set[str] = field(default_factory=set) diff --git a/pyproject.toml b/pyproject.toml index df4b73b..40fdd3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,7 @@ file = "" # Output file (leave empty for stdout) ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore +allowed_license_refs = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) diff --git a/tests/models/test_config.py b/tests/models/test_config.py index 40219fa..074f156 100644 --- a/tests/models/test_config.py +++ b/tests/models/test_config.py @@ -21,6 +21,7 @@ def test_basic_config() -> None: ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore +allowed_license_refs = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -44,6 +45,7 @@ def test_basic_config2() -> None: ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore +allowed_license_refs = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) diff --git a/tests/test_checker.py b/tests/test_checker.py index bb7dc8b..ee5e5e8 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -127,3 +127,58 @@ def test_matching_custom_license_reference_is_compatible( ) assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("dependency_license", "allowed_license_refs", "fail_licenses", "expected_incompatible"), + [ + ( + "LicenseRef-NVIDIA-Proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + None, + False, + ), + ( + "licenseref-nvidia-proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + None, + False, + ), + ( + "LicenseRef-Other-Proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + None, + True, + ), + ("PROPRIETARY", {"PROPRIETARY"}, None, True), + ( + "LicenseRef-NVIDIA-Proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + {"PROPRIETARY"}, + True, + ), + ], +) +def test_allowed_license_refs_match_exact_raw_references( + mock_package_info_manager: PackageInfoManager, + dependency_license: str, + allowed_license_refs: set[str], + fail_licenses: set[str] | None, + *, + expected_incompatible: bool, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo(name="private-package", version="1.2.3", license=dependency_license) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.PROPRIETARY, + package_info_manager=mock_package_info_manager, + allowed_license_refs=allowed_license_refs, + fail_licenses=fail_licenses, + ) + + assert incompatible == expected_incompatible, packages From 8d476eb7bfca73886f213ef43857f06e5edb14f3 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 11:46:40 -0400 Subject: [PATCH 09/14] Use full allowed license references name --- README.md | 4 ++-- licensecheck/checker.py | 12 +++++++----- licensecheck/io/cli.py | 4 ++-- licensecheck/models/config.py | 2 +- pyproject.toml | 2 +- tests/models/test_config.py | 4 ++-- tests/test_checker.py | 13 +++++++++---- 7 files changed, 24 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6f4e31a..e4f238d 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ file = "" # Output file (leave empty for stdout) ignore_packages = [] # Names, name==version entries, or globs to ignore fail_packages = [] # Names, name==version entries, or globs that fail ignore_licenses = [] # Licenses to ignore -allowed_license_refs = [] # Exact LicenseRef identifiers to accept +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -238,7 +238,7 @@ zero = false # Return non-zero exit code for incompatible licen "groups": [], "hide_output_parameters": [], "ignore_licenses": [], - "allowed_license_refs": [], + "allowed_license_references": [], "ignore_packages": [], "license": "mit", "only_licenses": [], diff --git a/licensecheck/checker.py b/licensecheck/checker.py index a92c43d..e2c0f02 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -30,10 +30,12 @@ def _matches_custom_license(this_license_text: str | None, dependency_license: s return project_license.startswith("licenseref-") and project_license == dependency_license -def _matches_allowed_license_ref(allowed_license_refs: set[str], dependency_license: str) -> bool: +def _matches_allowed_license_reference( + allowed_license_references: set[str], dependency_license: str +) -> bool: dependency_license = dependency_license.strip().casefold() return dependency_license.startswith("licenseref-") and dependency_license in { - license_ref.strip().casefold() for license_ref in allowed_license_refs + license_ref.strip().casefold() for license_ref in allowed_license_references } @@ -47,7 +49,7 @@ def check( ignore_packages: set[str] | None = None, fail_packages: set[str] | None = None, ignore_licenses: set[str] | None = None, - allowed_license_refs: set[str] | None = None, + allowed_license_references: set[str] | None = None, fail_licenses: set[str] | None = None, only_licenses: set[str] | None = None, skip_dependencies: set[str] | None = None, @@ -56,7 +58,7 @@ def check( ignore_packages = ignore_packages or set() fail_packages = fail_packages or set() ignore_licenses = ignore_licenses or set() - allowed_license_refs = allowed_license_refs or set() + allowed_license_references = allowed_license_references or set() fail_licenses = fail_licenses or set() only_licenses = only_licenses or set() skip_dependencies = skip_dependencies or set() @@ -88,7 +90,7 @@ def check( pass # package.licenseCompat = False elif license_matrix.licenseType(str(package.license), ignore_licenses) & failLicensesType: pass - elif _matches_allowed_license_ref(allowed_license_refs, str(package.license)): + elif _matches_allowed_license_reference(allowed_license_references, str(package.license)): package.licenseCompat = True elif _matches_custom_license(this_license_text, str(package.license)): package.licenseCompat = True diff --git a/licensecheck/io/cli.py b/licensecheck/io/cli.py index 809b64a..9dc0269 100644 --- a/licensecheck/io/cli.py +++ b/licensecheck/io/cli.py @@ -86,7 +86,7 @@ def cli() -> None: # pragma: no cover nargs="+", ) parser.add_argument( - "--allowed-license-refs", + "--allowed-license-references", help="set of exact LicenseRef-* identifiers to accept", nargs="+", ) @@ -180,7 +180,7 @@ def main(licensecheckConf: LC_Config) -> ExitCode: ignore_packages=licensecheckConf.ignore_packages, fail_packages=licensecheckConf.fail_packages, ignore_licenses=licensecheckConf.ignore_licenses, - allowed_license_refs=licensecheckConf.allowed_license_refs, + allowed_license_references=licensecheckConf.allowed_license_references, fail_licenses=licensecheckConf.fail_licenses, only_licenses=licensecheckConf.only_licenses, skip_dependencies=licensecheckConf.skip_dependencies, diff --git a/licensecheck/models/config.py b/licensecheck/models/config.py index ab1ea4a..860a2ff 100644 --- a/licensecheck/models/config.py +++ b/licensecheck/models/config.py @@ -25,7 +25,7 @@ class LC_Config(DefaultOnNoneModel): ignore_packages: set[str] = field(default_factory=set) fail_packages: set[str] = field(default_factory=set) ignore_licenses: set[str] = field(default_factory=set) - allowed_license_refs: set[str] = field(default_factory=set) + allowed_license_references: set[str] = field(default_factory=set) fail_licenses: set[str] = field(default_factory=set) only_licenses: set[str] = field(default_factory=set) skip_dependencies: set[str] = field(default_factory=set) diff --git a/pyproject.toml b/pyproject.toml index 40fdd3a..bb41df3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,7 @@ file = "" # Output file (leave empty for stdout) ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore -allowed_license_refs = [] # Exact LicenseRef identifiers to accept +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) diff --git a/tests/models/test_config.py b/tests/models/test_config.py index 074f156..c20b088 100644 --- a/tests/models/test_config.py +++ b/tests/models/test_config.py @@ -21,7 +21,7 @@ def test_basic_config() -> None: ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore -allowed_license_refs = [] # Exact LicenseRef identifiers to accept +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -45,7 +45,7 @@ def test_basic_config2() -> None: ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore -allowed_license_refs = [] # Exact LicenseRef identifiers to accept +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) diff --git a/tests/test_checker.py b/tests/test_checker.py index ee5e5e8..4a4ded2 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -130,7 +130,12 @@ def test_matching_custom_license_reference_is_compatible( @pytest.mark.parametrize( - ("dependency_license", "allowed_license_refs", "fail_licenses", "expected_incompatible"), + ( + "dependency_license", + "allowed_license_references", + "fail_licenses", + "expected_incompatible", + ), [ ( "LicenseRef-NVIDIA-Proprietary", @@ -159,10 +164,10 @@ def test_matching_custom_license_reference_is_compatible( ), ], ) -def test_allowed_license_refs_match_exact_raw_references( +def test_allowed_license_references_match_exact_raw_references( mock_package_info_manager: PackageInfoManager, dependency_license: str, - allowed_license_refs: set[str], + allowed_license_references: set[str], fail_licenses: set[str] | None, *, expected_incompatible: bool, @@ -177,7 +182,7 @@ def test_allowed_license_refs_match_exact_raw_references( extras=set(), this_license=License.PROPRIETARY, package_info_manager=mock_package_info_manager, - allowed_license_refs=allowed_license_refs, + allowed_license_references=allowed_license_references, fail_licenses=fail_licenses, ) From a4fffa1151210d42f570bcf2b2a4434496d30127 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 12:06:04 -0400 Subject: [PATCH 10/14] Prefer recognizable explicit license metadata --- licensecheck/packageinforesolver.py | 57 +++++++++++++++++++++++------ tests/test_packageinfo.py | 43 +++++++++++++++++++++- 2 files changed, 86 insertions(+), 14 deletions(-) diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index 4f2205d..5a29655 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -37,6 +37,10 @@ RAW_JOINS = " AND " HTTP_OK = 200 +EXPLICIT_LICENSE_ALIASES = { + "Apache 2.0": "Apache-2.0", +} + class UvUnavailableError(RuntimeError): """Raised when the optional uv executable is unavailable.""" @@ -68,6 +72,35 @@ def _has_usable_license(license_value: str | None) -> bool: ) +def _recognizable_explicit_license(license_value: str | None) -> str | None: + if not _has_usable_license(license_value): + return None + + value = str(license_value).strip() + if re.fullmatch(r"LicenseRef-[A-Za-z0-9.-]+", value): + return value + if value in EXPLICIT_LICENSE_ALIASES: + return EXPLICIT_LICENSE_ALIASES[value] + + with contextlib.suppress(license_expression.ExpressionError): + license_expression.get_spdx_licensing().parse(value, validate=True) + return value + return None + + +def _license_from_metadata( + license_expression_value: str | None, + classifiers: list[str] | None, + legacy_license: str | None, +) -> str | None: + return ( + license_expression_value + or _recognizable_explicit_license(legacy_license) + or from_classifiers(classifiers) + or legacy_license + ) + + def _versions_match(expected: str | None, actual: str | None) -> bool: if expected is None: return True @@ -819,10 +852,10 @@ def __init__(self, package: PackageInfo) -> None: self.meta = metadata.metadata(package.name) def get_license(self) -> str | None: - return ( - self.meta.get("License-Expression") - or from_classifiers(self.meta.get_all("Classifier")) - or self.meta.get("License") + return _license_from_metadata( + self.meta.get("License-Expression"), + self.meta.get_all("Classifier"), + self.meta.get("License"), ) def get_name(self) -> str | None: @@ -922,10 +955,10 @@ def lazy_fetch(self) -> None: def get_license(self) -> str | None: self.lazy_fetch() - return ( - self.meta.get("License-Expression") - or from_classifiers(self.meta.get_all("Classifier")) - or self.meta.get("License") + return _license_from_metadata( + self.meta.get("License-Expression"), + self.meta.get_all("Classifier"), + self.meta.get("License"), ) def get_name(self) -> str | None: @@ -1006,10 +1039,10 @@ def get_author(self) -> str: def get_license(self) -> str: response = self._response() - return ( - response.info.license_expression - or from_classifiers(response.info.classifiers) - or response.info.license + return _license_from_metadata( + response.info.license_expression, + response.info.classifiers, + response.info.license, ) def get_size(self) -> int | None: diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index c73dce9..21c88ed 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -14,6 +14,7 @@ LocalPackageInfo, PackageInfoManager, RemotePackageInfo, + _license_from_metadata, from_classifiers, normalize_license, ) @@ -64,7 +65,7 @@ def test_getPackageInfoPypi(remote_package_info: RemotePackageInfo) -> None: assert pkg.get_name() == "requests" assert pkg.get_author() == "Kenneth Reitz" - assert pkg.get_license() == "Apache Software License" + assert pkg.get_license() == "Apache-2.0" def test_remote_package_info_uses_versioned_pypi_endpoint( @@ -510,7 +511,7 @@ def test_getPackages(package_info_manager: PackageInfoManager) -> None: package = packages.pop() assert package.name == "requests" assert package.author == "Kenneth Reitz" - assert package.license == "Apache Software License" + assert package.license == "Apache-2.0" def test_getPackagesNotFound(package_info_manager: PackageInfoManager) -> None: @@ -536,6 +537,44 @@ def test_licenseFromEmptyClassifierlist() -> None: assert licenses == [None] +@pytest.mark.parametrize( + ("license_expression", "classifier", "legacy_license", "expected"), + [ + ( + "MIT", + "License :: Other/Proprietary License", + "LicenseRef-Example-Proprietary", + "MIT", + ), + ( + None, + "License :: Other/Proprietary License", + "LicenseRef-Example-Proprietary", + "LicenseRef-Example-Proprietary", + ), + ( + None, + "License :: Other/Proprietary License", + "Apache 2.0", + "Apache-2.0", + ), + ( + None, + "License :: OSI Approved :: MIT License", + "unrecognized license text", + "MIT License", + ), + ], +) +def test_explicit_license_metadata_precedes_classifiers_when_recognizable( + license_expression: str | None, + classifier: str, + legacy_license: str, + expected: str, +) -> None: + assert _license_from_metadata(license_expression, [classifier], legacy_license) == expected + + def test_getModuleSize() -> None: local_package_info = LocalPackageInfo(aux_packageinfo("this_package_does_not_exist")) local_package_info.get_size() From b64e7aca70c3644312de609b70c90d527db9bb2a Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 12:50:33 -0400 Subject: [PATCH 11/14] Support exact-version license overrides --- README.md | 2 + licensecheck/checker.py | 20 ++++++++++ licensecheck/io/cli.py | 1 + licensecheck/io/fmt.py | 8 +++- licensecheck/models/config.py | 28 ++++++++++++++ licensecheck/models/packageinfo.py | 2 + pyproject.toml | 1 + tests/io/test_cli_main.py | 1 + tests/io/test_formatter.py | 13 +++++++ tests/models/test_config.py | 22 +++++++++++ tests/test_checker.py | 60 ++++++++++++++++++++++++++++++ 11 files changed, 157 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4f238d..82d264d 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,7 @@ groups = [] # List of selected groups extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) ignore_packages = [] # Names, name==version entries, or globs to ignore +license_overrides = { "sample==1.2.3" = "BSD-3-Clause" } # Reviewed licenses for exact versions fail_packages = [] # Names, name==version entries, or globs that fail ignore_licenses = [] # Licenses to ignore allowed_license_references = [] # Exact LicenseRef identifiers to accept @@ -240,6 +241,7 @@ zero = false # Return non-zero exit code for incompatible licen "ignore_licenses": [], "allowed_license_references": [], "ignore_packages": [], + "license_overrides": {"sample==1.2.3": "BSD-3-Clause"}, "license": "mit", "only_licenses": [], "pypi_api": "https://pypi.org", diff --git a/licensecheck/checker.py b/licensecheck/checker.py index e2c0f02..ef9f4a7 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -4,6 +4,9 @@ from fnmatch import fnmatch +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + from licensecheck import license_matrix from licensecheck.models.constants import JOINS from licensecheck.models.license import License @@ -39,6 +42,18 @@ def _matches_allowed_license_reference( } +def _license_override(package: PackageInfo, license_overrides: dict[str, str]) -> str | None: + if package.version is None: + return None + for package_requirement, license_value in license_overrides.items(): + requirement = Requirement(package_requirement) + if canonicalize_name(requirement.name) == canonicalize_name( + package.name + ) and requirement.specifier.contains(package.version, prereleases=True): + return license_value.strip() + return None + + def check( requirements_paths: set[str], groups: set[str], @@ -47,6 +62,7 @@ def check( package_info_manager: PackageInfoManager, this_license_text: str | None = None, ignore_packages: set[str] | None = None, + license_overrides: dict[str, str] | None = None, fail_packages: set[str] | None = None, ignore_licenses: set[str] | None = None, allowed_license_references: set[str] | None = None, @@ -56,6 +72,7 @@ def check( ) -> tuple[bool, set[PackageInfo]]: # Def values ignore_packages = ignore_packages or set() + license_overrides = license_overrides or {} fail_packages = fail_packages or set() ignore_licenses = ignore_licenses or set() allowed_license_references = allowed_license_references or set() @@ -82,6 +99,9 @@ def check( # Check it is compatible with packages and add a note packages = package_info_manager.getPackages() for package in packages: + if override := _license_override(package, license_overrides): + package.license = override + package.licenseSource = "configured override" # Deal with --ignore-packages and --fail-packages package.licenseCompat = False if _package_matches(package, ignore_packages): diff --git a/licensecheck/io/cli.py b/licensecheck/io/cli.py index 9dc0269..6a30060 100644 --- a/licensecheck/io/cli.py +++ b/licensecheck/io/cli.py @@ -178,6 +178,7 @@ def main(licensecheckConf: LC_Config) -> ExitCode: this_license_text=this_license_text, package_info_manager=package_info_manager, ignore_packages=licensecheckConf.ignore_packages, + license_overrides=licensecheckConf.license_overrides, fail_packages=licensecheckConf.fail_packages, ignore_licenses=licensecheckConf.ignore_licenses, allowed_license_references=licensecheckConf.allowed_license_references, diff --git a/licensecheck/io/fmt.py b/licensecheck/io/fmt.py index 0d71b2d..bdb21cc 100644 --- a/licensecheck/io/fmt.py +++ b/licensecheck/io/fmt.py @@ -163,6 +163,9 @@ def ansi( table.add_column("Package", style="magenta") if license_bool := "license" in packages[0]: table.add_column("License(s)", style="magenta") + license_source_bool = any("licenseSource" in package for package in packages) + if license_source_bool: + table.add_column("License Source", style="magenta") licenseCompat = ( "[red]✖[/]", "[green]✔[/]", @@ -173,6 +176,7 @@ def ansi( ([licenseCompat[x.get("licenseCompat", 0)]] if licensecompat_bool else []) + ([x.get("name")] if name_bool else []) + ([x.get("license")] if license_bool else []) + + ([x.get("licenseSource", "")] if license_source_bool else []) ) ) for x in packages @@ -227,6 +231,7 @@ def markdown( "homePage": "HomePage", "author": "Author", "license": "License", + "licenseSource": "License Source", "licenseCompat": "Compatible", "size": "Size", } @@ -295,7 +300,8 @@ def rawCsv( _ = myLice string = StringIO() - writer = csv.DictWriter(string, fieldnames=list(packages[0]), lineterminator="\n") + fieldnames = list(dict.fromkeys(key for package in packages for key in package)) + writer = csv.DictWriter(string, fieldnames=fieldnames, lineterminator="\n") writer.writeheader() writer.writerows(packages) return string.getvalue() diff --git a/licensecheck/models/config.py b/licensecheck/models/config.py index 860a2ff..ab46c9c 100644 --- a/licensecheck/models/config.py +++ b/licensecheck/models/config.py @@ -4,6 +4,7 @@ from typing import Any, Literal from depgather.models.defaultonnone import DefaultOnNoneModel +from packaging.requirements import InvalidRequirement, Requirement from pydantic import field_validator from licensecheck.io.fmt import FMT @@ -23,6 +24,7 @@ class LC_Config(DefaultOnNoneModel): groups: set[str] = field(default_factory=set) extras: set[str] = field(default_factory=set) ignore_packages: set[str] = field(default_factory=set) + license_overrides: dict[str, str] = field(default_factory=dict) fail_packages: set[str] = field(default_factory=set) ignore_licenses: set[str] = field(default_factory=set) allowed_license_references: set[str] = field(default_factory=set) @@ -37,3 +39,29 @@ def normalize_format(cls, value: Any) -> Any | Literal[FMT.simple]: if value not in FMT: return FMT.simple return value + + @field_validator("license_overrides") + @classmethod + def validate_license_overrides(cls, value: dict[str, str]) -> dict[str, str]: + for package, license_value in value.items(): + try: + requirement = Requirement(package) + except InvalidRequirement as exc: + message = f"Invalid license override package: {package}" + raise ValueError(message) from exc + + specifiers = list(requirement.specifier) + if ( + requirement.url + or requirement.extras + or requirement.marker + or len(specifiers) != 1 + or specifiers[0].operator != "==" + or specifiers[0].version.endswith(".*") + ): + message = f"License override packages must use an exact name==version: {package}" + raise ValueError(message) + if not license_value.strip(): + message = f"License override must not be empty: {package}" + raise ValueError(message) + return value diff --git a/licensecheck/models/packageinfo.py b/licensecheck/models/packageinfo.py index 7d70380..8b07cb1 100644 --- a/licensecheck/models/packageinfo.py +++ b/licensecheck/models/packageinfo.py @@ -16,6 +16,7 @@ class PackageInfo: homePage: str | None = None author: str | None = None license: str | None = None + licenseSource: str | None = field(default=None, compare=False, hash=False) licenseCompat: bool = False errorCode: int = 0 @@ -35,4 +36,5 @@ def get_filtered_dict(self, hide_output_parameters: set[str]) -> dict: k: (v if v is not None else UNKNOWN) for k, v in self.__dict__.items() if k.upper() not in hide_output_parameters_upper + and not (k == "licenseSource" and v is None) } diff --git a/pyproject.toml b/pyproject.toml index bb41df3..6e5396e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,6 +106,7 @@ groups = [] # List of selected groups extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) ignore_packages = [] # Packages/dependencies to ignore +license_overrides = {} # Reviewed licenses for exact package versions fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore allowed_license_references = [] # Exact LicenseRef identifiers to accept diff --git a/tests/io/test_cli_main.py b/tests/io/test_cli_main.py index f70f8b5..0226d8b 100644 --- a/tests/io/test_cli_main.py +++ b/tests/io/test_cli_main.py @@ -209,6 +209,7 @@ def fake_check(**kwargs: dict[str, Any]) -> tuple[Literal[False], list[Any]]: assert called["groups"] == config.groups assert called["extras"] == config.extras assert called["skip_dependencies"] == config.skip_dependencies + assert called["license_overrides"] == config.license_overrides def test_main_closes_output_file( diff --git a/tests/io/test_formatter.py b/tests/io/test_formatter.py index 4fffea2..692f759 100644 --- a/tests/io/test_formatter.py +++ b/tests/io/test_formatter.py @@ -5,6 +5,7 @@ import pytest from licensecheck.io import fmt +from licensecheck.io.fmt import FMT from licensecheck.models.license import License from licensecheck.models.packageinfo import PackageInfo @@ -88,3 +89,15 @@ def assert_eq(actual_input: str, expected_output: str) -> bool: return False return [x.strip() for x in actual] == [x.strip() for x in expected] + + +def test_override_source_is_visible() -> None: + package = PackageInfo( + name="example", + version="1.0.0", + license="BSD-3-Clause", + licenseSource="configured override", + licenseCompat=True, + ) + + assert "configured override" in fmt.fmt(FMT.simple, myLice, [package]) diff --git a/tests/models/test_config.py b/tests/models/test_config.py index c20b088..b550f78 100644 --- a/tests/models/test_config.py +++ b/tests/models/test_config.py @@ -1,3 +1,4 @@ +import pytest import tomli from licensecheck.io.fmt import FMT @@ -19,6 +20,7 @@ def test_basic_config() -> None: extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) ignore_packages = [] # Packages/dependencies to ignore +license_overrides = { "sample==1.2.3" = "MIT" } # Reviewed exact-version licenses fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore allowed_license_references = [] # Exact LicenseRef identifiers to accept @@ -35,6 +37,7 @@ def test_basic_config() -> None: conf = LC_Config.model_validate(raw_conf["tool"]["licensecheck"]) assert conf.format == FMT.simple assert conf.pypi_api == "https://pypi.org" + assert conf.license_overrides == {"sample==1.2.3": "MIT"} def test_basic_config2() -> None: @@ -58,3 +61,22 @@ def test_basic_config2() -> None: conf = LC_Config.model_validate(raw_conf["tool"]["licensecheck"]) assert conf.format == FMT.simple assert conf.pypi_api == "" + + +@pytest.mark.parametrize( + "package", + [ + "sample", + "sample>=1.2.3", + "sample==1.*", + "sample==1.2.3; python_version > '3.11'", + ], +) +def test_license_overrides_require_exact_versions(package: str) -> None: + with pytest.raises(ValueError, match="exact name==version"): + LC_Config.model_validate({"license_overrides": {package: "MIT"}}) + + +def test_license_overrides_require_nonempty_licenses() -> None: + with pytest.raises(ValueError, match="must not be empty"): + LC_Config.model_validate({"license_overrides": {"sample==1.2.3": " "}}) diff --git a/tests/test_checker.py b/tests/test_checker.py index 4a4ded2..fde6019 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -187,3 +187,63 @@ def test_allowed_license_references_match_exact_raw_references( ) assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("override_package", "expected_incompatible", "expected_license_source"), + [ + ("private-package==1.2.3", False, "configured override"), + ("PRIVATE_package==1.2.3", False, "configured override"), + ("private-package==1.2.4", True, None), + ], +) +def test_license_overrides_apply_only_to_exact_versions( + mock_package_info_manager: PackageInfoManager, + override_package: str, + *, + expected_incompatible: bool, + expected_license_source: str | None, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo( + name="private-package", + version="1.2.3", + license="Other/Proprietary License", + ) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.MIT, + package_info_manager=mock_package_info_manager, + license_overrides={override_package: "BSD-3-Clause"}, + ) + package = packages.pop() + + assert incompatible == expected_incompatible + assert package.license == ( + "BSD-3-Clause" if expected_license_source else "Other/Proprietary License" + ) + assert package.licenseSource == expected_license_source + + +def test_license_overrides_still_obey_license_deny_rules( + mock_package_info_manager: PackageInfoManager, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo(name="private-package", version="1.2.3", license="MIT") + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.MIT, + package_info_manager=mock_package_info_manager, + license_overrides={"private-package==1.2.3": "GPL-3.0"}, + fail_licenses={"GPL-3.0"}, + ) + + assert incompatible, packages From 7573fbbe4e28547990977472e15e14798d610d85 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 13:54:47 -0400 Subject: [PATCH 12/14] Code review --- README.md | 6 +- licensecheck/checker.py | 56 +++++++----- licensecheck/license_matrix/__init__.py | 13 ++- licensecheck/packageinforesolver.py | 110 ++++++++++++++++-------- 4 files changed, 125 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 82d264d..584811e 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,9 @@ uv run licensecheck --only-licenses mit apache --show-only-failing -g dev usage: licensecheck [-h] [--license LICENSE] [--format FORMAT] [--requirements-paths REQUIREMENTS_PATHS [REQUIREMENTS_PATHS ...]] [--groups GROUPS [GROUPS ...]] [--extras EXTRAS [EXTRAS ...]] [--file FILE] [--ignore-packages IGNORE_PACKAGES [IGNORE_PACKAGES ...]] [--fail-packages FAIL_PACKAGES [FAIL_PACKAGES ...]] - [--ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...]] [--fail-licenses FAIL_LICENSES [FAIL_LICENSES ...]] + [--ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...]] + [--allowed-license-references ALLOWED_LICENSE_REFERENCES [ALLOWED_LICENSE_REFERENCES ...]] + [--fail-licenses FAIL_LICENSES [FAIL_LICENSES ...]] [--only-licenses ONLY_LICENSES [ONLY_LICENSES ...]] [--skip-dependencies SKIP_DEPENDENCIES [SKIP_DEPENDENCIES ...]] [--hide-output-parameters HIDE_OUTPUT_PARAMETERS [HIDE_OUTPUT_PARAMETERS ...]] [--show-only-failing] @@ -174,6 +176,8 @@ options: List of packages/dependencies to fail (compat=False); names, name==version, and globs are supported --ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...] List of licenses to ignore (skipped, compat may still be False) + --allowed-license-references ALLOWED_LICENSE_REFERENCES [ALLOWED_LICENSE_REFERENCES ...] + List of exact LicenseRef-* identifiers to accept --fail-licenses FAIL_LICENSES [FAIL_LICENSES ...] List of licenses to fail (compat=False) --only-licenses ONLY_LICENSES [ONLY_LICENSES ...] diff --git a/licensecheck/checker.py b/licensecheck/checker.py index ef9f4a7..d575ded 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -4,7 +4,9 @@ from fnmatch import fnmatch +from loguru import logger from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet from packaging.utils import canonicalize_name from licensecheck import license_matrix @@ -25,14 +27,6 @@ def _package_matches(package: PackageInfo, patterns: set[str]) -> bool: ) -def _matches_custom_license(this_license_text: str | None, dependency_license: str) -> bool: - if not this_license_text: - return False - project_license = this_license_text.strip().casefold() - dependency_license = dependency_license.strip().casefold() - return project_license.startswith("licenseref-") and project_license == dependency_license - - def _matches_allowed_license_reference( allowed_license_references: set[str], dependency_license: str ) -> bool: @@ -42,15 +36,35 @@ def _matches_allowed_license_reference( } -def _license_override(package: PackageInfo, license_overrides: dict[str, str]) -> str | None: - if package.version is None: - return None +def _parse_license_overrides( + license_overrides: dict[str, str], +) -> list[tuple[str, SpecifierSet, str]]: + """Parse the configured overrides once, rather than once per package.""" + parsed: list[tuple[str, SpecifierSet, str]] = [] for package_requirement, license_value in license_overrides.items(): requirement = Requirement(package_requirement) - if canonicalize_name(requirement.name) == canonicalize_name( - package.name - ) and requirement.specifier.contains(package.version, prereleases=True): - return license_value.strip() + if not requirement.specifier: + # An unversioned override would silently apply to every version of the package. + logger.warning( + f"Ignoring license override '{package_requirement}': " + f"an exact name==version is required" + ) + continue + parsed.append( + (canonicalize_name(requirement.name), requirement.specifier, license_value.strip()) + ) + return parsed + + +def _license_override( + package: PackageInfo, license_overrides: list[tuple[str, SpecifierSet, str]] +) -> str | None: + if package.version is None: + return None + package_name = canonicalize_name(package.name) + for name, specifier, license_value in license_overrides: + if name == package_name and specifier.contains(package.version, prereleases=True): + return license_value return None @@ -60,6 +74,7 @@ def check( extras: set[str], this_license: License, package_info_manager: PackageInfoManager, + *, this_license_text: str | None = None, ignore_packages: set[str] | None = None, license_overrides: dict[str, str] | None = None, @@ -72,10 +87,13 @@ def check( ) -> tuple[bool, set[PackageInfo]]: # Def values ignore_packages = ignore_packages or set() - license_overrides = license_overrides or {} + parsed_license_overrides = _parse_license_overrides(license_overrides or {}) fail_packages = fail_packages or set() ignore_licenses = ignore_licenses or set() - allowed_license_references = allowed_license_references or set() + # The project's own license reference is always an accepted reference + allowed_license_references = (allowed_license_references or set()) | ( + {this_license_text} if this_license_text else set() + ) fail_licenses = fail_licenses or set() only_licenses = only_licenses or set() skip_dependencies = skip_dependencies or set() @@ -99,7 +117,7 @@ def check( # Check it is compatible with packages and add a note packages = package_info_manager.getPackages() for package in packages: - if override := _license_override(package, license_overrides): + if override := _license_override(package, parsed_license_overrides): package.license = override package.licenseSource = "configured override" # Deal with --ignore-packages and --fail-packages @@ -112,8 +130,6 @@ def check( pass elif _matches_allowed_license_reference(allowed_license_references, str(package.license)): package.licenseCompat = True - elif _matches_custom_license(this_license_text, str(package.license)): - package.licenseCompat = True # Else get compat with myLice else: package.licenseCompat = license_matrix.depCompatWMyLice( diff --git a/licensecheck/license_matrix/__init__.py b/licensecheck/license_matrix/__init__.py index 6696c66..61b23b0 100644 --- a/licensecheck/license_matrix/__init__.py +++ b/licensecheck/license_matrix/__init__.py @@ -52,6 +52,13 @@ with Path(THISDIR / "matrix.csv").open(mode="r", newline="", encoding="utf-8") as csv_file: LICENSE_MATRIX: list[list[str]] = list[list[str]](csv.reader(csv_file)) +# Look the matrix up by license name. The csv column/row order does not match the +# declaration order of the License enum, so positional indexing silently misreads rows. +LICENSE_MATRIX_ROWS: dict[str, list[str]] = {row[0]: row for row in LICENSE_MATRIX[1:]} +LICENSE_MATRIX_COLUMNS: dict[str, int] = { + name: index for index, name in enumerate(LICENSE_MATRIX[0]) +} + termToLicenseData = { "UNKNOWN": L.UNKNOWN, @@ -225,9 +232,7 @@ def liceCompat( return False try: - licenses = list(L) - row, col = licenses.index(myLicense) + 1, licenses.index(lice) + 1 - return LICENSE_MATRIX[row][col] == "1" - except (IndexError, ValueError): + return LICENSE_MATRIX_ROWS[myLicense.name][LICENSE_MATRIX_COLUMNS[lice.name]] == "1" + except (IndexError, KeyError): logger.warning(f"Licenses {myLicense} and {lice} cannot be compared") return False diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index 5a29655..a82aaa2 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -4,6 +4,7 @@ import configparser import contextlib +import functools import os import re import subprocess @@ -15,7 +16,7 @@ from importlib.metadata._meta import PackageMetadata from pathlib import Path from typing import Any -from urllib.parse import unquote, urlparse +from urllib.parse import urlparse from urllib.request import url2pathname import license_expression @@ -26,6 +27,7 @@ from depgather.models.pypijson import ProjectResponse from depgather.parse import gather from license_expression import Licensing +from loguru import logger from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name from packaging.version import InvalidVersion, Version @@ -41,6 +43,10 @@ "Apache 2.0": "Apache-2.0", } +# uv can block indefinitely (for example prompting for index credentials), so cap each call +UV_RESOLVE_TIMEOUT_SECONDS = 900 +UV_ARTIFACT_TIMEOUT_SECONDS = 600 + class UvUnavailableError(RuntimeError): """Raised when the optional uv executable is unavailable.""" @@ -72,18 +78,29 @@ def _has_usable_license(license_value: str | None) -> bool: ) +@functools.lru_cache(maxsize=1) +def _spdx_licensing() -> Licensing: + """ + Build the SPDX licensing index once. + + ``license_expression.get_spdx_licensing`` re-reads and re-parses a large vendored JSON + index on every call, so it must not be called per package. + """ + return license_expression.get_spdx_licensing() + + def _recognizable_explicit_license(license_value: str | None) -> str | None: if not _has_usable_license(license_value): return None value = str(license_value).strip() - if re.fullmatch(r"LicenseRef-[A-Za-z0-9.-]+", value): + if re.fullmatch(r"LicenseRef-[A-Za-z0-9.-]+", value, flags=re.IGNORECASE): return value if value in EXPLICIT_LICENSE_ALIASES: return EXPLICIT_LICENSE_ALIASES[value] with contextlib.suppress(license_expression.ExpressionError): - license_expression.get_spdx_licensing().parse(value, validate=True) + _spdx_licensing().parse(value, validate=True) return value return None @@ -134,12 +151,13 @@ def _requirement_key( ) +@functools.cache def _normalized_index_url(url: str) -> str: parsed = urlparse(url) if not parsed.scheme and Path(url).is_absolute(): return Path(url).resolve().as_uri().rstrip("/") if parsed.scheme == "file": - return Path(url2pathname(unquote(parsed.path))).resolve().as_uri().rstrip("/") + return Path(url2pathname(parsed.path)).resolve().as_uri().rstrip("/") normalized = url.rstrip("/").removesuffix("/simple") parsed = urlparse(normalized) if parsed.hostname: @@ -220,7 +238,6 @@ def _configured_uv_indexes( uv_config: dict[str, Any], config_directory: Path, ) -> list[_UvIndex]: - raw_indexes = uv_config.get("index", []) if isinstance(raw_indexes, dict): raw_indexes = [raw_indexes] @@ -357,8 +374,12 @@ def _editable_project_path(line: str, base_path: Path) -> Path | None: if parsed_url.scheme and parsed_url.scheme != "file": return None - path_value = parsed_url.path if parsed_url.scheme == "file" else target - path = Path(url2pathname(unquote(path_value))) + # `url2pathname` already percent-decodes, so decoding again would corrupt literal "%" paths. + if parsed_url.scheme == "file": + path = Path(url2pathname(parsed_url.path)) + else: + # A bare path is not a URL; only strip a trailing "#egg=" style fragment. + path = Path(target.split("#", 1)[0]) if not path.is_absolute(): path = base_path / path return path.resolve() @@ -371,7 +392,7 @@ def _requirement_project_path(requirement: Requirement, base_path: Path) -> Path if parsed_url.scheme != "file": return None - path = Path(url2pathname(unquote(parsed_url.path))) + path = Path(url2pathname(parsed_url.path)) if not path.is_absolute(): path = base_path / path return path.resolve() @@ -479,9 +500,14 @@ def _gather_uv_requirements( text=True, check=False, cwd=requirements_path.parent, + stdin=subprocess.DEVNULL, + timeout=UV_RESOLVE_TIMEOUT_SECONDS, ) except FileNotFoundError as error: raise UvUnavailableError from error + except subprocess.TimeoutExpired as error: + message = f"Timed out after {UV_RESOLVE_TIMEOUT_SECONDS}s running: {' '.join(command)}" + raise RuntimeError(message) from error except OSError as error: raise RuntimeError from error @@ -499,18 +525,27 @@ def _gather_uv_requirements( return parsed_requirements, editable_paths, sources +def _project_indexes(directory: Path) -> list[_UvIndex]: + """ + Collect the uv indexes that apply to ``directory``. + + This reads configuration files and the environment, so it is resolved once per + requirements file rather than once per requirement. + """ + uv_config, config_directory = _read_uv_configuration(directory) + return [ + *_configured_uv_indexes(uv_config, config_directory), + *_environment_uv_indexes(), + ] + + def _resolution_context( - requirements_path: Path, + directory: Path, + indexes: list[_UvIndex], requirement: Requirement, source_url: str | None, base_index_url: str, ) -> _UvResolutionContext: - directory = requirements_path.parent.resolve() - uv_config, config_directory = _read_uv_configuration(directory) - indexes = [ - *_configured_uv_indexes(uv_config, config_directory), - *_environment_uv_indexes(), - ] index_args, index_environment = ( _index_invocation_for_source(source_url, indexes) if source_url else ((), ()) ) @@ -570,19 +605,16 @@ def resolve_requirements( skip_dependencies=skip_dependencies, base_index_url=self.base_pypi_url, ) - except UvUnavailableError: - resolved_requirements = gather( - skipDependencies=skip_dependencies, - groups=groups, - extras=extras, - requirementsPath=requirements_path_obj, - base_index_url=self.base_pypi_url, - ) - editable_paths = set() - source_urls = {} - except RuntimeError: - if requirements_path_obj.name == "pyproject.toml": + except RuntimeError as error: + # UvUnavailableError is a RuntimeError; only a genuine resolution failure + # for a pyproject.toml is fatal. + if not isinstance(error, UvUnavailableError) and ( + requirements_path_obj.name == "pyproject.toml" + ): raise + logger.warning( + f"Falling back to the legacy resolver for {requirements_path_obj}: {error}" + ) resolved_requirements = gather( skipDependencies=skip_dependencies, groups=groups, @@ -602,10 +634,13 @@ def resolve_requirements( resolved_requirements, requirements_path_obj.parent, ) + directory = requirements_path_obj.parent.resolve() + indexes = _project_indexes(directory) for requirement in resolved_requirements: key = _requirement_key(requirement) self.resolution_contexts[key] = _resolution_context( - requirements_path_obj, + directory, + indexes, requirement, source_urls.get(key), self.base_pypi_url, @@ -750,6 +785,7 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: local_name = lpi.get_name() if local_matches else None local_license = lpi.get_license() if local_matches else None + preferred_index: IndexPackageInfo | None = None if context is not None and context.prefer_artifact: preferred_index = IndexPackageInfo( package=base_pkg_info, @@ -760,7 +796,6 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: pkg_info = PackageInfo( name=package.name, version=base_pkg_info.version or preferred_index.get_version(), - size=preferred_index.get_size(), homePage=preferred_index.get_homePage(), author=preferred_index.get_author(), license=preferred_index.get_license(), @@ -783,15 +818,14 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: needs_index = (not local_name and rpi.http_code != HTTP_OK) or not any( _has_usable_license(value) for value in (local_license, remote_license) ) - ipi = ( - IndexPackageInfo( + # Reuse the artifact fetch already attempted above rather than re-running `uv pip install` + ipi = preferred_index + if ipi is None and needs_index: + ipi = IndexPackageInfo( package=base_pkg_info, requirement=package, context=context, ) - if needs_index - else None - ) index_name = ipi.get_name() if ipi is not None else None index_license = ipi.get_license() if ipi is not None else None license_candidates = (local_license, index_license, remote_license) @@ -940,11 +974,17 @@ def lazy_fetch(self) -> None: check=False, cwd=self.context.directory if self.context is not None else None, env=run_environment, + stdin=subprocess.DEVNULL, + timeout=UV_ARTIFACT_TIMEOUT_SECONDS, ) - except OSError: + except (OSError, subprocess.TimeoutExpired) as error: + logger.warning(f"Could not fetch the artifact for {self.package.name}: {error}") return if result.returncode != 0: + logger.warning( + f"Could not fetch the artifact for {self.package.name}: {result.stderr}" + ) return for distribution in metadata.distributions(path=[target]): From 174cd750807f50996fe15b7ce98732c48066df15 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 14:13:08 -0400 Subject: [PATCH 13/14] Simplify comment --- licensecheck/license_matrix/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/licensecheck/license_matrix/__init__.py b/licensecheck/license_matrix/__init__.py index 61b23b0..40a48fa 100644 --- a/licensecheck/license_matrix/__init__.py +++ b/licensecheck/license_matrix/__init__.py @@ -52,8 +52,7 @@ with Path(THISDIR / "matrix.csv").open(mode="r", newline="", encoding="utf-8") as csv_file: LICENSE_MATRIX: list[list[str]] = list[list[str]](csv.reader(csv_file)) -# Look the matrix up by license name. The csv column/row order does not match the -# declaration order of the License enum, so positional indexing silently misreads rows. +# Look the matrix up by license name because the csv column/row order does not match. LICENSE_MATRIX_ROWS: dict[str, list[str]] = {row[0]: row for row in LICENSE_MATRIX[1:]} LICENSE_MATRIX_COLUMNS: dict[str, int] = { name: index for index, name in enumerate(LICENSE_MATRIX[0]) From 3d26581de610b08a44e697d7d5bb1ee6af6639a2 Mon Sep 17 00:00:00 2001 From: Joshua Napoli Date: Sat, 1 Aug 2026 14:17:03 -0400 Subject: [PATCH 14/14] Remove useless comment --- licensecheck/packageinforesolver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index a82aaa2..804d1be 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -374,7 +374,6 @@ def _editable_project_path(line: str, base_path: Path) -> Path | None: if parsed_url.scheme and parsed_url.scheme != "file": return None - # `url2pathname` already percent-decodes, so decoding again would corrupt literal "%" paths. if parsed_url.scheme == "file": path = Path(url2pathname(parsed_url.path)) else: