Support lockfile generation and include lockfiles for NodeJSToolBase subsystems

build-support/bin/generate_builtin_lockfiles.py

@@ -29,8 +29,20 @@
 from pants.backend.helm.subsystems.post_renderer import HelmPostRendererSubsystem
 from pants.backend.java.lint.google_java_format.subsystem import GoogleJavaFormatSubsystem
 from pants.backend.java.subsystems.junit import JUnit
+from pants.backend.javascript.lint.prettier.subsystem import Prettier
+from pants.backend.javascript.package_manager import PackageManager
+from pants.backend.javascript.subsystems.nodejs import NodeJS
+from pants.backend.javascript.subsystems.nodejs_tool import (
+    NodeJSToolBase,
+    _lockfile_dest_for_resource,
+    _parse_package_name_and_version,
+    _tool_package_json_bytes,
+)
 from pants.backend.kotlin.lint.ktlint.subsystem import KtlintSubsystem
 from pants.backend.nfpm.native_libs.elfdeps.subsystem import Elfdeps
+from pants.backend.openapi.lint.openapi_format.subsystem import OpenApiFormatSubsystem
+from pants.backend.openapi.lint.spectral.subsystem import SpectralSubsystem
+from pants.backend.openapi.subsystems.redocly import Redocly
 from pants.backend.python.goals.coverage_py import CoverageSubsystem
 from pants.backend.python.lint.add_trailing_comma.subsystem import AddTrailingComma
 from pants.backend.python.lint.autoflake.subsystem import Autoflake
@@ -52,6 +64,7 @@
 from pants.backend.python.subsystems.setuptools_scm import SetuptoolsSCM
 from pants.backend.python.subsystems.twine import TwineSubsystem
 from pants.backend.python.typecheck.mypy.subsystem import MyPy
+from pants.backend.python.typecheck.pyright.subsystem import Pyright
 from pants.backend.python.typecheck.pytype.subsystem import Pytype
 from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata
 from pants.backend.scala.lint.scalafmt.subsystem import ScalafmtSubsystem
@@ -102,6 +115,10 @@ class PythonTool(Tool[PythonToolRequirementsBase]): ...
 class JvmTool(Tool[JvmToolBase]): ...
 
 
+@dataclass
+class NodeJSTool(Tool[NodeJSToolBase]): ...
+
+
 all_python_tools = tuple(
     sorted(
         [
@@ -166,7 +183,23 @@ class JvmTool(Tool[JvmToolBase]): ...
 )
 
 
-name_to_tool = {tool.name: tool for tool in (all_python_tools + all_jvm_tools)}
+all_nodejs_tools = tuple(
+    sorted(
+        [
+            NodeJSTool(Prettier, "pants.backend.javascript.lint.prettier"),
+            NodeJSTool(Pyright, "pants.backend.experimental.python.typecheck.pyright"),
+            NodeJSTool(SpectralSubsystem, "pants.backend.experimental.openapi.lint.spectral"),
+            NodeJSTool(
+                OpenApiFormatSubsystem, "pants.backend.experimental.openapi.lint.openapi_format"
+            ),
+            NodeJSTool(Redocly, "pants.backend.experimental.openapi"),
+        ],
+        key=lambda tool: tool.name,
+    )
+)
+
+
+name_to_tool = {tool.name: tool for tool in (all_python_tools + all_jvm_tools + all_nodejs_tools)}
 
 
 def create_parser() -> argparse.ArgumentParser:
@@ -188,6 +221,9 @@ def create_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--all-jvm", action="store_true", help="Regenerate all builtin JVM tool lockfiles."
     )
+    parser.add_argument(
+        "--all-nodejs", action="store_true", help="Regenerate all builtin NodeJS tool lockfiles."
+    )
     parser.add_argument(
         "--dry-run", action="store_true", help="Show Pants commands that would be run."
     )
@@ -287,6 +323,69 @@ def generate_jvm_tool_lockfiles(
         generate(tmp_buildroot, tools, jvm_args, dry_run, keep_sandboxes)
 
 
+def generate_nodejs_tool_lockfiles(
+    tools: Sequence[NodeJSTool], dry_run: bool, keep_sandboxes: KeepSandboxes
+) -> None:
+    cleanup_tmp = keep_sandboxes == KeepSandboxes.never
+
+    assert NodeJS.package_managers is not None
+    pm_versions: dict[str, str] = NodeJS.package_managers.kwargs["default"]
+    package_managers = [
+        PackageManager.npm(pm_versions["npm"]),
+        PackageManager.yarn(pm_versions["yarn"]),
+        PackageManager.pnpm(pm_versions["pnpm"]),
+    ]
+    pants_repo_root = get_buildroot()
+
+    for tool in tools:
+        pkg_name, pkg_version = _parse_package_name_and_version(tool.cls.default_version)
+
+        lockfile_resources = tool.cls.default_lockfile_resources
+        if not lockfile_resources:
+            logger.warning(f"Skipping {tool.name}: no default_lockfile_resources configured.")
+            continue
+
+        for pm in package_managers:
+            if pm.name not in lockfile_resources:
+                continue
+
+            resource_pkg, resource_filename = lockfile_resources[pm.name]
+            dest = os.path.join(
+                pants_repo_root, _lockfile_dest_for_resource(resource_pkg, resource_filename)
+            )
+            cmd = [
+                "npx",
+                "--yes",
+                f"{pm.name}@{pm.version}",
+                *pm.generate_lockfile_args,
+                "--ignore-scripts",
+            ]
+
+            if dry_run:
+                logger.info(f"Would run: {' '.join(cmd)} -> {dest}")
+                continue
+
+            with temporary_dir(cleanup=cleanup_tmp) as tmp_dir:
+                if not cleanup_tmp:
+                    logger.info(f"Preserving temp dir for {tool.name}/{pm.name}: {tmp_dir}")
+
+                with open(os.path.join(tmp_dir, "package.json"), "wb") as f:
+                    f.write(_tool_package_json_bytes(tool.name, pkg_name, pkg_version))
+
+                logger.info(f"Generating {pm.name} lockfile for {tool.name}...")
+                try:
+                    subprocess.run(cmd, cwd=tmp_dir, check=True, capture_output=True)
+                except subprocess.CalledProcessError as e:
+                    logger.error(
+                        f"Failed to generate {pm.name} lockfile for {tool.name}:\n"
+                        f"  stdout: {e.stdout.decode()}\n"
+                        f"  stderr: {e.stderr.decode()}"
+                    )
+                    raise
+                shutil.copy(os.path.join(tmp_dir, pm.lockfile_name), dest)
+                logger.debug(f"Copied {pm.lockfile_name} -> {dest}")
+
+
 def generate(
     buildroot: str,
     tools: Sequence[Tool],
@@ -360,27 +459,34 @@ def main() -> None:
 
     python_tools = []
     jvm_tools = []
+    nodejs_tools = []
     for name in args.tool:
         tool = name_to_tool[name]
         if isinstance(tool, PythonTool):
             python_tools.append(tool)
         elif isinstance(tool, JvmTool):
             jvm_tools.append(tool)
+        elif isinstance(tool, NodeJSTool):
+            nodejs_tools.append(tool)
         else:
             raise ValueError(f"Tool {name} has unknown type.")
     if args.all_python:
         python_tools.extend(all_python_tools)
     if args.all_jvm:
         jvm_tools.extend(all_jvm_tools)
-    if not python_tools and not jvm_tools:
+    if args.all_nodejs:
+        nodejs_tools.extend(all_nodejs_tools)
+    if not python_tools and not jvm_tools and not nodejs_tools:
         raise ValueError(
             "Must specify at least one tool, either via positional args, "
-            "or via the --all-python/--all-jvm flags."
+            "or via the --all-python/--all-jvm/--all-nodejs flags."
         )
     if python_tools:
         generate_python_tool_lockfiles(python_tools, args.dry_run, args.keep_sandboxes)
     if jvm_tools:
         generate_jvm_tool_lockfiles(jvm_tools, args.dry_run, args.keep_sandboxes)
+    if nodejs_tools:
+        generate_nodejs_tool_lockfiles(nodejs_tools, args.dry_run, args.keep_sandboxes)
 
 
 if __name__ == "__main__":

docs/notes/2.32.x.md

@@ -144,6 +144,8 @@ When generating lockfiles, the new `python.resolves_to_uploaded_prior_to` option
 
 #### Javascript
 
+Added support for generating lockfiles for NodeJS-based tool subsystems (`prettier`, `pyright`, `@redocly/cli`, `@stoplight/spectral-cli`, `openapi-format`) via `pants generate-lockfiles --resolve=<tool>`. Each tool now ships with a bundled lockfile pinning its transitive dependencies.
+
 #### TypeScript
 
 #### Go

src/python/pants/backend/javascript/goals/lockfile.py

@@ -13,8 +13,19 @@
     NodeJsProjectEnvironmentProcess,
 )
 from pants.backend.javascript.package_json import PackageJsonTarget
+from pants.backend.javascript.package_manager import PackageManager
 from pants.backend.javascript.resolve import NodeJSProjectResolves
-from pants.backend.javascript.subsystems.nodejs import UserChosenNodeJSResolveAliases
+from pants.backend.javascript.subsystems.nodejs import (
+    NodeJS,
+    NodeJSToolProcess,
+    UserChosenNodeJSResolveAliases,
+)
+from pants.backend.javascript.subsystems.nodejs_tool import (
+    NodeJSToolBase,
+    _lockfile_dest_for_resource,
+    _parse_package_name_and_version,
+    _tool_package_json_bytes,
+)
 from pants.core.goals.generate_lockfiles import (
     GenerateLockfile,
     GenerateLockfileResult,
@@ -23,13 +34,16 @@
     RequestedUserResolveNames,
     UserGenerateLockfiles,
 )
+from pants.core.goals.resolves import ExportableTool
 from pants.core.goals.tailor import TailorGoal
+from pants.engine.fs import CreateDigest, FileContent
 from pants.engine.internals.native_engine import AddPrefix
-from pants.engine.intrinsics import add_prefix
+from pants.engine.intrinsics import add_prefix, create_digest, get_digest_contents
 from pants.engine.process import fallible_to_exec_result_or_raise
 from pants.engine.rules import Rule, collect_rules, implicitly, rule
-from pants.engine.unions import UnionRule
+from pants.engine.unions import UnionMembership, UnionRule
 from pants.util.docutil import bin_name
+from pants.util.frozendict import FrozenDict
 from pants.util.ordered_set import FrozenOrderedSet
 from pants.util.strutil import pluralize, softwrap
 
@@ -125,11 +139,145 @@ async def generate_lockfile_from_package_jsons(
     return GenerateLockfileResult(output_digest, request.resolve_name, request.lockfile_dest)
 
 
+@dataclass(frozen=True)
+class GenerateNodeJSToolLockfile(GenerateLockfile):
+    package: str
+    package_manager: PackageManager
+
+
+class KnownNodeJSToolResolveNamesRequest(KnownUserResolveNamesRequest):
+    pass
+
+
+class RequestedNodeJSToolResolveNames(RequestedUserResolveNames):
+    pass
+
+
+@rule
+async def determine_nodejs_tool_resolves(
+    _: KnownNodeJSToolResolveNamesRequest,
+    union_membership: UnionMembership,
+) -> KnownUserResolveNames:
+    tool_classes = ExportableTool.filter_for_subclasses(union_membership, NodeJSToolBase)
+    names = tuple(
+        sorted(
+            tool_cls.options_scope
+            for tool_cls in tool_classes.values()
+            if tool_cls.default_lockfile_resources
+        )
+    )
+    return KnownUserResolveNames(
+        names=names,
+        option_name="[nodejs].tool_lockfiles",
+        requested_resolve_names_cls=RequestedNodeJSToolResolveNames,
+    )
+
+
+@rule
+async def setup_nodejs_tool_lockfile_requests(
+    requested: RequestedNodeJSToolResolveNames,
+    nodejs: NodeJS,
+    union_membership: UnionMembership,
+) -> UserGenerateLockfiles:
+    pkg_manager_and_version = nodejs.default_package_manager
+    if pkg_manager_and_version is None:
+        raise ValueError(
+            softwrap(
+                f"""
+                A package manager version must be configured in
+                [{nodejs.options_scope}].package_managers to generate NodeJS tool lockfiles.
+                """
+            )
+        )
+    pkg_manager = PackageManager.from_string(pkg_manager_and_version)
+
+    # Discover tool classes dynamically via the ExportableTool union membership.
+    tool_classes_by_scope: dict[str, type[NodeJSToolBase]] = {
+        scope: tool_cls
+        for scope, tool_cls in ExportableTool.filter_for_subclasses(
+            union_membership, NodeJSToolBase
+        ).items()
+        if tool_cls.default_lockfile_resources
+    }
+
+    requests = []
+    for name in requested:
+        tool_cls = tool_classes_by_scope.get(name)
+        if tool_cls is None:
+            continue
+        lockfile_resources = tool_cls.default_lockfile_resources
+        if lockfile_resources and pkg_manager.name in lockfile_resources:
+            resource_pkg, filename = lockfile_resources[pkg_manager.name]
+            lockfile_dest = _lockfile_dest_for_resource(resource_pkg, filename)
+        else:
+            lockfile_dest = f"{name}.{pkg_manager.lockfile_name}"
+
+        requests.append(
+            GenerateNodeJSToolLockfile(
+                resolve_name=name,
+                lockfile_dest=lockfile_dest,
+                diff=False,
+                package=tool_cls.default_version,
+                package_manager=pkg_manager,
+            )
+        )
+    return UserGenerateLockfiles(requests)
+
+
+@rule
+async def generate_nodejs_tool_lockfile(
+    request: GenerateNodeJSToolLockfile,
+) -> GenerateLockfileResult:
+    package_name, package_version = _parse_package_name_and_version(request.package)
+
+    input_digest = await create_digest(
+        CreateDigest(
+            [
+                FileContent(
+                    "package.json",
+                    _tool_package_json_bytes(request.resolve_name, package_name, package_version),
+                )
+            ]
+        ),
+        **implicitly(),
+    )
+
+    result = await fallible_to_exec_result_or_raise(
+        **implicitly(
+            NodeJSToolProcess(
+                request.package_manager.name,
+                request.package_manager.version,
+                args=request.package_manager.generate_lockfile_args,
+                description=(
+                    f"Generate {request.package_manager.lockfile_name} "
+                    f"for '{request.resolve_name}'."
+                ),
+                input_digest=input_digest,
+                output_files=(request.package_manager.lockfile_name,),
+                extra_env=FrozenDict(request.package_manager.extra_env),
+            )
+        )
+    )
+
+    # The sandbox output is `<lockfile_name>` at the digest root; relocate to `lockfile_dest`
+    # so `workspace.write_digest` lands it at the expected path (and filename).
+    digest_contents = await get_digest_contents(result.output_digest)
+    [file_content] = digest_contents
+    output_digest = await create_digest(
+        CreateDigest([FileContent(request.lockfile_dest, file_content.content)]),
+        **implicitly(),
+    )
+    return GenerateLockfileResult(output_digest, request.resolve_name, request.lockfile_dest)
+
+
 def rules() -> Iterable[Rule | UnionRule]:
     return (
         *collect_rules(),
         *nodejs_project_environment.rules(),
         UnionRule(GenerateLockfile, GeneratePackageLockJsonFile),
         UnionRule(KnownUserResolveNamesRequest, KnownPackageJsonUserResolveNamesRequest),
         UnionRule(RequestedUserResolveNames, RequestedPackageJsonUserResolveNames),
+        UnionRule(GenerateLockfile, GenerateNodeJSToolLockfile),
+        UnionRule(KnownUserResolveNamesRequest, KnownNodeJSToolResolveNamesRequest),
+        UnionRule(RequestedUserResolveNames, RequestedNodeJSToolResolveNames),
     )