Package validation: accounting for version pinning

scripts/generate-package-report.js

@@ -1,14 +1,19 @@
 const fs = require('fs');
 const path = require('path');
 const { execSync } = require('child_process');
+const { pathToFileURL } = require('url');
+
+// ESM loader that rewrites version-pinned specifiers ("got@13.0.0" -> "got")
+// so the import test can resolve them from node_modules.
+const VERSION_STRIP_LOADER = pathToFileURL(
+  path.join(__dirname, 'version-strip-loader.mjs')
+).href;
 
 // Apps with known validation issues to suppress so they don't trigger failure
 // reports. Keep this list small and document why each entry is here.
 // Value can be `true` to ignore all checks for the app, or an array of check
 // names (e.g. ['import', 'packageDependencies']) to ignore only specific ones.
-const IGNORED_VALIDATIONS = {
-  e2b: true, // Uses Pipedream's version-pinned import ("@e2b/code-interpreter@1.0.3"), which the dependency/import checks misread as a missing dep / invalid URL.
-};
+const IGNORED_VALIDATIONS = {};
 
 // Native Node.js modules that don't need to be in package.json
 const NATIVE_MODULES = new Set([
@@ -20,6 +25,43 @@ const NATIVE_MODULES = new Set([
   'zlib', 'async_hooks', 'inspector', 'trace_events', 'http2'
 ]);
 
+// Pipedream's runtime always installs the latest version of an imported
+// package and ignores package.json, so components that must stay on an older
+// version pin it in the import specifier itself (e.g.
+// `import { WebClient } from "@slack/web-api@8.0.0"`). Parse those specifiers
+// into their package name, pinned version and subpath.
+function parseImportSpecifier(specifier) {
+  const scoped = specifier.startsWith('@');
+  // For scoped packages the leading "@" is part of the name, not a version.
+  const body = scoped ? specifier.slice(1) : specifier;
+  const [nameAndSubpath, version] = splitOnVersion(body);
+  const parts = nameAndSubpath.split('/');
+  const nameParts = scoped ? parts.slice(0, 2) : parts.slice(0, 1);
+  const packageName = (scoped ? '@' : '') + nameParts.join('/');
+
+  return {
+    packageName,
+    version: version || null,
+    subpath: parts.slice(nameParts.length).join('/') || null
+  };
+}
+
+// Splits "web-api@8.0.0" into ["web-api", "8.0.0"], leaving unversioned
+// specifiers (and subpaths that merely contain an "@") untouched.
+function splitOnVersion(body) {
+  const match = body.match(/^([^@]+)@(\d[^/]*)$/);
+  return match
+    ? [match[1], match[2]]
+    : [body, null];
+}
+
+// Treat "^8.0.0", "~8.0.0" and "8.0.0" as declaring the same version so a
+// version-pinned import only fails when it genuinely disagrees with
+// package.json.
+function normalizeDeclaredVersion(range) {
+  return range.replace(/^[\^~=v\s]+/, '');
+}
+
 // Parse command line arguments
 const args = process.argv.slice(2);
 const isReportOnly = args.includes('--report-only');
@@ -442,22 +484,15 @@ function validatePackageDependencies(packageJson, app) {
   let match;
 
   while ((match = packageImportRegex.exec(content)) !== null) {
-    const packageName = match[1];
-    // Extract the base package name (handle scoped packages and subpaths)
-    let basePackageName;
-    if (packageName.startsWith('@')) {
-      // Scoped package like @pipedream/platform or @aws-sdk/client-s3
-      const parts = packageName.split('/');
-      basePackageName = `${parts[0]}/${parts[1]}`;
-    } else {
-      // Regular package like axios or lodash (could have subpath like lodash/get)
-      basePackageName = packageName.split('/')[0];
-    }
-
+    const originalImport = match[1];
+    // Handles scoped packages, subpaths and version-pinned specifiers
+    const { packageName, version } = parseImportSpecifier(originalImport);
+
     packageImports.push({
-      packageName: basePackageName,
+      packageName,
+      pinnedVersion: version,
       fullMatch: match[0],
-      originalImport: packageName
+      originalImport
     });
   }
 
@@ -471,6 +506,8 @@ function validatePackageDependencies(packageJson, app) {
   const devDependencies = packageJson.devDependencies || {};
   const allDependencies = { ...dependencies, ...devDependencies };
 
+  const versionMismatches = [];
+
   // Remove duplicates
   const uniquePackages = [...new Set(packageImports.map(imp => imp.packageName))];
 
@@ -483,13 +520,30 @@ function validatePackageDependencies(packageJson, app) {
       return;
     }
 
-    if (!allDependencies[packageName]) {
+    const declaredVersion = allDependencies[packageName];
+    if (!declaredVersion) {
       const exampleImport = packageImports.find(imp => imp.packageName === packageName);
       missingDependencies.push({
         packageName,
         importStatement: exampleImport.fullMatch
       });
+      return;
     }
+
+    // A version-pinned import must agree with package.json, otherwise the
+    // published package installs a different version than the one the
+    // component asked the runtime for.
+    packageImports
+      .filter(imp => imp.packageName === packageName && imp.pinnedVersion)
+      .forEach((imp) => {
+        if (normalizeDeclaredVersion(declaredVersion) !== imp.pinnedVersion) {
+          versionMismatches.push({
+            packageName,
+            pinnedVersion: imp.pinnedVersion,
+            declaredVersion
+          });
+        }
+      });
   });
 
   if (missingDependencies.length > 0) {
@@ -498,6 +552,13 @@ function validatePackageDependencies(packageJson, app) {
       .join(', ');
     throw new Error(`Package imports require corresponding dependencies. Missing dependencies: ${missingList}`);
   }
+
+  if (versionMismatches.length > 0) {
+    const mismatchList = versionMismatches
+      .map(dep => `${dep.packageName} imported as ${dep.pinnedVersion} but declared as ${dep.declaredVersion}`)
+      .join(', ');
+    throw new Error(`Version-pinned imports must match package.json: ${mismatchList}`);
+  }
 }
 
 function validateImport(packageName, app, packageJson) {
@@ -509,7 +570,7 @@ function validateImport(packageName, app, packageJson) {
 
   // Syntax check
   try {
-    execSync(`node --check ${mainFile}`, { 
+    execSync(`node --check "${mainFile}"`, {
       stdio: 'pipe',
       timeout: 5000 
     });
@@ -519,9 +580,10 @@ function validateImport(packageName, app, packageJson) {
 
   // Import test using file path
   const testFile = path.join('components', app, '__import_test__.mjs');
+  const mainFileUrl = pathToFileURL(mainFile).href;
   const testContent = `
 try {
-  const pkg = await import("file://${mainFile}");
+  const pkg = await import("${mainFileUrl}");
 
   if (!pkg.default) {
     throw new Error("No default export found");
@@ -542,7 +604,9 @@ try {
 
   try {
     fs.writeFileSync(testFile, testContent);
-    execSync(`node ${testFile}`, { 
+    // The loader strips Pipedream's version-pinned import syntax
+    // ("@slack/web-api@8.0.0") so Node can resolve the installed package.
+    execSync(`node --experimental-loader "${VERSION_STRIP_LOADER}" "${testFile}"`, {
       stdio: 'pipe',
       cwd: process.cwd(),
       timeout: 10000

scripts/version-strip-loader.mjs

@@ -1,9 +1,16 @@
 export async function resolve(specifier, context, nextResolve) {
-  // Strip version from imports like "got@13.0.0" -> "got"
-  const versionedImportMatch = specifier.match(/^(.+)@[\d\.\-\w]+$/);
-  if (versionedImportMatch) {
-    specifier = versionedImportMatch[1];
+  // Strip Pipedream's version-pinned import syntax: "got@13.0.0" -> "got".
+  // The leading "@" of a scoped package is not a version separator, so only
+  // look for a "@<digit>..." suffix after it.
+  const versionedImportMatch = specifier.match(/^(@?[^@]+)@\d[^/]*$/);
+  if (!versionedImportMatch) {
+    return nextResolve(specifier, context);
   }
-
-  return nextResolve(specifier, context);
-}
+
+  try {
+    return await nextResolve(versionedImportMatch[1], context);
+  } catch (error) {
+    // The "@" wasn't a version pin after all - resolve the specifier as-is.
+    return nextResolve(specifier, context);
+  }
+}