Merge pull request #296 from Azure/copilot/bugfix-selective-api-operation-filter

Honor operation extraction filters in OpenAPI specifications

Author: azaslonov

docs/guides/filtering-resources.md

@@ -249,6 +249,13 @@ apis:
 - If a sub-resource key is an **empty array** (`[]`), all sub-resources of that type are excluded
 - If a sub-resource key lists **names**, only those sub-resources are included
 
+During extraction, the `operations` filter also applies to the API's OpenAPI specification
+(`specification.yaml` or `specification.json`, including Swagger 2.0). Excluded operations
+are removed from the specification, and paths with no remaining operations are removed.
+Shared definitions and metadata on retained paths are preserved. Omitting `operations`
+leaves the specification unchanged; `operations: []` removes all operations from it.
+GraphQL, WSDL, and WADL specifications are not filtered.
+
 ### Workspace sub-resource filters
 
 The configuration format supports specifying which workspace-scoped resources to extract:

src/services/api-extractor.ts

@@ -12,7 +12,8 @@ import { IArtifactStore } from '../clients/iartifact-store.js';
 import { ApimServiceContext, ResourceDescriptor } from '../models/types.js';
 import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js';
 import { FilterConfig } from '../models/config.js';
-import { shouldIncludeResource } from './filter-service.js';
+import * as yaml from 'js-yaml';
+import { extractRootApiName, shouldIncludeResource } from './filter-service.js';
 import { extractResourceType, ExtractedResource } from './resource-extractor.js';
 import { redactAndWarnPolicySecrets } from './secret-redactor.js';
 import { logger } from '../lib/logger.js';
@@ -21,6 +22,22 @@ import { getNamePart } from '../lib/resource-path.js';
 import { normalizeWsdl } from '../lib/wsdl-normalizer.js';
 import { isWorkspaceScope, extractNameFromLink } from '../lib/workspace-link.js';
 
+/**
+ * `JSON.parse`'s reviver `context` argument and `JSON.rawJSON` are ES2024
+ * runtime additions (supported by the project's Node.js target) not yet
+ * reflected in the configured TS `lib`. Augment the global `JSON` interface
+ * rather than casting through `unknown` at each call site.
+ */
+declare global {
+  interface JSON {
+    parse(
+      text: string,
+      reviver: (this: unknown, key: string, value: unknown, context: { source?: string }) => unknown
+    ): unknown;
+    rawJSON(text: string): unknown;
+  }
+}
+
 /**
  * Result of API-specific extraction for a single API.
  */
@@ -97,7 +114,7 @@ export async function extractApiResources(
   // Extract API specification (uses already-extracted schemas to detect
   // synthetic GraphQL without a second list call).
   const specificationResult = await extractApiSpecification(
-    client, store, context, apiDescriptor, apiJson, outputDir, result.schemas
+    client, store, context, apiDescriptor, apiJson, outputDir, result.schemas, filter
   );
   result.specification = specificationResult.extracted;
   result.errorCount += specificationResult.errorCount;
@@ -274,7 +291,8 @@ async function extractApiSpecification(
   apiDescriptor: ResourceDescriptor,
   apiJson: Record<string, unknown>,
   outputDir: string,
-  extractedSchemas: ExtractedResource[]
+  extractedSchemas: ExtractedResource[],
+  filter?: FilterConfig
 ): Promise<{ extracted: boolean; errorCount: number }> {
   const properties = apiJson.properties as Record<string, unknown> | undefined;
   const apiType = properties?.type as string | undefined;
@@ -319,9 +337,12 @@ async function extractApiSpecification(
     // APIM's WSDL export can emit wsdl:part references qualified with the wrong
     // namespace prefix and multiple service ports, which its own importer then
     // rejects. Normalize so the extracted artifact round-trips through publish.
-    const content = spec.format === 'wsdl'
+    let content = spec.format === 'wsdl'
       ? normalizeWsdl(spec.content)
       : spec.content;
+    if (spec.format === 'yaml' || spec.format === 'json') {
+      content = filterOpenApiOperations(content, spec.format, apiDescriptor, filter);
+    }
 
     await store.writeContent(
       outputDir,
@@ -340,6 +361,190 @@ async function extractApiSpecification(
   }
 }
 
+/**
+ * Apply the same operation filter to the specification as to operation artifacts.
+ * Keep shared definitions and path metadata, but remove paths without selected methods.
+ *
+ * Parses and reserializes the document, taking care to avoid the precision/type
+ * loss that `JSON.parse`/js-yaml's defaults would otherwise introduce for scalars
+ * unrelated to the operations being removed (e.g. int64 examples round to the
+ * nearest double, bare YAML date-like strings are coerced to `Date`). This is not
+ * a full byte-for-byte preserving editor: YAML comments are not retained, and
+ * integers originally written in hex/octal/binary notation are re-emitted in
+ * decimal form, since the reserialization still rebuilds the whole document.
+ */
+function filterOpenApiOperations(
+  content: string,
+  format: 'yaml' | 'json',
+  apiDescriptor: ResourceDescriptor,
+  filter?: FilterConfig
+): string {
+  const apiName = getNamePart(apiDescriptor.nameParts, 0);
+  const rootName = extractRootApiName(apiName).toLowerCase();
+  const subFilterKey = Object.keys(filter?.apiSubFilters ?? {}).find(
+    key => key.toLowerCase() === rootName
+  );
+  if (!subFilterKey || filter?.apiSubFilters?.[subFilterKey].operations === undefined) {
+    return content;
+  }
+
+  const spec = parseOpenApiDocument(content, format);
+  const methods = new Set(['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace']);
+  let modified = false;
+
+  for (const field of ['paths', 'x-ms-paths']) {
+    const paths = spec[field];
+    if (!paths || typeof paths !== 'object' || Array.isArray(paths)) continue;
+
+    for (const [path, value] of Object.entries(paths)) {
+      if (!path.startsWith('/') || !value || typeof value !== 'object' || Array.isArray(value)) continue;
+      const pathItem = value as Record<string, unknown>;
+      let hasSelectedOperation = false;
+
+      for (const method of Object.keys(pathItem)) {
+        if (!methods.has(method)) continue;
+        const operation = pathItem[method] as Record<string, unknown> | null;
+        const operationId = operation?.operationId;
+        if (typeof operationId === 'string' && shouldIncludeResource({
+          ...apiDescriptor,
+          type: ResourceType.ApiOperation,
+          nameParts: [apiName, operationId],
+        }, filter)) {
+          hasSelectedOperation = true;
+        } else {
+          delete pathItem[method];
+          modified = true;
+        }
+      }
+
+      if (!hasSelectedOperation) {
+        delete (paths as Record<string, unknown>)[path];
+        modified = true;
+      }
+    }
+  }
+
+  if (!modified) return content;
+  return stringifyOpenApiDocument(spec, format);
+}
+
+/**
+ * Parse a JSON/YAML OpenAPI document without losing precision on unrelated
+ * scalars. `JSON.parse`/`js-yaml` normally round very large integers (e.g.
+ * int64 examples) to the nearest representable double and, for YAML, coerce
+ * bare date-like scalars to `Date` objects. Both are avoided here so that
+ * only the operations targeted by the filter are actually changed.
+ */
+function parseOpenApiDocument(content: string, format: 'yaml' | 'json'): Record<string, unknown> {
+  if (format === 'json') {
+    const reviver = (_key: string, value: unknown, context: { source?: string }): unknown => {
+      const source = context?.source;
+      if (typeof value === 'number' && source !== undefined) {
+        return JSON.rawJSON(source);
+      }
+      return value;
+    };
+    return JSON.parse(content, reviver) as Record<string, unknown>;
+  }
+  return yaml.load(content, { schema: PRECISE_YAML_SCHEMA }) as Record<string, unknown>;
+}
+
+/**
+ * Reserialize a document previously parsed by {@link parseOpenApiDocument},
+ * emitting the preserved raw/precise scalars back in their original form.
+ */
+function stringifyOpenApiDocument(spec: Record<string, unknown>, format: 'yaml' | 'json'): string {
+  return format === 'json'
+    ? JSON.stringify(spec, null, 2)
+    : yaml.dump(spec, { lineWidth: -1, schema: PRECISE_YAML_SCHEMA });
+}
+
+/**
+ * YAML `int` type that preserves full precision for integers outside the safe
+ * integer range (e.g. int64 examples) by representing them as `BigInt`
+ * instead of a lossy JS `number`. Built on the standard Core schema, which
+ * (unlike the Default schema) does not auto-convert bare date-like scalars to
+ * `Date` objects.
+ */
+const PRECISE_YAML_SCHEMA = yaml.CORE_SCHEMA.extend({
+  implicit: [
+    new yaml.Type('tag:yaml.org,2002:int', {
+      kind: 'scalar',
+      resolve: resolveYamlInteger,
+      construct: (data: string) => parsePreciseYamlInteger(data),
+      predicate: (data: unknown) => isPreciseYamlInteger(data),
+      // Always represented in decimal: the original literal style (e.g. hex,
+      // octal, binary) isn't retained on the plain number/BigInt values produced
+      // by `construct`, so a multi-style representer couldn't be selected from
+      // them. Non-decimal integers unrelated to the filtered operations are
+      // therefore re-emitted in decimal form; see the module-level docstring.
+      represent: (data: unknown) => castPreciseYamlValue(data).toString(10),
+    }),
+  ],
+});
+
+function castPreciseYamlValue(data: unknown): number | bigint {
+  return data as number | bigint;
+}
+
+/** Ported from js-yaml's built-in int type resolver (not exported publicly). */
+function resolveYamlInteger(data: string | null): boolean {
+  if (data === null || data.length === 0) return false;
+
+  let index = 0;
+  const max = data.length;
+  let ch = data[index];
+
+  if (ch === '-' || ch === '+') {
+    ch = data[++index];
+  }
+
+  if (ch === '0') {
+    if (index + 1 === max) return true;
+    ch = data[++index];
+
+    if (ch === 'b') return /^[01]+$/.test(data.slice(index + 1));
+    if (ch === 'x') return /^[0-9a-fA-F]+$/.test(data.slice(index + 1));
+    if (ch === 'o') return /^[0-7]+$/.test(data.slice(index + 1));
+  }
+
+  return /^[0-9]+$/.test(data.slice(index));
+}
+
+/**
+ * Parses a resolved YAML integer scalar, promoting to `BigInt` only when the
+ * value would otherwise lose precision as a JS `number`.
+ */
+function parsePreciseYamlInteger(data: string): number | bigint {
+  let value = data;
+  let sign = 1;
+  const ch0 = value[0];
+  if (ch0 === '-' || ch0 === '+') {
+    if (ch0 === '-') sign = -1;
+    value = value.slice(1);
+  }
+  if (value === '0') return 0;
+
+  let magnitude: number;
+  if (value[0] === '0' && value[1] === 'b') magnitude = parseInt(value.slice(2), 2);
+  else if (value[0] === '0' && value[1] === 'x') magnitude = parseInt(value.slice(2), 16);
+  else if (value[0] === '0' && value[1] === 'o') magnitude = parseInt(value.slice(2), 8);
+  else magnitude = parseInt(value, 10);
+
+  const result = sign * magnitude;
+  if (Number.isSafeInteger(result)) return result;
+
+  const preciseMagnitude = BigInt(value);
+  return sign === -1 ? -preciseMagnitude : preciseMagnitude;
+}
+
+function isPreciseYamlInteger(data: unknown): boolean {
+  if (typeof data === 'bigint') return true;
+  return Object.prototype.toString.call(data) === '[object Number]' &&
+    (data as number) % 1 === 0 &&
+    !Object.is(data, -0);
+}
+
 /**
  * Returns true if any already-extracted ApiSchema resource has a contentType
  * indicating a GraphQL schema. Used to distinguish synthetic GraphQL APIs