diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 1001a34e5..1ab024e77 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -3,15 +3,34 @@ name: Publish package on NPM on: push: tags: - - v* # Any version tag + - 'v[0-9]+.[0-9]+.[0-9]+' permissions: - id-token: write # To publish on NPM with provenance and to federate tokens + id-token: write # For OIDC publishing with provenance and to federate tokens contents: write # Required for the draft release jobs: + pre-release-checks: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Install node + uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + with: + node-version: 22.19.0 + - name: Install project dependencies + run: yarn install --immutable --mode=skip-build + - name: Check NPM packages + run: yarn check-npm-packages + env: + # Used to post comments on the PR + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + create-draft-release: runs-on: ubuntu-latest + needs: pre-release-checks outputs: release-id: ${{ steps.draft-release.outputs.result }} steps: @@ -258,11 +277,11 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 with: - node-version: 24.11.1 # Version supporting OIDC - registry-url: "https://registry.npmjs.org" + node-version: '24' # Needed for OIDC publishing + registry-url: 'https://registry.npmjs.org' - run: yarn install --immutable - run: yarn build - - run: yarn publish:all + - run: yarn publish:all --provenance bump-ci-integrations: name: Bump datadog-ci in integration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f882c99b..9c3b3d314 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,6 +3,26 @@ Pull requests for bug fixes are welcome, but before submitting new features or changes to current functionality, [open an issue](https://github.com/DataDog/datadog-ci/issues/new) and discuss your ideas or propose the changes you wish to make. After a resolution is reached, a PR can be submitted for review. +### Listing NPM packages + +This repository is a monorepo containing multiple packages published to NPM. You can list all packages with the following command: + +```sh +npm search 'maintainer:datadog keywords:datadog-ci' +``` + +To only list the plugins: + +```sh +npm search 'maintainer:datadog keywords:datadog-ci,plugin' +``` + +You can also use the following datadog-ci command to get more information: + +```sh +yarn launch plugin list --all +``` + ### Running command in development environment When developing the tool, it is possible to run commands using `yarn launch`. It relies on `tsx`, so it does not require building the project for every new change. diff --git a/bin/check-npm-packages.sh b/bin/check-npm-packages.sh new file mode 100755 index 000000000..61ab49c20 --- /dev/null +++ b/bin/check-npm-packages.sh @@ -0,0 +1,183 @@ +#!/bin/bash + +set -euo pipefail + +# This script checks if all local packages are published to NPM. +# It can also first-time publish missing packages when run with --fix. +# +# Usage: +# ./bin/check-npm-packages.sh # Check mode (default) - exits 1 if packages are missing +# ./bin/check-npm-packages.sh --fix # Fix mode - publishes missing packages +# ./bin/check-npm-packages.sh --fix --dry-run # Fix mode with dry-run - simulates publishing + +MODE="check" +DRY_RUN=false + +while [[ $# -gt 0 ]]; do + case $1 in + --fix) + MODE="fix" + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -*) + echo "Unknown option: $1" + echo "Usage: $0 [--fix] [--dry-run]" + exit 1 + ;; + *) + echo "Unknown argument: $1" + echo "Usage: $0 [--fix] [--dry-run]" + exit 1 + ;; + esac +done + +if [ "$DRY_RUN" = true ] && [ "$MODE" != "fix" ]; then + echo "Error: --dry-run can only be used with --fix" + exit 1 +fi + +# Colors +BOLD='\033[1m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' + +echo -e "${BOLD}Checking for unpublished packages...${NC}" +echo + +# Get local and remote packages +local_packages=$(yarn workspaces list --json --no-private | jq -r '.name' | sort) +remote_packages=$(npm search 'maintainer:datadog keywords:datadog-ci' --json | jq -r '.[].name' | sort) + +# Find packages that exist locally but not on NPM +missing_packages=() +while IFS= read -r pkg; do + if [ -n "$pkg" ] && ! echo "$remote_packages" | grep -q "^${pkg}$"; then + missing_packages+=("$pkg") + fi +done <<< "$local_packages" + +# Exit early if everything is good +if [ ${#missing_packages[@]} -eq 0 ]; then + echo -e "${GREEN}All local packages exist on NPM ✅${NC}" + exit 0 +fi + +# Otherwise, report missing packages +echo -e "${RED}The following packages are not published to NPM yet:${NC}" +for pkg in "${missing_packages[@]}"; do + echo " - $pkg" +done + +# In CI environment, post a comment on the PR +if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ] && [ -n "${GITHUB_SHA:-}" ]; then + # Get the PR number and author associated with this commit + PR_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls") + + PR_NUMBER=$(echo "$PR_RESPONSE" | jq -r '.[0].number // empty') + PR_AUTHOR=$(echo "$PR_RESPONSE" | jq -r '.[0].user.login // empty') + + DIFF_OUTPUT=$(diff -u --label "Published packages (Actual)" --label "Local packages (Expected)" \ + <(echo "$remote_packages") <(echo "$local_packages")) || true + + COMMENT_BODY="### Some packages were not first-time published to NPM yet ❌ + +\`\`\`diff +$DIFF_OUTPUT +\`\`\` + +Hi @$PR_AUTHOR, please **ask an admin** to follow the instructions at https://datadoghq.atlassian.net/wiki/x/QYDRaQE" + + if [ -n "$PR_NUMBER" ]; then + # Post comment on the PR + curl -s -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + -d "$(jq -n --arg body "$COMMENT_BODY" '{body: $body}')" > /dev/null + + echo -e "${BLUE}Posted comment on PR #$PR_NUMBER (author: @$PR_AUTHOR)${NC}" + else + # Fallback when PR is not found + echo -e "${RED}No PR found for commit $GITHUB_SHA${NC}" + echo -e "${BLUE}This would be the comment body:${NC}" + echo "$COMMENT_BODY" + fi +fi + +# Do not continue if we are in check mode +if [ "$MODE" = "check" ]; then + echo + echo -e "${BOLD}Run with --fix to publish these packages${NC}" + echo -e "See instructions at ${BLUE}https://datadoghq.atlassian.net/wiki/x/QYDRaQE${NC}" + exit 1 +fi + +# Fix mode - publish missing packages +echo +echo -e "${BOLD}Publishing missing packages to NPM...${NC}" +echo +echo -e "${BOLD}Please read the instructions${NC} at ${BLUE}https://datadoghq.atlassian.net/wiki/x/QYDRaQE${NC} before proceeding." +echo + +if [ "$DRY_RUN" = true ]; then + echo -e "${BOLD}[DRY-RUN]${NC} None of the packages will actually be published." + echo +fi + +read -rsp "Enter your NPM auth token: " INIT_NPM_AUTH_TOKEN +echo +if [ -z "$INIT_NPM_AUTH_TOKEN" ]; then + echo "Error: NPM auth token cannot be empty" + exit 1 +fi + +# Export this for subsequent yarn commands in the script +export INIT_NPM_AUTH_TOKEN + +# Do not hardcode the token in .yarnrc.yml, it will be read from the environment variable +yarn config set npmAuthToken '${INIT_NPM_AUTH_TOKEN}' +echo + +for pkg in "${missing_packages[@]}"; do + echo -e "${BLUE}Publishing ${BOLD}$pkg${NC}${BLUE}...${NC}" + + # Get the package directory + pkg_dir=$(yarn workspaces list --json | jq -r "select(.name == \"$pkg\") | .location") + pkg_json="$pkg_dir/package.json" + + # Save original version + original_version=$(jq -r .version "$pkg_json") + + # Set version to 0.0.1 for first-time publish + jq '.version = "0.0.1"' "$pkg_json" | sponge "$pkg_json" + + if [ "$DRY_RUN" = true ]; then + echo " [DRY-RUN] Would publish $pkg@0.0.1" + yarn workspace "$pkg" npm publish --dry-run 2>&1 | sed 's/^/ /' + else + yarn workspace "$pkg" npm publish 2>&1 | sed 's/^/ /' + echo -e " ${GREEN}Successfully published $pkg@0.0.1${NC}" + fi + + # Restore original version + jq --arg version "$original_version" '.version = $version' "$pkg_json" | sponge "$pkg_json" + echo +done + +echo -e "${BOLD}Cleaning up...${NC}" +yarn config unset npmAuthToken + +echo +if [ "$DRY_RUN" = true ]; then + echo -e "${GREEN}[DRY-RUN] Would have published ${#missing_packages[@]} package(s)${NC}" +else + echo -e "${GREEN}Successfully published ${#missing_packages[@]} package(s)${NC}" +fi diff --git a/bin/create-plugin.sh b/bin/create-plugin.sh new file mode 100755 index 000000000..ca2fbf659 --- /dev/null +++ b/bin/create-plugin.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -euo pipefail + +SCOPE="" + +while [[ $# -gt 0 ]]; do + case $1 in + -*) + echo "Unknown option: $1" + echo "Usage: $0 " + exit 1 + ;; + *) + SCOPE="$1" + shift + ;; + esac +done + +if [ -z "$SCOPE" ]; then + echo "Usage: $0 " + exit 1 +fi +PLUGIN_PKG="@datadog/datadog-ci-plugin-$SCOPE" +PLUGIN_DIR="packages/plugin-$SCOPE" + +if [ -d "$PLUGIN_DIR" ]; then + echo "Plugin directory $PLUGIN_DIR already exists!" + echo "This script should only be run once per scope, before migrate.sh" + exit 1 +fi + +BOLD='\033[1m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "This script will initialize an empty package for ${BLUE}${BOLD}$PLUGIN_PKG${NC}" +echo + +echo -e "${BOLD}1. Creating plugin directory structure${NC}" +mkdir -p "$PLUGIN_DIR" +env cp LICENSE "$PLUGIN_DIR" +echo "Empty package" > "$PLUGIN_DIR/README.md" +cat > "$PLUGIN_DIR/package.json" <&1 | sed 's/^/ /' + +echo +echo -e "${GREEN}Package created successfully${NC}" + +echo +echo -e "${BLUE}If needed, you can now run: ${BOLD}bin/migrate.sh $SCOPE${NC}" diff --git a/bin/lint-packages.ts b/bin/lint-packages.ts index d253afe74..513804d6d 100644 --- a/bin/lint-packages.ts +++ b/bin/lint-packages.ts @@ -102,8 +102,8 @@ const findCommands = (folder: string, scope: string): string[] => { } catch (e) { if (e instanceof Error && e.message.includes('no such file or directory')) { const alternateFile = path.join('packages/base/src/commands', scope, 'cli.ts') - console.log(chalk.yellow(`Could not find commands in ${chalk.bold(folder)}, this means it's being migrated...`)) - console.log(chalk.yellow(`Reading imports in ${chalk.bold(alternateFile)} instead.\n`)) + console.log(chalk.yellow(`Could not find commands in ${chalk.bold(folder)}. A migration may be in progress...`)) + console.log(chalk.yellow(`Trying to read imports in ${chalk.bold(alternateFile)} instead.\n`)) const content = fs.readFileSync(alternateFile, 'utf8') @@ -218,10 +218,19 @@ const pluginPackages = fs .readdirSync('packages') .filter((dir) => dir.startsWith('plugin-')) .map((dir) => { + let loadedPackage: Package try { - const {folder, packageJson} = loadPackage(dir) + loadedPackage = loadPackage(dir) + } catch { + console.log(chalk.yellow(`Could not load ${chalk.bold(dir)} package. Skipping it...\n`)) - const scope = dir.replace('plugin-', '') + return undefined + } + + const {folder, packageJson} = loadedPackage + const scope = dir.replace('plugin-', '') + + try { const commands = findCommands(folder, scope) return { @@ -231,9 +240,11 @@ const pluginPackages = fs commands, } } catch { - console.log(chalk.yellow(`Could not load ${chalk.bold(dir)} package. Skipping it...\n`)) - - return undefined + console.log(chalk.bold.red(`Invalid state for ${folder}.`)) + console.log( + `Did you recently run ${chalk.bold(`yarn plugin:create ${scope}`)}? Please either run ${chalk.bold(`bin/migrate.sh ${scope}`)} and finish the migration, or complete the structure of the plugin before merging your PR.` + ) + process.exit(1) } }) .filter((p): p is PluginPackage => p !== undefined) @@ -568,6 +579,7 @@ if (Object.keys(impactedGithubActions).length > 0) { // #endregion if (fix) { + // Both commands always exit with 0, even when they make changes exec('yarn syncpack fix') exec('yarn syncpack format') } else { @@ -582,7 +594,12 @@ if (fix) { } if (fix) { - exec('yarn knip --fix') + try { + // This command exits with 1 when it makes changes + exec('yarn knip --fix') + } catch { + // ignore error + } } else { try { exec('yarn knip') diff --git a/bin/migrate.sh b/bin/migrate.sh index 8f1964da2..4175ebd4e 100755 --- a/bin/migrate.sh +++ b/bin/migrate.sh @@ -25,6 +25,28 @@ SRC_DIR="packages/datadog-ci/src/commands/$SCOPE" DST_DIR="$PLUGIN_DIR/src" BASE_DIR="packages/base/src/commands/$SCOPE" +BOLD='\033[1m' +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Check that init-package.sh was run first +if [ ! -d "$PLUGIN_DIR" ]; then + echo -e "${RED}Plugin directory ${BOLD}$PLUGIN_DIR${NC} does not exist!${NC}" + echo + echo -e "${BLUE}Please run ${BOLD}yarn plugin:create $SCOPE${NC} to initialize the package." + exit 1 +fi + +# Check that this script wasn't already run (tsconfig.json is created by migrate.sh) +if [ -f "$PLUGIN_DIR/tsconfig.json" ]; then + echo -e "${RED}Plugin directory ${BOLD}$PLUGIN_DIR${NC}${RED} already has a tsconfig.json file!${NC}" + echo + echo -e "${BLUE}This indicates the package was already migrated.${NC}" + exit 1 +fi + echo 1. Move the folder if [ ! -d "$SRC_DIR" ]; then echo "Source directory $SRC_DIR does not exist!" @@ -32,80 +54,17 @@ if [ ! -d "$SRC_DIR" ]; then fi if [ -d "$DST_DIR" ]; then echo "Destination directory $DST_DIR already exists!" - echo "You can run \`rm -rf packages/plugin-$SCOPE && rm -rf $BASE_DIR\` to clean up a previous run of this script." + echo "You can run \`rm -rf packages/plugin-$SCOPE/src && rm -rf $BASE_DIR\` to clean up a previous run of this script." exit 1 fi -mkdir -p "$PLUGIN_DIR" env mv "$SRC_DIR" "$DST_DIR" env mv "$DST_DIR/README.md" "$PLUGIN_DIR" -env cp LICENSE "$PLUGIN_DIR" mkdir -p "$BASE_DIR" mv "$DST_DIR/cli.ts" "$BASE_DIR/cli.ts" echo "Moved $SRC_DIR to $DST_DIR" -echo 2. Create package.json -cat > "$PLUGIN_DIR/package.json" < "$PLUGIN_DIR/tsconfig.json" <