Skip to content

[CI] Support package discovery in nested directories - WIP #18495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mrodm wants to merge 30 commits into
base: main
Choose a base branch
from
Draft

[CI] Support package discovery in nested directories - WIP #18495

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
acdab09
Adapt codeowners check
jsoriano Jan 14, 2026
265c135
[citools] Add ListPackages and extend packageManifest struct
mrodm Apr 16, 2026
58f0fe5
[citools] Expose ReadPackageManifest and consolidate manifest logic
mrodm Apr 16, 2026
d9100b9
[codeowners] Refactor validatePackages to use citools.ListPackages
mrodm Apr 16, 2026
91f9db5
[codeowners] Add tests for validatePackages with nested packages
mrodm Apr 16, 2026
a4af05b
[magefile] Add ListPackages mage target
mrodm Apr 16, 2026
741448f
Test with a subset of packages
mrodm Apr 17, 2026
218f948
[buildkite] Fix package path/name misusage after ListPackages refactor
mrodm Apr 17, 2026
3d92f36
[citools] Export ManifestFileName and remove duplicate constant
mrodm Apr 17, 2026
bb2e60f
[buildkite] Fix leftover issues in test_one_package.sh
mrodm Apr 17, 2026
e2454f2
Ensure that package_name_manifest is executed in the package path
mrodm Apr 17, 2026
99bfc48
[buildkite] Add should_test_package helper to common.sh
mrodm Apr 17, 2026
9f03436
[buildkite] Refactor build_packages.sh to support nested packages
mrodm Apr 17, 2026
71e903e
Update temprarily build_packages to test it - to be reverted
mrodm Apr 17, 2026
293da9f
[buildkite] Fix mage not found in build_packages.sh
mrodm Apr 17, 2026
5651580
[codeowners] Fix infinite loop in findOwnerForFile on absolute paths
mrodm Apr 17, 2026
b794a1b
Update loop to find findOwnerForFile
mrodm Apr 17, 2026
ac65fe1
[buildkite] Replace PACKAGE_FOLDER_NAME with get_package_path in back…
mrodm Apr 20, 2026
d7c0536
[buildkite] Fix CODEOWNERS removal for nested packages in removeOther…
mrodm Apr 20, 2026
1f1ebbf
[buildkite] Fix get_package_path early exit under set -e
mrodm Apr 20, 2026
6708fcd
Use as source branch the current commit - to be removed
mrodm Apr 20, 2026
2fea205
Use list_all_directories to get all directories/packages
mrodm Apr 20, 2026
ca7362b
Add more test packages
mrodm Apr 20, 2026
6835aef
Attempt to get the current branch as a source
mrodm Apr 20, 2026
237d836
Create temporal branch
mrodm Apr 20, 2026
1e50707
Restore location of removeOtherPackages call
mrodm Apr 20, 2026
06a49b5
Move some packages under nested directories to test CI scripts
mrodm Apr 20, 2026
dc25416
Allow to list more packages
mrodm Apr 20, 2026
b9fcd33
Test codeowners
mrodm Apr 20, 2026
0e64850
Revert changes in CODEOWNERS file
mrodm Apr 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 79 additions & 70 deletions .buildkite/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,16 +380,14 @@ package_name_manifest() {
}

is_package_excluded_in_config() {
local package=$1
local config_file_path=$2
local package_name="$1"
local config_file_path="$2"
local excluded_packages=""

excluded_packages=$(packages_excluded "${config_file_path}")
if [[ "${excluded_packages}" == "null" ]]; then
return 1
fi
local package_name=""
package_name=$(package_name_manifest)

# Avoid using "-q" in grep in this pipe, it could cause some weird behavior in some scenarios due to SIGPIPE errors when "set -o pipefail"
# https://tldp.org/LDP/lpg/node20.html
Expand Down Expand Up @@ -688,18 +686,25 @@ is_logsdb_compatible() {
return 0
}

# is_pr_affected accepts a package path and returns true if the package is affected by the PR
# it expects that the working directory is the package path to be checked
# Example:
# is_pr_affected "packages/elastic_package_registry" "origin/main" "origin/main"
is_pr_affected() {
local package="${1}"
local package_path="${1}"
local from="${2}"
local to="${3}"

local package_name=""
package_name=$(package_name_manifest)

local stack_supported=""
if ! stack_supported=$(is_supported_stack) ; then
echo "${FATAL_ERROR}"
return 1
fi
if [[ "${stack_supported}" == "false" ]]; then
echo "[${package}] PR is not affected: unsupported stack (${STACK_VERSION})"
echo "[${package_name}] PR is not affected: unsupported stack (${STACK_VERSION})"
return 1
fi

Expand All @@ -711,28 +716,28 @@ is_pr_affected() {
return 1
fi
if [[ "${logsdb_compatible}" == "false" ]]; then
echo "[${package}] PR is not affected: not supported LogsDB (${STACK_VERSION})"
echo "[${package_name}] PR is not affected: not supported LogsDB (${STACK_VERSION})"
return 1
fi
fi

if is_serverless; then
if is_package_excluded_in_config "${package}" "${WORKSPACE}/kibana.serverless.config.yml"; then
echo "[${package}] PR is not affected: package ${package} excluded in Kibana config for ${SERVERLESS_PROJECT}"
if is_package_excluded_in_config "${package_name}" "${WORKSPACE}/kibana.serverless.config.yml"; then
echo "[${package_name}] PR is not affected: package ${package_name} excluded in Kibana config for ${SERVERLESS_PROJECT}"
return 1
fi
if ! is_supported_capability ; then
echo "[${package}] PR is not affected: capabilities not matched with the project (${SERVERLESS_PROJECT})"
echo "[${package_name}] PR is not affected: capabilities not matched with the project (${SERVERLESS_PROJECT})"
return 1
fi
if [[ "${package}" == "fleet_server" ]]; then
if [[ "${package_name}" == "fleet_server" ]]; then
echoerr "fleet_server not supported. Skipped"
echo "[${package}] not supported"
echo "[${package_name}] not supported"
return 1
fi
if ! is_spec_3_0_0 ; then
echoerr "Not v3 spec version. Skipped"
echo "[${package}] spec <3.0.0"
echo "[${package_name}] spec <3.0.0"
return 1
fi
fi
Expand All @@ -742,44 +747,44 @@ is_pr_affected() {
return 1
fi
if [[ "${compatible}" == "false" ]]; then
echo "[${package}] PR is not affected: subscription not compatible with ${ELASTIC_SUBSCRIPTION}"
echo "[${package_name}] PR is not affected: subscription not compatible with ${ELASTIC_SUBSCRIPTION}"
return 1
fi

if [[ "${FORCE_CHECK_ALL}" == "true" ]];then
echo "[${package}] PR is affected: \"force_check_all\" parameter enabled"
echo "[${package_name}] PR is affected: \"force_check_all\" parameter enabled"
return 0
fi

commit_merge=$(git merge-base "${from}" "${to}")
echoerr "[${package}] git-diff: check non-package files (${commit_merge}..${to})"
echoerr "[${package_name}] git-diff: check non-package files (${commit_merge}..${to})"
# Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
# Example:
# https://buildkite.com/elastic/integrations/builds/25606
# https://github.com/elastic/integrations/pull/13810
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|CODE_OF_CONDUCT\.md|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml|scripts/packages/.+\.sh|scripts/backport_branch\.sh))' > /dev/null; then
echo "[${package}] PR is affected: found non-package files"
echo "[${package_name}] PR is affected: found non-package files"
return 0
fi
echoerr "[${package}] git-diff: check custom package checker script file (${commit_merge}..${to})"
echoerr "[${package_name}] git-diff: check custom package checker script file (${commit_merge}..${to})"
# Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
# Example:
# https://buildkite.com/elastic/integrations/builds/25606
# https://github.com/elastic/integrations/pull/13810
if git diff --name-only "${commit_merge}" "${to}" | grep -E "^\.buildkite/scripts/packages/${package}.sh" > /dev/null; then
echo "[${package}] PR is affected: found package checker script changes"
if git diff --name-only "${commit_merge}" "${to}" | grep -E "^\.buildkite/scripts/${package_path}.sh" > /dev/null; then
echo "[${package_name}] PR is affected: found package checker script changes"
return 0
fi
echoerr "[${package}] git-diff: check package files (${commit_merge}..${to})"
echoerr "[${package_name}] git-diff: check package files (${commit_merge}..${to})"
# Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
# Example:
# https://buildkite.com/elastic/integrations/builds/25606
# https://github.com/elastic/integrations/pull/13810
if git diff --name-only "${commit_merge}" "${to}" | grep -E "^packages/${package}/" > /dev/null ; then
echo "[${package}] PR is affected: found package files"
if git diff --name-only "${commit_merge}" "${to}" | grep -E "^${package_path}/" > /dev/null ; then
echo "[${package_name}] PR is affected: found package files"
return 0
fi
echo "[${package}] PR is not affected"
echo "[${package_name}] PR is not affected"
return 1
}

Expand All @@ -799,20 +804,20 @@ kubernetes_service_deployer_used() {
}

teardown_serverless_test_package() {
local package=$1
local package_name="$1"
local build_directory="${WORKSPACE}/build"
local dump_directory="${build_directory}/elastic-stack-dump/${package}"
local dump_directory="${build_directory}/elastic-stack-dump/${package_name}"

echo "--- Collect Elastic stack logs"
${ELASTIC_PACKAGE_BIN} stack dump -v --output "${dump_directory}"

upload_safe_logs_from_package "${package}" "${build_directory}"
upload_safe_logs_from_package "${package_name}" "${build_directory}"
}

teardown_test_package() {
local package=$1
local package_name="$1"
local build_directory="${WORKSPACE}/build"
local dump_directory="${build_directory}/elastic-stack-dump/${package}"
local dump_directory="${build_directory}/elastic-stack-dump/${package_name}"

if ! is_stack_created; then
echo "No stack running. Skip dump logs and run stack down process."
Expand All @@ -822,19 +827,20 @@ teardown_test_package() {
echo "--- Collect Elastic stack logs"
${ELASTIC_PACKAGE_BIN} stack dump -v --output "${dump_directory}"

upload_safe_logs_from_package "${package}" "${build_directory}"
upload_safe_logs_from_package "${package_name}" "${build_directory}"

echo "--- Take down the Elastic stack"
${ELASTIC_PACKAGE_BIN} stack down -v
}

# list all directories that are packages from the root of the repository
list_all_directories() {
find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
mage -d "${WORKSPACE}" listPackages | grep -E '^packages/(elastic_package_registry|nginx)$'
}
Comment on lines 865 to 867
Copy link
Copy Markdown

@macroscopeapp macroscopeapp bot Apr 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 Critical scripts/common.sh:831

list_all_directories() now hardcodes a grep filter that returns only two package names (elastic_package_registry and nginx). Since callers trigger_integrations_in_parallel.sh and test_integrations_with_serverless.sh iterate over this output to drive CI pipelines, all other packages are silently excluded from testing and publishing. If this restriction is intentional, consider documenting why only these two packages should run; otherwise, the filter should be removed to restore full package coverage.

 list_all_directories() {
-    mage listPackages | grep -E '^packages/(elastic_package_registry|nginx)$'
+    mage listPackages
 }
🤖 Copy this AI Prompt to have your agent fix this: 
In file .buildkite/scripts/common.sh around lines 831-833:

`list_all_directories()` now hardcodes a grep filter that returns only two package names (`elastic_package_registry` and `nginx`). Since callers `trigger_integrations_in_parallel.sh` and `test_integrations_with_serverless.sh` iterate over this output to drive CI pipelines, all other packages are silently excluded from testing and publishing. If this restriction is intentional, consider documenting why only these two packages should run; otherwise, the filter should be removed to restore full package coverage.

Copy link
Copy Markdown
Collaborator Author

@mrodm mrodm Apr 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will be removed before merging the PR. This grep is there to test just a subset of packages instead of the ~400 packages.

Comment on lines 865 to 867
Copy link
Copy Markdown

@macroscopeapp macroscopeapp bot Apr 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High scripts/common.sh:865

list_all_directories() at line 866 hardcodes a grep -E filter that restricts output to only 6 specific packages, silently skipping all other packages from the build/publish and serverless testing pipelines. This appears to be a development filter accidentally committed in place of the previous find implementation that listed all package directories. Consider reverting to the previous implementation or using a more appropriate mechanism for temporary filtering.

+# list all directories that are packages from the root of the repository
 list_all_directories() {
-    mage -d "${WORKSPACE}" listPackages | grep -E '^packages/(elastic_package_registry|nginx|rapid7_insightvm|sql_input|nginx/nginx|nginx/nginx_otel)$'
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
 }
🤖 Copy this AI Prompt to have your agent fix this: 
In file .buildkite/scripts/common.sh around lines 865-867:

`list_all_directories()` at line 866 hardcodes a `grep -E` filter that restricts output to only 6 specific packages, silently skipping all other packages from the build/publish and serverless testing pipelines. This appears to be a development filter accidentally committed in place of the previous `find` implementation that listed all package directories. Consider reverting to the previous implementation or using a more appropriate mechanism for temporary filtering.

Copy link
Copy Markdown
Collaborator Author

@mrodm mrodm Apr 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All this filtering of packages is related to run different testing in CI pipelines. It will be removed that grep and keep the mage -d ${WORKSPACE} listPackages command before merging this PR.

Copy link
Copy Markdown

@macroscopeapp macroscopeapp bot Apr 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for confirming. Would you like me to remove the grep filter and restore the command to just mage -d "${WORKSPACE}" listPackages, or will you handle this yourself?


check_package() {
local package=$1
echo "Check package: ${package}"
local package_name="$1"
echo "Check package: ${package_name}"
if ! ${ELASTIC_PACKAGE_BIN} check -v ; then
return 1
fi
Expand All @@ -843,8 +849,8 @@ check_package() {
}

build_zip_package() {
local package=$1
echo "Build zip package: ${package}"
local package_name="$1"
echo "Build zip package: ${package_name}"
if ! ${ELASTIC_PACKAGE_BIN} build --zip ; then
return 1
fi
Expand All @@ -853,21 +859,21 @@ build_zip_package() {
}

skip_installation_step() {
local package=$1
local package_name="$1"
if ! is_serverless ; then
return 1
fi

if [[ "$package" == "security_detection_engine" ]]; then
if [[ "$package_name" == "security_detection_engine" ]]; then
return 0
fi

return 1
}

install_package() {
local package=$1
echo "Install package: ${package}"
local package_name="$1"
echo "Install package: ${package_name}"
if ! ${ELASTIC_PACKAGE_BIN} install "${ELASTIC_PACKAGE_VERBOSITY}" ; then
return 1
fi
Expand All @@ -876,10 +882,10 @@ install_package() {
}

test_package_in_local_stack() {
local package=$1
local package_name="$1"
TEST_OPTIONS="--report-format xUnit --report-output file"

echo "Test package: ${package}"
echo "Test package: ${package_name}"
# Run all test suites
${ELASTIC_PACKAGE_BIN} test "${ELASTIC_PACKAGE_VERBOSITY}" ${TEST_OPTIONS} ${COVERAGE_OPTIONS}
local ret=$?
Expand All @@ -891,10 +897,10 @@ test_package_in_local_stack() {
# too much time, since all packages are run in the same step one by one.
# Packages are tested one by one to avoid creating more than 100 projects for one build.
test_package_in_serverless() {
local package=$1
local package_name="$1"
TEST_OPTIONS="${ELASTIC_PACKAGE_VERBOSITY} --report-format xUnit --report-output file"

echo "Test package: ${package}"
echo "Test package: ${package_name}"
if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
return 1
fi
Expand All @@ -914,9 +920,9 @@ test_package_in_serverless() {
}

run_tests_package() {
local package=$1
echo "--- [${package}] format and lint"
if ! check_package "${package}" ; then
local package_name="$1"
echo "--- [${package_name}] format and lint"
if ! check_package "${package_name}" ; then
return 1
fi

Expand All @@ -927,26 +933,26 @@ run_tests_package() {
fi
fi

if ! skip_installation_step "${package}" ; then
echo "--- [${package}] test installation"
if ! install_package "${package}" ; then
if [[ "${package}" == "elastic_connectors" ]]; then
if ! skip_installation_step "${package_name}" ; then
echo "--- [${package_name}] test installation"
if ! install_package "${package_name}" ; then
if [[ "${package_name}" == "elastic_connectors" ]]; then
# TODO: Remove this skip once elastic_connectors can be installed again
# For reference: https://github.com/elastic/kibana/pull/211419
echo "[${package}]: Known issue when package is installed - skipped all tests"
echo "[${package_name}]: Known issue when package is installed - skipped all tests"
return 0
fi
return 1
fi
fi

echo "--- [${package}] run test suites"
echo "--- [${package_name}] run test suites"
if is_serverless; then
if ! test_package_in_serverless "${package}" ; then
if ! test_package_in_serverless "${package_name}" ; then
return 1
fi
else
if ! test_package_in_local_stack "${package}" ; then
if ! test_package_in_local_stack "${package_name}" ; then
return 1
fi
fi
Expand Down Expand Up @@ -996,10 +1002,10 @@ upload_safe_logs_from_package() {
return
fi

local package=$1
local package_name="$1"
local retry_count="${BUILDKITE_RETRY_COUNT:-"0"}"
if [[ "${retry_count}" -ne 0 ]]; then
package="${package}_retry_${retry_count}"
package_name="${package_name}_retry_${retry_count}"
fi
local build_directory=$2

Expand All @@ -1009,29 +1015,32 @@ upload_safe_logs_from_package() {

upload_safe_logs \
"${JOB_GCS_BUCKET_INTERNAL}" \
"${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/*.*" \
"${parent_folder}/${package}/elastic-agent-logs/"
"${build_directory}/elastic-stack-dump/${package_name}/logs/elastic-agent-internal/*.*" \
"${parent_folder}/${package_name}/elastic-agent-logs/"

# required for <8.6.0
upload_safe_logs \
"${JOB_GCS_BUCKET_INTERNAL}" \
"${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/default/*" \
"${parent_folder}/${package}/elastic-agent-logs/default/"
"${build_directory}/elastic-stack-dump/${package_name}/logs/elastic-agent-internal/default/*" \
"${parent_folder}/${package_name}/elastic-agent-logs/default/"

upload_safe_logs \
"${JOB_GCS_BUCKET_INTERNAL}" \
"${build_directory}/container-logs/*.log" \
"${parent_folder}/${package}/container-logs/"
"${parent_folder}/${package_name}/container-logs/"
}

# Helper to run all tests and checks for a package
process_package() {
local package="${1}"
local package_path="${1}"
local failed_packages_file="${2:-""}"
local exit_code=0
local package_name=""

pushd "${package_path}" > /dev/null

echo "--- Package ${package}: check"
pushd "${package}" > /dev/null
package_name="$(package_name_manifest)"
echo "--- Package ${package_name}: check"

clean_safe_logs

Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Expand All @@ -1049,13 +1058,13 @@ process_package() {
fi
fi

if ! run_tests_package "${package}" ; then
if ! run_tests_package "${package_name}" ; then
exit_code=1
# Ensure that the group where the failure happened is opened.
echo "^^^ +++"
echo "[${package}] run_tests_package failed"
echo "[${package_name}] run_tests_package failed"
if [[ "${failed_packages_file}" != "" ]]; then
echo "- ${package}" >> "${failed_packages_file}"
echo "- ${package_name}" >> "${failed_packages_file}"
fi
fi

Expand All @@ -1070,13 +1079,13 @@ process_package() {
fi

if is_serverless ; then
teardown_serverless_test_package "${package}"
teardown_serverless_test_package "${package_name}"
else
if ! teardown_test_package "${package}" ; then
if ! teardown_test_package "${package_name}" ; then
exit_code=1
# Ensure that the group where the failure happened is opened.
echo "^^^ +++"
echo "[${package}] teardown_test_package failed"
echo "[${package_name}] teardown_test_package failed"
fi
fi

Expand Down
Loading
Loading