diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 2538e32d651..5bafe5e62d5 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -3,6 +3,7 @@ name: "integrations-backport" env: + SETUP_GVM_VERSION: "v0.6.0" YQ_VERSION: 'v4.35.2' # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" @@ -35,10 +36,6 @@ steps: key: "PACKAGE_NAME" required: true default: "" - - text: "Enter name of the folder for the package (in most cases coincides with PACKAGE_NAME)" - key: "PACKAGE_FOLDER_NAME" - required: true - default: "" - text: "Enter package version (examples: 1.5.7, 1.0.0-beta1)" key: "PACKAGE_VERSION" required: true diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh index 1c63b93924d..c1381766779 100755 --- a/.buildkite/scripts/backport_branch.sh +++ b/.buildkite/scripts/backport_branch.sh @@ -16,12 +16,11 @@ trap cleanup_gh EXIT DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default "${DRY_RUN:-"true"}")" BASE_COMMIT="$(buildkite-agent meta-data get BASE_COMMIT --default "${BASE_COMMIT:-""}")" PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default "${PACKAGE_NAME:-""}")" -PACKAGE_FOLDER_NAME="$(buildkite-agent meta-data get PACKAGE_FOLDER_NAME --default "${PACKAGE_FOLDER_NAME:-""}")" PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default "${PACKAGE_VERSION:-""}")" REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get REMOVE_OTHER_PACKAGES --default "${REMOVE_OTHER_PACKAGES:-"false"}")" -if [[ -z "$PACKAGE_NAME" ]] || [[ -z "$PACKAGE_FOLDER_NAME" ]] || [[ -z "$PACKAGE_VERSION" ]]; then - buildkite-agent annotate "The variables **PACKAGE_NAME**, **PACKAGE_FOLDER_NAME** or **PACKAGE_VERSION** aren't defined, please try again" --style "warning" +if [[ -z "$PACKAGE_NAME" ]] || [[ -z "$PACKAGE_VERSION" ]]; then + buildkite-agent annotate "The variables **PACKAGE_NAME** or **PACKAGE_VERSION** aren't defined, please try again" --style "warning" exit 1 fi @@ -30,7 +29,6 @@ PARAMETERS=( "**DRY_RUN**=$DRY_RUN" "**BASE_COMMIT**=$BASE_COMMIT" "**PACKAGE_NAME**=$PACKAGE_NAME" - "**PACKAGE_FOLDER_NAME**=$PACKAGE_FOLDER_NAME" "**PACKAGE_VERSION**=$PACKAGE_VERSION" "**REMOVE_OTHER_PACKAGES**=$REMOVE_OTHER_PACKAGES" ) @@ -43,6 +41,10 @@ echo "Parameters: ${PARAMETERS[*]}" | sed 's/ /\n- /g' | buildkite-agent annotat FULL_ZIP_PACKAGE_NAME="${PACKAGE_NAME}-${PACKAGE_VERSION}.zip" TRIMMED_PACKAGE_VERSION="$(echo "$PACKAGE_VERSION" | cut -d '.' -f -2)" SOURCE_BRANCH="main" +# To be removed +git checkout -b test_main +SOURCE_BRANCH="$(git rev-parse --abbrev-ref HEAD)" +echo "--- SOURCE_BRANCH: ${SOURCE_BRANCH}" BACKPORT_BRANCH_NAME="backport-${PACKAGE_NAME}-${TRIMMED_PACKAGE_VERSION}" PACKAGES_FOLDER_PATH="packages" MSG="" @@ -85,6 +87,24 @@ branchExist() { fi } +# get_package_path returns the path of the package with the given name as +# defined in the manifest.yml `name` field. Returns 1 if not found. +get_package_path() { + local package_name="${1}" + local package_path="" + + while IFS= read -r package_path; do + local name + name=$(yq -r '.name' "${package_path}/manifest.yml") + if [[ "${name}" == "${package_name}" ]]; then + echo "${package_path}" + return 0 + fi + done < <(list_all_directories) + + return 1 +} + createLocalBackportBranch() { local branch_name=$1 local source_commit=$2 @@ -97,18 +117,18 @@ createLocalBackportBranch() { } removeOtherPackages() { - local sourceFolder=$1 - local currentPackage="" - local dir - for dir in "$sourceFolder"/*; do - if [[ -d "$dir" ]] && [[ "$(basename "$dir")" != "$PACKAGE_FOLDER_NAME" ]]; then - echo "Removing directory: $dir" - rm -rf "$dir" - - currentPackage=$(basename "${dir}") - echo "Removing ${currentPackage} from .github/CODEOWNERS" - sed -i "/^\/packages\/${currentPackage}\//d" .github/CODEOWNERS - sed -i "/^\/packages\/${currentPackage} /d" .github/CODEOWNERS + local package_path_to_keep="${1}" + local package_path + local package_paths="" + package_paths=$(list_all_directories) + for package_path in ${package_paths}; do + if [[ -d "$package_path" ]] && [[ "${package_path}" != "${package_path_to_keep}" ]]; then + echo "Removing directory: ${package_path}" + rm -rf "$package_path" + + echo "Removing ${package_path} from .github/CODEOWNERS" + sed -i "\|^/${package_path}/|d" .github/CODEOWNERS + sed -i "\|^/${package_path} |d" .github/CODEOWNERS fi done } @@ -204,7 +224,7 @@ updateBackportBranchContents() { if [ "${REMOVE_OTHER_PACKAGES}" == "true" ]; then echo "--- Removing all packages from $PACKAGES_FOLDER_PATH folder" - removeOtherPackages "${PACKAGES_FOLDER_PATH}" + removeOtherPackages "${PACKAGE_PATH}" ls -la "${PACKAGES_FOLDER_PATH}" git add "${PACKAGES_FOLDER_PATH}/" @@ -228,7 +248,7 @@ updateBackportBranchContents() { if [ "$DRY_RUN" == "true" ];then echo "--- DRY_RUN mode, nothing will be pushed." # Show just the relevant files diff (go.mod, go.sum, .buildkite, dev, .go-version, .github/CODEOWNERS and package to be backported) - git --no-pager diff "$SOURCE_BRANCH...$BACKPORT_BRANCH_NAME" .buildkite/ dev/ go.sum go.mod .go-version tools.go .github/CODEOWNERS "packages/${PACKAGE_FOLDER_NAME}" + git --no-pager diff "$SOURCE_BRANCH...$BACKPORT_BRANCH_NAME" .buildkite/ dev/ go.sum go.mod .go-version tools.go .github/CODEOWNERS "${PACKAGE_PATH}" else echo "--- Pushing..." git push origin "$BACKPORT_BRANCH_NAME" @@ -245,6 +265,15 @@ fi add_bin_path with_yq +with_mage + +echo "--- Resolve package path from PACKAGE_NAME" +PACKAGE_PATH="$(get_package_path "${PACKAGE_NAME}" || true)" +if [[ -z "${PACKAGE_PATH}" ]]; then + buildkite-agent annotate "Package **${PACKAGE_NAME}** not found" --style "error" + exit 1 +fi +echo "Package path: ${PACKAGE_PATH}" echo "--- Check if the package is published" if ! isPackagePublished "$FULL_ZIP_PACKAGE_NAME"; then @@ -268,15 +297,15 @@ if branchExist "$BACKPORT_BRANCH_NAME"; then fi # backport branch does not exist, running checks and create branch -version="$(git show "${BASE_COMMIT}":"packages/${PACKAGE_FOLDER_NAME}/manifest.yml" | yq -r .version)" +version="$(git show "${BASE_COMMIT}":"${PACKAGE_PATH}/manifest.yml" | yq -r .version)" echo "--- Check if version from ${BASE_COMMIT} (${version}) matches with version from input step ${PACKAGE_VERSION}" if [[ "${version}" != "${PACKAGE_VERSION}" ]]; then - buildkite-agent annotate "Unexpected version found in packages/${PACKAGE_FOLDER_NAME}/manifest.yml" --style "error" + buildkite-agent annotate "Unexpected version found in ${PACKAGE_PATH}/manifest.yml" --style "error" exit 1 fi echo "---Check that this changeset is the one creating the version $PACKAGE_NAME" -if ! git show -p "${BASE_COMMIT}" "packages/${PACKAGE_FOLDER_NAME}/manifest.yml" | grep -E "^\+version: \"{0,1}${PACKAGE_VERSION}" ; then +if ! git show -p "${BASE_COMMIT}" "${PACKAGE_PATH}/manifest.yml" | grep -E "^\+version: \"{0,1}${PACKAGE_VERSION}" ; then buildkite-agent annotate "This changeset does not creates the version ${PACKAGE_VERSION}" --style "error" exit 1 fi diff --git a/.buildkite/scripts/build_packages.sh b/.buildkite/scripts/build_packages.sh index dfc6ada97bc..7e906de397c 100644 --- a/.buildkite/scripts/build_packages.sh +++ b/.buildkite/scripts/build_packages.sh @@ -48,21 +48,21 @@ report_build_failure() { } build_packages() { - pushd packages > /dev/null || exit 1 + local packages="" + local version="" + local name="" + local package_zip="" + local package_path="" - for it in $(find . -maxdepth 1 -mindepth 1 -type d); do - local package - local version - local name - package=$(basename "${it}") - echo "Package ${package}: check" + packages=$(list_all_directories) + for package_path in ${packages}; do + pushd "${package_path}" > /dev/null || exit 1 + echo "Package \"${package_path}\": check" - pushd "${package}" > /dev/null || exit 1 + version=$(yq .version manifest.yml) + name=$(yq .name manifest.yml) - version=$(cat manifest.yml | yq .version) - name=$(cat manifest.yml | yq .name) - - local package_zip="${name}-${version}.zip" + package_zip="${name}-${version}.zip" if is_already_published "${package_zip}" ; then echo "Skipping. ${package_zip} already published" @@ -70,15 +70,14 @@ build_packages() { continue fi - echo "Build package as zip: ${package}" - if check_and_build_package "${package}" ; then + echo "Build package as zip: ${package_path}" + if check_and_build_package "${package_path}" ; then unpublished="true" else - report_build_failure "${package}" + report_build_failure "${package_path}" fi popd > /dev/null || exit 1 done - popd > /dev/null || exit 1 } if [ "${SKIP_PUBLISHING}" == "true" ] ; then @@ -88,13 +87,13 @@ fi if skipPublishing ; then echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published" - exit 0 + # exit 0 fi add_bin_path with_yq -with_go +with_mage use_elastic_package echo "--- Build packages" @@ -117,7 +116,7 @@ cp "${BUILD_PACKAGES_FOLDER}"/*.zip "${ARTIFACTS_FOLDER}"/ if [ "${DRY_RUN}" == "true" ]; then echo "DRY_RUN enabled. Publish packages steps skipped." - exit 0 + # exit 0 fi # triggering dynamically the steps for signing and publishing @@ -148,7 +147,7 @@ steps: env: SIGNING_STEP_KEY: "sign-service" ARTIFACTS_FOLDER: "packageArtifacts" - DRY_RUN: "${DRY_RUN}" + DRY_RUN: "true" agents: image: "${LINUX_AGENT_IMAGE}" cpu: "8" @@ -158,4 +157,5 @@ steps: allow_failure: false EOF -buildkite-agent pipeline upload "${PIPELINE_FILE}" +cat "${PIPELINE_FILE}" +# buildkite-agent pipeline upload "${PIPELINE_FILE}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2dd185ff62b..09b56d43bb1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -380,16 +380,14 @@ package_name_manifest() { } is_package_excluded_in_config() { - local package=$1 - local config_file_path=$2 + local package_name="$1" + local config_file_path="$2" local excluded_packages="" excluded_packages=$(packages_excluded "${config_file_path}") if [[ "${excluded_packages}" == "null" ]]; then return 1 fi - local package_name="" - package_name=$(package_name_manifest) # Avoid using "-q" in grep in this pipe, it could cause some weird behavior in some scenarios due to SIGPIPE errors when "set -o pipefail" # https://tldp.org/LDP/lpg/node20.html @@ -688,18 +686,25 @@ is_logsdb_compatible() { return 0 } +# is_pr_affected accepts a package path and returns true if the package is affected by the PR +# it expects that the working directory is the package path to be checked +# Example: +# is_pr_affected "packages/elastic_package_registry" "origin/main" "origin/main" is_pr_affected() { - local package="${1}" + local package_path="${1}" local from="${2}" local to="${3}" + local package_name="" + package_name=$(package_name_manifest) + local stack_supported="" if ! stack_supported=$(is_supported_stack) ; then echo "${FATAL_ERROR}" return 1 fi if [[ "${stack_supported}" == "false" ]]; then - echo "[${package}] PR is not affected: unsupported stack (${STACK_VERSION})" + echo "[${package_name}] PR is not affected: unsupported stack (${STACK_VERSION})" return 1 fi @@ -711,28 +716,28 @@ is_pr_affected() { return 1 fi if [[ "${logsdb_compatible}" == "false" ]]; then - echo "[${package}] PR is not affected: not supported LogsDB (${STACK_VERSION})" + echo "[${package_name}] PR is not affected: not supported LogsDB (${STACK_VERSION})" return 1 fi fi if is_serverless; then - if is_package_excluded_in_config "${package}" "${WORKSPACE}/kibana.serverless.config.yml"; then - echo "[${package}] PR is not affected: package ${package} excluded in Kibana config for ${SERVERLESS_PROJECT}" + if is_package_excluded_in_config "${package_name}" "${WORKSPACE}/kibana.serverless.config.yml"; then + echo "[${package_name}] PR is not affected: package ${package_name} excluded in Kibana config for ${SERVERLESS_PROJECT}" return 1 fi if ! is_supported_capability ; then - echo "[${package}] PR is not affected: capabilities not matched with the project (${SERVERLESS_PROJECT})" + echo "[${package_name}] PR is not affected: capabilities not matched with the project (${SERVERLESS_PROJECT})" return 1 fi - if [[ "${package}" == "fleet_server" ]]; then + if [[ "${package_name}" == "fleet_server" ]]; then echoerr "fleet_server not supported. Skipped" - echo "[${package}] not supported" + echo "[${package_name}] not supported" return 1 fi if ! is_spec_3_0_0 ; then echoerr "Not v3 spec version. Skipped" - echo "[${package}] spec <3.0.0" + echo "[${package_name}] spec <3.0.0" return 1 fi fi @@ -742,47 +747,75 @@ is_pr_affected() { return 1 fi if [[ "${compatible}" == "false" ]]; then - echo "[${package}] PR is not affected: subscription not compatible with ${ELASTIC_SUBSCRIPTION}" + echo "[${package_name}] PR is not affected: subscription not compatible with ${ELASTIC_SUBSCRIPTION}" return 1 fi if [[ "${FORCE_CHECK_ALL}" == "true" ]];then - echo "[${package}] PR is affected: \"force_check_all\" parameter enabled" + echo "[${package_name}] PR is affected: \"force_check_all\" parameter enabled" return 0 fi commit_merge=$(git merge-base "${from}" "${to}") - echoerr "[${package}] git-diff: check non-package files (${commit_merge}..${to})" + echoerr "[${package_name}] git-diff: check non-package files (${commit_merge}..${to})" # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail" # Example: # https://buildkite.com/elastic/integrations/builds/25606 # https://github.com/elastic/integrations/pull/13810 if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|CODE_OF_CONDUCT\.md|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml|scripts/packages/.+\.sh|scripts/backport_branch\.sh))' > /dev/null; then - echo "[${package}] PR is affected: found non-package files" + echo "[${package_name}] PR is affected: found non-package files" return 0 fi - echoerr "[${package}] git-diff: check custom package checker script file (${commit_merge}..${to})" + echoerr "[${package_name}] git-diff: check custom package checker script file (${commit_merge}..${to})" # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail" # Example: # https://buildkite.com/elastic/integrations/builds/25606 # https://github.com/elastic/integrations/pull/13810 - if git diff --name-only "${commit_merge}" "${to}" | grep -E "^\.buildkite/scripts/packages/${package}.sh" > /dev/null; then - echo "[${package}] PR is affected: found package checker script changes" + if git diff --name-only "${commit_merge}" "${to}" | grep -E "^\.buildkite/scripts/${package_path}.sh" > /dev/null; then + echo "[${package_name}] PR is affected: found package checker script changes" return 0 fi - echoerr "[${package}] git-diff: check package files (${commit_merge}..${to})" + echoerr "[${package_name}] git-diff: check package files (${commit_merge}..${to})" # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail" # Example: # https://buildkite.com/elastic/integrations/builds/25606 # https://github.com/elastic/integrations/pull/13810 - if git diff --name-only "${commit_merge}" "${to}" | grep -E "^packages/${package}/" > /dev/null ; then - echo "[${package}] PR is affected: found package files" + if git diff --name-only "${commit_merge}" "${to}" | grep -E "^${package_path}/" > /dev/null ; then + echo "[${package_name}] PR is affected: found package files" return 0 fi - echo "[${package}] PR is not affected" + echo "[${package_name}] PR is not affected" return 1 } +# should_test_package checks if a package is affected by the current PR. +# Prints the reason to stderr. Returns 0 if the package should be tested, +# 1 if it should be skipped. Exits on fatal error. +should_test_package() { + local package_path="${1}" + local from="${2}" + local to="${3}" + + local reason="" + local skip="false" + + pushd "${package_path}" > /dev/null + if ! reason=$(is_pr_affected "${package_path}" "${from}" "${to}"); then + skip="true" + if [[ "${reason}" == "${FATAL_ERROR}" ]]; then + echo "Unexpected failure checking ${package_path}" >&2 + exit 1 + fi + fi + popd > /dev/null + + echoerr "${reason}" + if [[ "${skip}" == "true" ]]; then + return 1 + fi + return 0 +} + is_pr() { if [[ "${BUILDKITE_PULL_REQUEST}" == "false" && "${BUILDKITE_TAG}" == "" ]]; then return 1 @@ -799,20 +832,20 @@ kubernetes_service_deployer_used() { } teardown_serverless_test_package() { - local package=$1 + local package_name="$1" local build_directory="${WORKSPACE}/build" - local dump_directory="${build_directory}/elastic-stack-dump/${package}" + local dump_directory="${build_directory}/elastic-stack-dump/${package_name}" echo "--- Collect Elastic stack logs" ${ELASTIC_PACKAGE_BIN} stack dump -v --output "${dump_directory}" - upload_safe_logs_from_package "${package}" "${build_directory}" + upload_safe_logs_from_package "${package_name}" "${build_directory}" } teardown_test_package() { - local package=$1 + local package_name="$1" local build_directory="${WORKSPACE}/build" - local dump_directory="${build_directory}/elastic-stack-dump/${package}" + local dump_directory="${build_directory}/elastic-stack-dump/${package_name}" if ! is_stack_created; then echo "No stack running. Skip dump logs and run stack down process." @@ -822,19 +855,20 @@ teardown_test_package() { echo "--- Collect Elastic stack logs" ${ELASTIC_PACKAGE_BIN} stack dump -v --output "${dump_directory}" - upload_safe_logs_from_package "${package}" "${build_directory}" + upload_safe_logs_from_package "${package_name}" "${build_directory}" echo "--- Take down the Elastic stack" ${ELASTIC_PACKAGE_BIN} stack down -v } +# list all directories that are packages from the root of the repository list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort + mage -d "${WORKSPACE}" listPackages | grep -E '^packages/(elastic_package_registry|nginx|rapid7_insightvm|sql_input|nginx/nginx|nginx/nginx_otel)$' } check_package() { - local package=$1 - echo "Check package: ${package}" + local package_name="$1" + echo "Check package: ${package_name}" if ! ${ELASTIC_PACKAGE_BIN} check -v ; then return 1 fi @@ -843,8 +877,8 @@ check_package() { } build_zip_package() { - local package=$1 - echo "Build zip package: ${package}" + local package_name="$1" + echo "Build zip package: ${package_name}" if ! ${ELASTIC_PACKAGE_BIN} build --zip ; then return 1 fi @@ -853,12 +887,12 @@ build_zip_package() { } skip_installation_step() { - local package=$1 + local package_name="$1" if ! is_serverless ; then return 1 fi - if [[ "$package" == "security_detection_engine" ]]; then + if [[ "$package_name" == "security_detection_engine" ]]; then return 0 fi @@ -866,8 +900,8 @@ skip_installation_step() { } install_package() { - local package=$1 - echo "Install package: ${package}" + local package_name="$1" + echo "Install package: ${package_name}" if ! ${ELASTIC_PACKAGE_BIN} install "${ELASTIC_PACKAGE_VERBOSITY}" ; then return 1 fi @@ -876,10 +910,10 @@ install_package() { } test_package_in_local_stack() { - local package=$1 + local package_name="$1" TEST_OPTIONS="--report-format xUnit --report-output file" - echo "Test package: ${package}" + echo "Test package: ${package_name}" # Run all test suites ${ELASTIC_PACKAGE_BIN} test "${ELASTIC_PACKAGE_VERBOSITY}" ${TEST_OPTIONS} ${COVERAGE_OPTIONS} local ret=$? @@ -891,10 +925,10 @@ test_package_in_local_stack() { # too much time, since all packages are run in the same step one by one. # Packages are tested one by one to avoid creating more than 100 projects for one build. test_package_in_serverless() { - local package=$1 + local package_name="$1" TEST_OPTIONS="${ELASTIC_PACKAGE_VERBOSITY} --report-format xUnit --report-output file" - echo "Test package: ${package}" + echo "Test package: ${package_name}" if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then return 1 fi @@ -914,9 +948,9 @@ test_package_in_serverless() { } run_tests_package() { - local package=$1 - echo "--- [${package}] format and lint" - if ! check_package "${package}" ; then + local package_name="$1" + echo "--- [${package_name}] format and lint" + if ! check_package "${package_name}" ; then return 1 fi @@ -927,26 +961,26 @@ run_tests_package() { fi fi - if ! skip_installation_step "${package}" ; then - echo "--- [${package}] test installation" - if ! install_package "${package}" ; then - if [[ "${package}" == "elastic_connectors" ]]; then + if ! skip_installation_step "${package_name}" ; then + echo "--- [${package_name}] test installation" + if ! install_package "${package_name}" ; then + if [[ "${package_name}" == "elastic_connectors" ]]; then # TODO: Remove this skip once elastic_connectors can be installed again # For reference: https://github.com/elastic/kibana/pull/211419 - echo "[${package}]: Known issue when package is installed - skipped all tests" + echo "[${package_name}]: Known issue when package is installed - skipped all tests" return 0 fi return 1 fi fi - echo "--- [${package}] run test suites" + echo "--- [${package_name}] run test suites" if is_serverless; then - if ! test_package_in_serverless "${package}" ; then + if ! test_package_in_serverless "${package_name}" ; then return 1 fi else - if ! test_package_in_local_stack "${package}" ; then + if ! test_package_in_local_stack "${package_name}" ; then return 1 fi fi @@ -996,10 +1030,10 @@ upload_safe_logs_from_package() { return fi - local package=$1 + local package_name="$1" local retry_count="${BUILDKITE_RETRY_COUNT:-"0"}" if [[ "${retry_count}" -ne 0 ]]; then - package="${package}_retry_${retry_count}" + package_name="${package_name}_retry_${retry_count}" fi local build_directory=$2 @@ -1009,29 +1043,32 @@ upload_safe_logs_from_package() { upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ - "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/*.*" \ - "${parent_folder}/${package}/elastic-agent-logs/" + "${build_directory}/elastic-stack-dump/${package_name}/logs/elastic-agent-internal/*.*" \ + "${parent_folder}/${package_name}/elastic-agent-logs/" # required for <8.6.0 upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ - "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/default/*" \ - "${parent_folder}/${package}/elastic-agent-logs/default/" + "${build_directory}/elastic-stack-dump/${package_name}/logs/elastic-agent-internal/default/*" \ + "${parent_folder}/${package_name}/elastic-agent-logs/default/" upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "${build_directory}/container-logs/*.log" \ - "${parent_folder}/${package}/container-logs/" + "${parent_folder}/${package_name}/container-logs/" } # Helper to run all tests and checks for a package process_package() { - local package="${1}" + local package_path="${1}" local failed_packages_file="${2:-""}" local exit_code=0 + local package_name="" + + pushd "${package_path}" > /dev/null - echo "--- Package ${package}: check" - pushd "${package}" > /dev/null + package_name="$(package_name_manifest)" + echo "--- Package ${package_name}: check" clean_safe_logs @@ -1049,13 +1086,13 @@ process_package() { fi fi - if ! run_tests_package "${package}" ; then + if ! run_tests_package "${package_name}" ; then exit_code=1 # Ensure that the group where the failure happened is opened. echo "^^^ +++" - echo "[${package}] run_tests_package failed" + echo "[${package_name}] run_tests_package failed" if [[ "${failed_packages_file}" != "" ]]; then - echo "- ${package}" >> "${failed_packages_file}" + echo "- ${package_name}" >> "${failed_packages_file}" fi fi @@ -1070,13 +1107,13 @@ process_package() { fi if is_serverless ; then - teardown_serverless_test_package "${package}" + teardown_serverless_test_package "${package_name}" else - if ! teardown_test_package "${package}" ; then + if ! teardown_test_package "${package_name}" ; then exit_code=1 # Ensure that the group where the failure happened is opened. echo "^^^ +++" - echo "[${package}] teardown_test_package failed" + echo "[${package_name}] teardown_test_package failed" fi fi diff --git a/.buildkite/scripts/test_integrations_with_serverless.sh b/.buildkite/scripts/test_integrations_with_serverless.sh index a87ba5be21f..9bbbfb57551 100755 --- a/.buildkite/scripts/test_integrations_with_serverless.sh +++ b/.buildkite/scripts/test_integrations_with_serverless.sh @@ -76,36 +76,18 @@ echo "Checking with commits: from: '${from}' to: '${to}'" any_package_failing=0 -pushd packages > /dev/null -for package in $(list_all_directories); do - echo "--- [$package] check if it is required to be tested" - pushd "${package}" > /dev/null - skip_package=false - failure=false - if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then - skip_package=true - if [[ "${reason}" == "${FATAL_ERROR}" ]]; then - failure=true - fi - fi - popd > /dev/null - if [[ "${failure}" == "true" ]]; then - echo "Unexpected failure checking ${package}" - exit 1 - fi - - echo "${reason}" - - if [[ "${skip_package}" == "true" ]]; then - echo "- ${reason}" >> "${SKIPPED_PACKAGES_FILE_PATH}" +PACKAGE_LIST=$(list_all_directories) +for package_path in ${PACKAGE_LIST}; do + echo "--- [$package_path] check if it is required to be tested" + if ! should_test_package "${package_path}" "${from}" "${to}"; then + echo "- ${package_path}" >> "${SKIPPED_PACKAGES_FILE_PATH}" continue fi - if ! process_package "${package}" "${FAILED_PACKAGES_FILE_PATH}" ; then + if ! process_package "${package_path}" "${FAILED_PACKAGES_FILE_PATH}" ; then any_package_failing=1 fi done -popd > /dev/null if running_on_buildkite ; then if [ -f "${SKIPPED_PACKAGES_FILE_PATH}" ]; then diff --git a/.buildkite/scripts/test_one_package.sh b/.buildkite/scripts/test_one_package.sh index 58cffaa4921..1e7e7d4731c 100755 --- a/.buildkite/scripts/test_one_package.sh +++ b/.buildkite/scripts/test_one_package.sh @@ -4,8 +4,9 @@ source .buildkite/scripts/common.sh set -euo pipefail -# package name -package="$1" +# package path +package_path="$1" +package_name="$(basename "${package_path}")" if [ ! -d packages ]; then echo "Missing packages folder" @@ -25,23 +26,21 @@ with_kubernetes use_elastic_package -pushd packages > /dev/null exit_code=0 -if ! process_package "${package}" ; then +if ! process_package "${package_path}" ; then # keep this message as a collapsed group in Buildkite, so it # is not hidden by the previous collapsed group. - echo "--- [${package}] failed" + echo "--- [${package_name}] failed" exit_code=1 fi -popd > /dev/null if [ "${exit_code}" -ne 0 ] ; then exit "${exit_code}" fi -custom_package_checker_script_path="${SCRIPTS_BUILDKITE_PATH}/packages/${package}.sh" +custom_package_checker_script_path="${SCRIPTS_BUILDKITE_PATH}/${package_path}.sh" if [ -x "$custom_package_checker_script_path" ]; then - echo "--- [${package}] Run individual package checker" + echo "--- [${package_name}] Run individual package checker" "$custom_package_checker_script_path" -fi \ No newline at end of file +fi diff --git a/.buildkite/scripts/trigger_integrations_in_parallel.sh b/.buildkite/scripts/trigger_integrations_in_parallel.sh index 15967e066e3..30a39cb225a 100755 --- a/.buildkite/scripts/trigger_integrations_in_parallel.sh +++ b/.buildkite/scripts/trigger_integrations_in_parallel.sh @@ -9,9 +9,8 @@ add_bin_path with_yq with_mage -pushd packages > /dev/null +echo "--- List all directories" PACKAGE_LIST=$(list_all_directories) -popd > /dev/null PIPELINE_FILE="packages_pipeline.yml" touch packages_pipeline.yml @@ -51,34 +50,23 @@ fi packages_to_test=0 -for package in ${PACKAGE_LIST}; do +for package_path in ${PACKAGE_LIST}; do # check if needed to create an step for this package - echo "--- [$package] check if it is required to be tested" - pushd "packages/${package}" > /dev/null - skip_package="false" - failure="false" - if ! reason=$(is_pr_affected "${package}" "${from}" "${to}") ; then - skip_package="true" - if [[ "${reason}" == "${FATAL_ERROR}" ]]; then - failure=true - fi - fi - popd > /dev/null - if [[ "${failure}" == "true" ]]; then - echo "Unexpected failure checking ${package}" - exit 1 - fi - - echoerr "${reason}" - if [[ "${skip_package}" == "true" ]] ; then + echo "--- [$package_path] check if it is required to be tested" + if ! should_test_package "${package_path}" "${from}" "${to}"; then continue fi + # package names (manifest.yml) are unique, so it can be used as key for the step + pushd "${package_path}" > /dev/null + package_name=$(package_name_manifest) + popd > /dev/null + packages_to_test=$((packages_to_test+1)) cat << EOF >> ${PIPELINE_FILE} - - label: "Check integrations ${package}" - key: "test-integrations-${package}" - command: ".buildkite/scripts/test_one_package.sh ${package} ${from} ${to}" + - label: "Check integrations ${package_name}" + key: "test-integrations-${package_name}" + command: ".buildkite/scripts/test_one_package.sh ${package_path} ${from} ${to}" timeout_in_minutes: 300 agents: provider: gcp diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a33be47220b..47d92412661 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -380,7 +380,7 @@ /packages/nginx @elastic/obs-infraobs-integrations /packages/nginx_ingress_controller @elastic/obs-ds-hosted-services /packages/nginx_ingress_controller_otel @elastic/obs-infraobs-integrations -/packages/nginx_otel @elastic/obs-infraobs-integrations +/packages/nginx/nginx_otel @elastic/obs-infraobs-integrations /packages/nginx_input_otel @elastic/obs-infraobs-integrations /packages/nozomi_networks @elastic/security-service-integrations /packages/nvidia_gpu @elastic/obs-infraobs-integrations diff --git a/dev/citools/kibana.go b/dev/citools/kibana.go index aef47205cb0..ae6a21d1113 100644 --- a/dev/citools/kibana.go +++ b/dev/citools/kibana.go @@ -12,7 +12,7 @@ import ( ) func KibanaConstraintPackage(path string) (*semver.Constraints, error) { - manifest, err := readPackageManifest(path) + manifest, err := ReadPackageManifest(path) if err != nil { return nil, fmt.Errorf("failed to read package manifest: %w", err) } diff --git a/dev/citools/logsdb.go b/dev/citools/logsdb.go index d2434c75032..1ed90c8222d 100644 --- a/dev/citools/logsdb.go +++ b/dev/citools/logsdb.go @@ -21,7 +21,7 @@ func IsVersionLessThanLogsDBGA(version *semver.Version) bool { } func packageKibanaConstraint(path string) (*semver.Constraints, error) { - manifest, err := readPackageManifest(path) + manifest, err := ReadPackageManifest(path) if err != nil { return nil, err } diff --git a/dev/citools/packagemanifest.go b/dev/citools/packagemanifest.go index 7c2c61ca9e2..dc9a861d955 100644 --- a/dev/citools/packagemanifest.go +++ b/dev/citools/packagemanifest.go @@ -6,11 +6,14 @@ package citools import ( "fmt" + "slices" "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" ) +var validTypes = []string{"integration", "input", "content"} + // kibanaConditions defines conditions for Kibana (e.g. required version). type kibanaConditions struct { Version string `config:"version" json:"version" yaml:"version"` @@ -28,12 +31,22 @@ type conditions struct { } type packageManifest struct { - Name string `config:"name" json:"name" yaml:"name"` - License string `config:"license" json:"license" yaml:"license"` - Conditions conditions `config:"conditions" json:"conditions" yaml:"conditions"` + FormatVersion string `config:"format_version" json:"format_version" yaml:"format_version"` + Name string `config:"name" json:"name" yaml:"name"` + Type string `config:"type" json:"type" yaml:"type"` + Version string `config:"version" json:"version" yaml:"version"` + License string `config:"license" json:"license" yaml:"license"` + Conditions conditions `config:"conditions" json:"conditions" yaml:"conditions"` +} + +func (m *packageManifest) IsValid() bool { + if m.FormatVersion == "" || m.Name == "" || m.Type == "" || m.Version == "" { + return false + } + return slices.Contains(validTypes, m.Type) } -func readPackageManifest(path string) (*packageManifest, error) { +func ReadPackageManifest(path string) (*packageManifest, error) { cfg, err := yaml.NewConfigWithFile(path, ucfg.PathSep(".")) if err != nil { return nil, fmt.Errorf("reading file failed (path: %s): %w", path, err) diff --git a/dev/citools/packages.go b/dev/citools/packages.go new file mode 100644 index 00000000000..411bd4e71e1 --- /dev/null +++ b/dev/citools/packages.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package citools + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" +) + +const ManifestFileName = "manifest.yml" + +// ListPackages returns the sorted paths of all packages found under dir. +func ListPackages(dir string) ([]string, error) { + var paths []string + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + manifestPath := filepath.Join(path, ManifestFileName) + _, statErr := os.Stat(manifestPath) + if errors.Is(statErr, os.ErrNotExist) { + return nil + } else if statErr != nil { + return fmt.Errorf("error statting manifest %s: %w", manifestPath, statErr) + } + manifest, err := ReadPackageManifest(manifestPath) + if err != nil { + return fmt.Errorf("error reading manifest %s: %w", manifestPath, err) + } + if !manifest.IsValid() { + return nil + } + paths = append(paths, path) + // No need to look deeper once a package is found. + return filepath.SkipDir + }) + if err != nil { + return nil, fmt.Errorf("error listing packages in %s: %w", dir, err) + } + slices.Sort(paths) + return paths, nil +} diff --git a/dev/citools/subscription.go b/dev/citools/subscription.go index a14169df68d..b611304f25c 100644 --- a/dev/citools/subscription.go +++ b/dev/citools/subscription.go @@ -9,7 +9,7 @@ import ( ) func packageSubscription(path string) (string, error) { - manifest, err := readPackageManifest(path) + manifest, err := ReadPackageManifest(path) if err != nil { return "", err } diff --git a/dev/codeowners/codeowners.go b/dev/codeowners/codeowners.go index 84391405914..270b5860e56 100644 --- a/dev/codeowners/codeowners.go +++ b/dev/codeowners/codeowners.go @@ -8,11 +8,12 @@ import ( "bufio" "fmt" "os" - "path" "path/filepath" "strings" "gopkg.in/yaml.v3" + + "github.com/elastic/integrations/dev/citools" ) const DefaultCodeownersPath = ".github/CODEOWNERS" @@ -63,34 +64,26 @@ type githubOwners struct { // data_streams, it checks that all data_streams are explicitly owned by a single owner. Such ownership // sharing packages are identified by having at least one data_stream with explicit ownership in codeowners. func validatePackages(codeowners *githubOwners, packagesDir string) error { - packageDirEntries, err := os.ReadDir(packagesDir) + paths, err := citools.ListPackages(packagesDir) if err != nil { - return fmt.Errorf("error reading directory '%s': %w", packagesDir, err) - } - - if len(packageDirEntries) == 0 { - if len(codeowners.owners) == 0 { - return nil - } - return fmt.Errorf("no packages found in %q", packagesDir) + return fmt.Errorf("error listing packages in %s: %w", packagesDir, err) } - - for _, packageDirEntry := range packageDirEntries { - packageName := packageDirEntry.Name() - - packagePath := path.Join(packagesDir, packageName) - - packageManifestPath := path.Join(packagePath, "manifest.yml") - err = codeowners.checkManifest(packageManifestPath) + for _, path := range paths { + err = codeowners.checkManifest(filepath.Join(path, citools.ManifestFileName)) if err != nil { - return fmt.Errorf("error checking manifest '%s': %w", packageManifestPath, err) + return fmt.Errorf("error checking manifest '%s': %w", path, err) } - - err = codeowners.checkDataStreams(packagePath) + err = codeowners.checkDataStreams(path) if err != nil { - return fmt.Errorf("error checking data streams from '%s': %w", packagePath, err) + return fmt.Errorf("error checking data streams from '%s': %w", path, err) } + } + if len(paths) == 0 { + if len(codeowners.owners) == 0 { + return nil + } + return fmt.Errorf("no packages found in %q", packagesDir) } return nil @@ -170,10 +163,9 @@ func (codeowners *githubOwners) checkSingleField(field string) error { } func (codeowners *githubOwners) checkManifest(path string) error { - pkgDir := filepath.Dir(path) - owners, found := codeowners.owners["/"+pkgDir] + owners, found := codeowners.findOwnerForFile(path) if !found { - return fmt.Errorf("there is no owner for %q in %q", pkgDir, codeowners.path) + return fmt.Errorf("there is no owner for %q in %q", filepath.Dir(path), codeowners.path) } content, err := os.ReadFile(path) @@ -208,8 +200,32 @@ func (codeowners *githubOwners) checkManifest(path string) error { return nil } +func (codeowners *githubOwners) findOwnerForFile(path string) ([]string, bool) { + // Usually paths are related to the root of the repository. Examples: + // - "packages/package-name/manifest.yml" + // - "packages/technology/package-name/manifest.yml" + // Just in case, if an absolute path is provided, we remove the leading separator. + if filepath.IsAbs(path) { + path = strings.TrimPrefix(path, string(filepath.Separator)) + } + ownerDir := filepath.Dir(path) + for { + owners, found := codeowners.owners["/"+filepath.ToSlash(ownerDir)] + if found { + return owners, found + } + + ownerDir = filepath.Dir(ownerDir) + if ownerDir == "." { + break + } + } + + return nil, false +} + func (codeowners *githubOwners) checkDataStreams(packagePath string) error { - packageDataStreamsPath := path.Join(packagePath, "data_stream") + packageDataStreamsPath := filepath.Join(packagePath, "data_stream") if _, err := os.Stat(packageDataStreamsPath); os.IsNotExist(err) { // package doesn't have data_streams return nil @@ -229,8 +245,8 @@ func (codeowners *githubOwners) checkDataStreams(packagePath string) error { var dataStreamsWithoutOwner []string for _, dataStreamDirEntry := range dataStreamDirEntries { dataStreamName := dataStreamDirEntry.Name() - dataStreamDir := path.Join(packageDataStreamsPath, dataStreamName) - dataStreamOwners, found := codeowners.owners["/"+dataStreamDir] + dataStreamDir := filepath.Join(packageDataStreamsPath, dataStreamName) + dataStreamOwners, found := codeowners.owners["/"+filepath.ToSlash(dataStreamDir)] if !found { dataStreamsWithoutOwner = append(dataStreamsWithoutOwner, dataStreamDir) continue diff --git a/dev/codeowners/codeowners_test.go b/dev/codeowners/codeowners_test.go index e84f218352d..8e9f62b8cd0 100644 --- a/dev/codeowners/codeowners_test.go +++ b/dev/codeowners/codeowners_test.go @@ -95,6 +95,31 @@ func TestValidatePackages(t *testing.T) { packageDir: "testdata/test_packages", valid: true, }, + { + codeownersPath: "testdata/CODEOWNERS-nested-valid", + packageDir: "testdata/nested_packages", + valid: true, + }, + { + codeownersPath: "testdata/CODEOWNERS-nested-missing-owner", + packageDir: "testdata/nested_packages", + valid: false, + }, + { + codeownersPath: "testdata/CODEOWNERS-nested-category-owner", + packageDir: "testdata/nested_packages", + valid: true, + }, + { + codeownersPath: "testdata/CODEOWNERS-nested-streams-valid", + packageDir: "testdata/nested_packages", + valid: true, + }, + { + codeownersPath: "testdata/CODEOWNERS-nested-streams-missing-owners", + packageDir: "testdata/nested_packages", + valid: false, + }, } for _, c := range cases { diff --git a/dev/codeowners/testdata/CODEOWNERS-nested-category-owner b/dev/codeowners/testdata/CODEOWNERS-nested-category-owner new file mode 100644 index 00000000000..db3cc9189cd --- /dev/null +++ b/dev/codeowners/testdata/CODEOWNERS-nested-category-owner @@ -0,0 +1,2 @@ +/testdata/nested_packages/package_top @elastic/integrations-developer-experience +/testdata/nested_packages/category @elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/CODEOWNERS-nested-missing-owner b/dev/codeowners/testdata/CODEOWNERS-nested-missing-owner new file mode 100644 index 00000000000..3ef5d7ca2f7 --- /dev/null +++ b/dev/codeowners/testdata/CODEOWNERS-nested-missing-owner @@ -0,0 +1,2 @@ +/testdata/nested_packages/package_top @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_1 @elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/CODEOWNERS-nested-streams-missing-owners b/dev/codeowners/testdata/CODEOWNERS-nested-streams-missing-owners new file mode 100644 index 00000000000..d70c2f93b82 --- /dev/null +++ b/dev/codeowners/testdata/CODEOWNERS-nested-streams-missing-owners @@ -0,0 +1,4 @@ +/testdata/nested_packages/package_top @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_1 @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_1/data_stream/stream_1 @pkoutsovasilis +/testdata/nested_packages/category/package_nested_2 @elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/CODEOWNERS-nested-streams-valid b/dev/codeowners/testdata/CODEOWNERS-nested-streams-valid new file mode 100644 index 00000000000..168f5b26054 --- /dev/null +++ b/dev/codeowners/testdata/CODEOWNERS-nested-streams-valid @@ -0,0 +1,5 @@ +/testdata/nested_packages/package_top @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_1 @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_1/data_stream/stream_1 @pkoutsovasilis +/testdata/nested_packages/category/package_nested_1/data_stream/stream_2 @pkoutsovasilis +/testdata/nested_packages/category/package_nested_2 @elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/CODEOWNERS-nested-valid b/dev/codeowners/testdata/CODEOWNERS-nested-valid new file mode 100644 index 00000000000..3674da6be52 --- /dev/null +++ b/dev/codeowners/testdata/CODEOWNERS-nested-valid @@ -0,0 +1,3 @@ +/testdata/nested_packages/package_top @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_1 @elastic/integrations-developer-experience +/testdata/nested_packages/category/package_nested_2 @elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/devexp/manifest.yml b/dev/codeowners/testdata/devexp/manifest.yml index 6e10c80be6e..f74ae81b82c 100644 --- a/dev/codeowners/testdata/devexp/manifest.yml +++ b/dev/codeowners/testdata/devexp/manifest.yml @@ -1,2 +1,6 @@ +format_version: 1.0.0 +name: devexp +type: integration +version: 1.0.0 owner: github: elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/nested_packages/category/package_nested_1/data_stream/stream_1/.keep b/dev/codeowners/testdata/nested_packages/category/package_nested_1/data_stream/stream_1/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dev/codeowners/testdata/nested_packages/category/package_nested_1/data_stream/stream_2/.keep b/dev/codeowners/testdata/nested_packages/category/package_nested_1/data_stream/stream_2/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dev/codeowners/testdata/nested_packages/category/package_nested_1/manifest.yml b/dev/codeowners/testdata/nested_packages/category/package_nested_1/manifest.yml new file mode 100644 index 00000000000..1e37f754abc --- /dev/null +++ b/dev/codeowners/testdata/nested_packages/category/package_nested_1/manifest.yml @@ -0,0 +1,6 @@ +format_version: 1.0.0 +name: package_nested_1 +type: integration +version: 1.0.0 +owner: + github: elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/nested_packages/category/package_nested_2/manifest.yml b/dev/codeowners/testdata/nested_packages/category/package_nested_2/manifest.yml new file mode 100644 index 00000000000..9f875919e7a --- /dev/null +++ b/dev/codeowners/testdata/nested_packages/category/package_nested_2/manifest.yml @@ -0,0 +1,6 @@ +format_version: 1.0.0 +name: package_nested_2 +type: integration +version: 1.0.0 +owner: + github: elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/nested_packages/package_top/manifest.yml b/dev/codeowners/testdata/nested_packages/package_top/manifest.yml new file mode 100644 index 00000000000..b963185633e --- /dev/null +++ b/dev/codeowners/testdata/nested_packages/package_top/manifest.yml @@ -0,0 +1,6 @@ +format_version: 1.0.0 +name: package_top +type: integration +version: 1.0.0 +owner: + github: elastic/integrations-developer-experience diff --git a/dev/codeowners/testdata/test_packages/package_1/manifest.yml b/dev/codeowners/testdata/test_packages/package_1/manifest.yml index 6e10c80be6e..1af7799981b 100644 --- a/dev/codeowners/testdata/test_packages/package_1/manifest.yml +++ b/dev/codeowners/testdata/test_packages/package_1/manifest.yml @@ -1,2 +1,6 @@ +format_version: 1.0.0 +name: package_1 +type: integration +version: 1.0.0 owner: github: elastic/integrations-developer-experience diff --git a/dev/packagenames/manifest.go b/dev/packagenames/manifest.go deleted file mode 100644 index b52a62c05b1..00000000000 --- a/dev/packagenames/manifest.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package packagenames - -import ( - "fmt" - "os" - "slices" - - "gopkg.in/yaml.v3" -) - -var validTypes = []string{"integration", "input", "content"} - -type packageManifest struct { - FormatVersion string `yaml:"format_version"` - Name string `yaml:"name"` - Type string `yaml:"type"` - Version string `yaml:"version"` -} - -func (m *packageManifest) isValid() bool { - if m.FormatVersion == "" || m.Name == "" || m.Type == "" || m.Version == "" { - return false - } - return slices.Contains(validTypes, m.Type) -} - -func readManifest(path string) (*packageManifest, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var m packageManifest - if err := yaml.Unmarshal(data, &m); err != nil { - return nil, fmt.Errorf("error parsing manifest: %w", err) - } - return &m, nil -} diff --git a/dev/packagenames/packagenames.go b/dev/packagenames/packagenames.go index 2e9b8bc22b9..2b6bed1c2e4 100644 --- a/dev/packagenames/packagenames.go +++ b/dev/packagenames/packagenames.go @@ -5,19 +5,16 @@ package packagenames import ( - "errors" "fmt" - "io/fs" "path/filepath" "slices" "strings" -) -const ( - packagesDir = "packages" - manifestFileName = "manifest.yml" + "github.com/elastic/integrations/dev/citools" ) +const packagesDir = "packages" + // Check validates that no two packages share the same name under the default packages directory. func Check() error { if err := checkPackageNames(packagesDir); err != nil { @@ -27,43 +24,17 @@ func Check() error { } func checkPackageNames(dir string) error { - paths, err := walkPackagePaths(dir) + paths, err := citools.ListPackages(dir) if err != nil { return fmt.Errorf("error finding packages: %w", err) } return checkDuplicateNames(paths) } -func walkPackagePaths(dir string) ([]string, error) { - var paths []string - err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - return nil - } - manifestPath := filepath.Join(path, manifestFileName) - manifest, err := readManifest(manifestPath) - if errors.Is(err, fs.ErrNotExist) { - return nil - } else if err != nil { - return fmt.Errorf("error reading manifest %s: %w", manifestPath, err) - } - if !manifest.isValid() { - return nil - } - paths = append(paths, path) - // No need to look deeper, we already found a package. - return filepath.SkipDir - }) - return paths, err -} - func checkDuplicateNames(paths []string) error { seen := make(map[string][]string) for _, path := range paths { - manifest, err := readManifest(filepath.Join(path, manifestFileName)) + manifest, err := citools.ReadPackageManifest(filepath.Join(path, citools.ManifestFileName)) if err != nil { return fmt.Errorf("error reading manifest in %s: %w", path, err) } diff --git a/dev/packagenames/packagenames_test.go b/dev/packagenames/packagenames_test.go index cf185835bfb..1c8312f3d8d 100644 --- a/dev/packagenames/packagenames_test.go +++ b/dev/packagenames/packagenames_test.go @@ -9,19 +9,22 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/integrations/dev/citools" ) func TestNoDuplicates(t *testing.T) { - paths, err := walkPackagePaths("testdata/no_duplicates") + paths, err := citools.ListPackages("testdata/no_duplicates") require.NoError(t, err) assert.Len(t, paths, 2) + t.Logf("paths: %v", paths) err = checkDuplicateNames(paths) assert.NoError(t, err) } func TestDuplicates(t *testing.T) { - paths, err := walkPackagePaths("testdata/duplicates") + paths, err := citools.ListPackages("testdata/duplicates") require.NoError(t, err) assert.Len(t, paths, 2) @@ -30,7 +33,7 @@ func TestDuplicates(t *testing.T) { } func TestInvalidManifestsIgnored(t *testing.T) { - paths, err := walkPackagePaths("testdata/invalid_manifests") + paths, err := citools.ListPackages("testdata/invalid_manifests") require.NoError(t, err) // p2 has no type field and must be ignored assert.Len(t, paths, 1) @@ -40,7 +43,7 @@ func TestInvalidManifestsIgnored(t *testing.T) { } func TestNestedNoDuplicates(t *testing.T) { - paths, err := walkPackagePaths("testdata/nested/no_duplicates") + paths, err := citools.ListPackages("testdata/nested/no_duplicates") require.NoError(t, err) // p3 at first level + technology/p1 and technology/p2 at second level assert.Len(t, paths, 3) @@ -50,7 +53,7 @@ func TestNestedNoDuplicates(t *testing.T) { } func TestNestedDuplicates(t *testing.T) { - paths, err := walkPackagePaths("testdata/nested/duplicates") + paths, err := citools.ListPackages("testdata/nested/duplicates") require.NoError(t, err) // p3 at first level + technology/p1 and technology/p2 at second level assert.Len(t, paths, 3) diff --git a/magefile.go b/magefile.go index cd9cd067055..7298e6e5870 100644 --- a/magefile.go +++ b/magefile.go @@ -218,6 +218,19 @@ func ReportFailedTests(ctx context.Context, testResultsFolder string) error { return testsreporter.Check(ctx, testResultsFolder, options) } +// ListPackages lists all packages found under the packages directory. +func ListPackages() error { + const packagesDir = "packages" + packages, err := citools.ListPackages(packagesDir) + if err != nil { + return fmt.Errorf("failed to list packages: %w", err) + } + for _, p := range packages { + fmt.Println(p) + } + return nil +} + // IsSubscriptionCompatible checks whether or not the package in the current directory allows to run with the given subscription (ELASTIC_SUBSCRIPTION env var). func IsSubscriptionCompatible() error { subscription := os.Getenv("ELASTIC_SUBSCRIPTION") diff --git a/packages/nginx/_dev/benchmark/rally/access-benchmark.yml b/packages/nginx/nginx/_dev/benchmark/rally/access-benchmark.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/access-benchmark.yml rename to packages/nginx/nginx/_dev/benchmark/rally/access-benchmark.yml diff --git a/packages/nginx/_dev/benchmark/rally/access-benchmark/config.yml b/packages/nginx/nginx/_dev/benchmark/rally/access-benchmark/config.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/access-benchmark/config.yml rename to packages/nginx/nginx/_dev/benchmark/rally/access-benchmark/config.yml diff --git a/packages/nginx/_dev/benchmark/rally/access-benchmark/fields.yml b/packages/nginx/nginx/_dev/benchmark/rally/access-benchmark/fields.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/access-benchmark/fields.yml rename to packages/nginx/nginx/_dev/benchmark/rally/access-benchmark/fields.yml diff --git a/packages/nginx/_dev/benchmark/rally/access-benchmark/template.ndjson b/packages/nginx/nginx/_dev/benchmark/rally/access-benchmark/template.ndjson similarity index 100% rename from packages/nginx/_dev/benchmark/rally/access-benchmark/template.ndjson rename to packages/nginx/nginx/_dev/benchmark/rally/access-benchmark/template.ndjson diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark.yml b/packages/nginx/nginx/_dev/benchmark/rally/error-benchmark.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/error-benchmark.yml rename to packages/nginx/nginx/_dev/benchmark/rally/error-benchmark.yml diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml b/packages/nginx/nginx/_dev/benchmark/rally/error-benchmark/config.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml rename to packages/nginx/nginx/_dev/benchmark/rally/error-benchmark/config.yml diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml b/packages/nginx/nginx/_dev/benchmark/rally/error-benchmark/fields.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml rename to packages/nginx/nginx/_dev/benchmark/rally/error-benchmark/fields.yml diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson b/packages/nginx/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson similarity index 100% rename from packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson rename to packages/nginx/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson diff --git a/packages/nginx/_dev/benchmark/rally/stubstatus-benchmark.yml b/packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/stubstatus-benchmark.yml rename to packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark.yml diff --git a/packages/nginx/_dev/benchmark/rally/stubstatus-benchmark/config.yml b/packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark/config.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/stubstatus-benchmark/config.yml rename to packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark/config.yml diff --git a/packages/nginx/_dev/benchmark/rally/stubstatus-benchmark/fields.yml b/packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark/fields.yml similarity index 100% rename from packages/nginx/_dev/benchmark/rally/stubstatus-benchmark/fields.yml rename to packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark/fields.yml diff --git a/packages/nginx/_dev/benchmark/rally/stubstatus-benchmark/template.ndjson b/packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark/template.ndjson similarity index 100% rename from packages/nginx/_dev/benchmark/rally/stubstatus-benchmark/template.ndjson rename to packages/nginx/nginx/_dev/benchmark/rally/stubstatus-benchmark/template.ndjson diff --git a/packages/nginx/_dev/build/build.yml b/packages/nginx/nginx/_dev/build/build.yml similarity index 100% rename from packages/nginx/_dev/build/build.yml rename to packages/nginx/nginx/_dev/build/build.yml diff --git a/packages/nginx/_dev/build/docs/README.md b/packages/nginx/nginx/_dev/build/docs/README.md similarity index 100% rename from packages/nginx/_dev/build/docs/README.md rename to packages/nginx/nginx/_dev/build/docs/README.md diff --git a/packages/nginx/_dev/deploy/docker/Dockerfile b/packages/nginx/nginx/_dev/deploy/docker/Dockerfile similarity index 100% rename from packages/nginx/_dev/deploy/docker/Dockerfile rename to packages/nginx/nginx/_dev/deploy/docker/Dockerfile diff --git a/packages/nginx/_dev/deploy/docker/docker-compose.yml b/packages/nginx/nginx/_dev/deploy/docker/docker-compose.yml similarity index 100% rename from packages/nginx/_dev/deploy/docker/docker-compose.yml rename to packages/nginx/nginx/_dev/deploy/docker/docker-compose.yml diff --git a/packages/nginx/_dev/deploy/docker/nginx.conf b/packages/nginx/nginx/_dev/deploy/docker/nginx.conf similarity index 100% rename from packages/nginx/_dev/deploy/docker/nginx.conf rename to packages/nginx/nginx/_dev/deploy/docker/nginx.conf diff --git a/packages/nginx/_dev/deploy/variants.yml b/packages/nginx/nginx/_dev/deploy/variants.yml similarity index 100% rename from packages/nginx/_dev/deploy/variants.yml rename to packages/nginx/nginx/_dev/deploy/variants.yml diff --git a/packages/nginx/changelog.yml b/packages/nginx/nginx/changelog.yml similarity index 100% rename from packages/nginx/changelog.yml rename to packages/nginx/nginx/changelog.yml diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-access.log similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-access.log diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json b/packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json rename to packages/nginx/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json diff --git a/packages/nginx/data_stream/access/_dev/test/system/test-default-config.yml b/packages/nginx/nginx/data_stream/access/_dev/test/system/test-default-config.yml similarity index 100% rename from packages/nginx/data_stream/access/_dev/test/system/test-default-config.yml rename to packages/nginx/nginx/data_stream/access/_dev/test/system/test-default-config.yml diff --git a/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs b/packages/nginx/nginx/data_stream/access/agent/stream/stream.yml.hbs similarity index 100% rename from packages/nginx/data_stream/access/agent/stream/stream.yml.hbs rename to packages/nginx/nginx/data_stream/access/agent/stream/stream.yml.hbs diff --git a/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/nginx/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml similarity index 100% rename from packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml rename to packages/nginx/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/nginx/data_stream/access/fields/agent.yml b/packages/nginx/nginx/data_stream/access/fields/agent.yml similarity index 100% rename from packages/nginx/data_stream/access/fields/agent.yml rename to packages/nginx/nginx/data_stream/access/fields/agent.yml diff --git a/packages/nginx/data_stream/access/fields/base-fields.yml b/packages/nginx/nginx/data_stream/access/fields/base-fields.yml similarity index 100% rename from packages/nginx/data_stream/access/fields/base-fields.yml rename to packages/nginx/nginx/data_stream/access/fields/base-fields.yml diff --git a/packages/nginx/data_stream/access/fields/fields.yml b/packages/nginx/nginx/data_stream/access/fields/fields.yml similarity index 100% rename from packages/nginx/data_stream/access/fields/fields.yml rename to packages/nginx/nginx/data_stream/access/fields/fields.yml diff --git a/packages/nginx/data_stream/access/manifest.yml b/packages/nginx/nginx/data_stream/access/manifest.yml similarity index 100% rename from packages/nginx/data_stream/access/manifest.yml rename to packages/nginx/nginx/data_stream/access/manifest.yml diff --git a/packages/nginx/data_stream/access/sample_event.json b/packages/nginx/nginx/data_stream/access/sample_event.json similarity index 100% rename from packages/nginx/data_stream/access/sample_event.json rename to packages/nginx/nginx/data_stream/access/sample_event.json diff --git a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log b/packages/nginx/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log similarity index 100% rename from packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log rename to packages/nginx/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log diff --git a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml b/packages/nginx/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml similarity index 100% rename from packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml rename to packages/nginx/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml diff --git a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json b/packages/nginx/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json similarity index 100% rename from packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json rename to packages/nginx/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json diff --git a/packages/nginx/data_stream/error/_dev/test/system/test-default-config.yml b/packages/nginx/nginx/data_stream/error/_dev/test/system/test-default-config.yml similarity index 100% rename from packages/nginx/data_stream/error/_dev/test/system/test-default-config.yml rename to packages/nginx/nginx/data_stream/error/_dev/test/system/test-default-config.yml diff --git a/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs b/packages/nginx/nginx/data_stream/error/agent/stream/stream.yml.hbs similarity index 100% rename from packages/nginx/data_stream/error/agent/stream/stream.yml.hbs rename to packages/nginx/nginx/data_stream/error/agent/stream/stream.yml.hbs diff --git a/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/packages/nginx/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml similarity index 100% rename from packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml rename to packages/nginx/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/nginx/data_stream/error/fields/agent.yml b/packages/nginx/nginx/data_stream/error/fields/agent.yml similarity index 100% rename from packages/nginx/data_stream/error/fields/agent.yml rename to packages/nginx/nginx/data_stream/error/fields/agent.yml diff --git a/packages/nginx/data_stream/error/fields/base-fields.yml b/packages/nginx/nginx/data_stream/error/fields/base-fields.yml similarity index 100% rename from packages/nginx/data_stream/error/fields/base-fields.yml rename to packages/nginx/nginx/data_stream/error/fields/base-fields.yml diff --git a/packages/nginx/data_stream/error/fields/fields.yml b/packages/nginx/nginx/data_stream/error/fields/fields.yml similarity index 100% rename from packages/nginx/data_stream/error/fields/fields.yml rename to packages/nginx/nginx/data_stream/error/fields/fields.yml diff --git a/packages/nginx/data_stream/error/manifest.yml b/packages/nginx/nginx/data_stream/error/manifest.yml similarity index 100% rename from packages/nginx/data_stream/error/manifest.yml rename to packages/nginx/nginx/data_stream/error/manifest.yml diff --git a/packages/nginx/data_stream/error/sample_event.json b/packages/nginx/nginx/data_stream/error/sample_event.json similarity index 100% rename from packages/nginx/data_stream/error/sample_event.json rename to packages/nginx/nginx/data_stream/error/sample_event.json diff --git a/packages/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml b/packages/nginx/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml similarity index 100% rename from packages/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml rename to packages/nginx/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml diff --git a/packages/nginx/data_stream/stubstatus/agent/stream/stream.yml.hbs b/packages/nginx/nginx/data_stream/stubstatus/agent/stream/stream.yml.hbs similarity index 100% rename from packages/nginx/data_stream/stubstatus/agent/stream/stream.yml.hbs rename to packages/nginx/nginx/data_stream/stubstatus/agent/stream/stream.yml.hbs diff --git a/packages/nginx/data_stream/stubstatus/fields/agent.yml b/packages/nginx/nginx/data_stream/stubstatus/fields/agent.yml similarity index 100% rename from packages/nginx/data_stream/stubstatus/fields/agent.yml rename to packages/nginx/nginx/data_stream/stubstatus/fields/agent.yml diff --git a/packages/nginx/data_stream/stubstatus/fields/base-fields.yml b/packages/nginx/nginx/data_stream/stubstatus/fields/base-fields.yml similarity index 100% rename from packages/nginx/data_stream/stubstatus/fields/base-fields.yml rename to packages/nginx/nginx/data_stream/stubstatus/fields/base-fields.yml diff --git a/packages/nginx/data_stream/stubstatus/fields/ecs.yml b/packages/nginx/nginx/data_stream/stubstatus/fields/ecs.yml similarity index 100% rename from packages/nginx/data_stream/stubstatus/fields/ecs.yml rename to packages/nginx/nginx/data_stream/stubstatus/fields/ecs.yml diff --git a/packages/nginx/data_stream/stubstatus/fields/fields.yml b/packages/nginx/nginx/data_stream/stubstatus/fields/fields.yml similarity index 100% rename from packages/nginx/data_stream/stubstatus/fields/fields.yml rename to packages/nginx/nginx/data_stream/stubstatus/fields/fields.yml diff --git a/packages/nginx/data_stream/stubstatus/manifest.yml b/packages/nginx/nginx/data_stream/stubstatus/manifest.yml similarity index 100% rename from packages/nginx/data_stream/stubstatus/manifest.yml rename to packages/nginx/nginx/data_stream/stubstatus/manifest.yml diff --git a/packages/nginx/data_stream/stubstatus/sample_event.json b/packages/nginx/nginx/data_stream/stubstatus/sample_event.json similarity index 100% rename from packages/nginx/data_stream/stubstatus/sample_event.json rename to packages/nginx/nginx/data_stream/stubstatus/sample_event.json diff --git a/packages/nginx/docs/README.md b/packages/nginx/nginx/docs/README.md similarity index 100% rename from packages/nginx/docs/README.md rename to packages/nginx/nginx/docs/README.md diff --git a/packages/nginx/img/logo_nginx.svg b/packages/nginx/nginx/img/logo_nginx.svg similarity index 100% rename from packages/nginx/img/logo_nginx.svg rename to packages/nginx/nginx/img/logo_nginx.svg diff --git a/packages/nginx/img/nginx-logs-access-error.png b/packages/nginx/nginx/img/nginx-logs-access-error.png similarity index 100% rename from packages/nginx/img/nginx-logs-access-error.png rename to packages/nginx/nginx/img/nginx-logs-access-error.png diff --git a/packages/nginx/img/nginx-logs-overview.png b/packages/nginx/nginx/img/nginx-logs-overview.png similarity index 100% rename from packages/nginx/img/nginx-logs-overview.png rename to packages/nginx/nginx/img/nginx-logs-overview.png diff --git a/packages/nginx/img/nginx-metrics-overview.png b/packages/nginx/nginx/img/nginx-metrics-overview.png similarity index 100% rename from packages/nginx/img/nginx-metrics-overview.png rename to packages/nginx/nginx/img/nginx-metrics-overview.png diff --git a/packages/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json b/packages/nginx/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json similarity index 100% rename from packages/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json rename to packages/nginx/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json diff --git a/packages/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json b/packages/nginx/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json similarity index 100% rename from packages/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json rename to packages/nginx/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json diff --git a/packages/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json b/packages/nginx/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json similarity index 100% rename from packages/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json rename to packages/nginx/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json diff --git a/packages/nginx/kibana/ml_module/nginx-Logs-ml.json b/packages/nginx/nginx/kibana/ml_module/nginx-Logs-ml.json similarity index 100% rename from packages/nginx/kibana/ml_module/nginx-Logs-ml.json rename to packages/nginx/nginx/kibana/ml_module/nginx-Logs-ml.json diff --git a/packages/nginx/manifest.yml b/packages/nginx/nginx/manifest.yml similarity index 100% rename from packages/nginx/manifest.yml rename to packages/nginx/nginx/manifest.yml diff --git a/packages/nginx_otel/_dev/shared/kibana/01-request-health.yaml b/packages/nginx/nginx_otel/_dev/shared/kibana/01-request-health.yaml similarity index 100% rename from packages/nginx_otel/_dev/shared/kibana/01-request-health.yaml rename to packages/nginx/nginx_otel/_dev/shared/kibana/01-request-health.yaml diff --git a/packages/nginx_otel/_dev/shared/kibana/02-server-internals.yaml b/packages/nginx/nginx_otel/_dev/shared/kibana/02-server-internals.yaml similarity index 100% rename from packages/nginx_otel/_dev/shared/kibana/02-server-internals.yaml rename to packages/nginx/nginx_otel/_dev/shared/kibana/02-server-internals.yaml diff --git a/packages/nginx_otel/_dev/shared/kibana/03-traffic-capacity.yaml b/packages/nginx/nginx_otel/_dev/shared/kibana/03-traffic-capacity.yaml similarity index 100% rename from packages/nginx_otel/_dev/shared/kibana/03-traffic-capacity.yaml rename to packages/nginx/nginx_otel/_dev/shared/kibana/03-traffic-capacity.yaml diff --git a/packages/nginx_otel/changelog.yml b/packages/nginx/nginx_otel/changelog.yml similarity index 100% rename from packages/nginx_otel/changelog.yml rename to packages/nginx/nginx_otel/changelog.yml diff --git a/packages/nginx_otel/docs/README.md b/packages/nginx/nginx_otel/docs/README.md similarity index 100% rename from packages/nginx_otel/docs/README.md rename to packages/nginx/nginx_otel/docs/README.md diff --git a/packages/nginx_otel/img/logo_nginx.svg b/packages/nginx/nginx_otel/img/logo_nginx.svg similarity index 100% rename from packages/nginx_otel/img/logo_nginx.svg rename to packages/nginx/nginx_otel/img/logo_nginx.svg diff --git a/packages/nginx_otel/img/nginx-request-health.png b/packages/nginx/nginx_otel/img/nginx-request-health.png similarity index 100% rename from packages/nginx_otel/img/nginx-request-health.png rename to packages/nginx/nginx_otel/img/nginx-request-health.png diff --git a/packages/nginx_otel/img/nginx-server-internals.png b/packages/nginx/nginx_otel/img/nginx-server-internals.png similarity index 100% rename from packages/nginx_otel/img/nginx-server-internals.png rename to packages/nginx/nginx_otel/img/nginx-server-internals.png diff --git a/packages/nginx_otel/img/nginx-traffic-capacity.png b/packages/nginx/nginx_otel/img/nginx-traffic-capacity.png similarity index 100% rename from packages/nginx_otel/img/nginx-traffic-capacity.png rename to packages/nginx/nginx_otel/img/nginx-traffic-capacity.png diff --git a/packages/nginx_otel/img/nginx_otel_logo.svg b/packages/nginx/nginx_otel/img/nginx_otel_logo.svg similarity index 100% rename from packages/nginx_otel/img/nginx_otel_logo.svg rename to packages/nginx/nginx_otel/img/nginx_otel_logo.svg diff --git a/packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-dropped-connections.json b/packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-dropped-connections.json similarity index 100% rename from packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-dropped-connections.json rename to packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-dropped-connections.json diff --git a/packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-error-log-spike.json b/packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-error-log-spike.json similarity index 100% rename from packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-error-log-spike.json rename to packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-error-log-spike.json diff --git a/packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-4xx-error-rate.json b/packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-4xx-error-rate.json similarity index 100% rename from packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-4xx-error-rate.json rename to packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-4xx-error-rate.json diff --git a/packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-5xx-error-rate.json b/packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-5xx-error-rate.json similarity index 100% rename from packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-5xx-error-rate.json rename to packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-5xx-error-rate.json diff --git a/packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-active-connections.json b/packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-active-connections.json similarity index 100% rename from packages/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-active-connections.json rename to packages/nginx/nginx_otel/kibana/alerting_rule_template/nginx_otel-high-active-connections.json diff --git a/packages/nginx_otel/kibana/dashboard/nginx_otel-a1b2c3d4-e5f6-7890-abcd-ef1234567890.json b/packages/nginx/nginx_otel/kibana/dashboard/nginx_otel-a1b2c3d4-e5f6-7890-abcd-ef1234567890.json similarity index 100% rename from packages/nginx_otel/kibana/dashboard/nginx_otel-a1b2c3d4-e5f6-7890-abcd-ef1234567890.json rename to packages/nginx/nginx_otel/kibana/dashboard/nginx_otel-a1b2c3d4-e5f6-7890-abcd-ef1234567890.json diff --git a/packages/nginx_otel/kibana/dashboard/nginx_otel-b2c3d4e5-f6a7-8901-bcde-f12345678901.json b/packages/nginx/nginx_otel/kibana/dashboard/nginx_otel-b2c3d4e5-f6a7-8901-bcde-f12345678901.json similarity index 100% rename from packages/nginx_otel/kibana/dashboard/nginx_otel-b2c3d4e5-f6a7-8901-bcde-f12345678901.json rename to packages/nginx/nginx_otel/kibana/dashboard/nginx_otel-b2c3d4e5-f6a7-8901-bcde-f12345678901.json diff --git a/packages/nginx_otel/kibana/dashboard/nginx_otel-c3d4e5f6-a7b8-9012-cdef-123456789012.json b/packages/nginx/nginx_otel/kibana/dashboard/nginx_otel-c3d4e5f6-a7b8-9012-cdef-123456789012.json similarity index 100% rename from packages/nginx_otel/kibana/dashboard/nginx_otel-c3d4e5f6-a7b8-9012-cdef-123456789012.json rename to packages/nginx/nginx_otel/kibana/dashboard/nginx_otel-c3d4e5f6-a7b8-9012-cdef-123456789012.json diff --git a/packages/nginx_otel/kibana/slo_template/nginx_otel-connection-drop-rate-99.5-Rolling30Days.json b/packages/nginx/nginx_otel/kibana/slo_template/nginx_otel-connection-drop-rate-99.5-Rolling30Days.json similarity index 100% rename from packages/nginx_otel/kibana/slo_template/nginx_otel-connection-drop-rate-99.5-Rolling30Days.json rename to packages/nginx/nginx_otel/kibana/slo_template/nginx_otel-connection-drop-rate-99.5-Rolling30Days.json diff --git a/packages/nginx_otel/kibana/slo_template/nginx_otel-request-availability-99-Rolling30Days.json b/packages/nginx/nginx_otel/kibana/slo_template/nginx_otel-request-availability-99-Rolling30Days.json similarity index 100% rename from packages/nginx_otel/kibana/slo_template/nginx_otel-request-availability-99-Rolling30Days.json rename to packages/nginx/nginx_otel/kibana/slo_template/nginx_otel-request-availability-99-Rolling30Days.json diff --git a/packages/nginx_otel/manifest.yml b/packages/nginx/nginx_otel/manifest.yml similarity index 100% rename from packages/nginx_otel/manifest.yml rename to packages/nginx/nginx_otel/manifest.yml