diff --git a/.gitattributes b/.gitattributes index 87528b50b..d9a3a6d28 100644 --- a/.gitattributes +++ b/.gitattributes @@ -50,4 +50,5 @@ *.dbproj text=auto *.sln text=auto eol=crlf -*.sh eol=lf \ No newline at end of file +*.sh eol=lf +Dockerfile text eol=lf \ No newline at end of file diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml new file mode 100644 index 000000000..fba343d45 --- /dev/null +++ b/.github/workflows/ghcr.yml @@ -0,0 +1,45 @@ +name: Build and push image to GHCR + +on: + push: + branches: [main] + tags: ['v*.*.*'] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/${{ github.repository_owner }}/bagetter + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,format=short + type=semver,pattern={{version}} + + - name: Set up Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build and push + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release-helm.yml b/.github/workflows/release-helm.yml index 9074966c9..5ad15537c 100644 --- a/.github/workflows/release-helm.yml +++ b/.github/workflows/release-helm.yml @@ -1,9 +1,19 @@ -name: Release-Helm - -on: - workflow_dispatch: - inputs: - tags: - description: 'Tags' - required: true - type: string +name: Release-Helm + +on: + workflow_dispatch: + inputs: + tags: + description: 'Tags' + required: true + type: string + +jobs: + release-helm: + name: Release Helm chart + runs-on: ubuntu-latest + steps: + - name: Show release workflow + run: | + echo "Helm chart releases are handled by .github/workflows/release.yml on version tags." + echo "Requested tags: ${{ inputs.tags }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e395ceb76..32623b18e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,23 +4,52 @@ on: push: tags: - 'v*.*.*' + workflow_dispatch: + inputs: + tag: + description: 'Release tag to create (for example, v1.2.3)' + required: true + type: string + workflow_call: + inputs: + tag: + description: 'Release tag to create (for example, v1.2.3)' + required: true + type: string jobs: version: runs-on: ubuntu-latest steps: - id: get-version-tag + env: + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_REF_TYPE: ${{ github.ref_type }} + GITHUB_SHA: ${{ github.sha }} + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} run: | - export GITHUB_REF_NAME="${{ github.ref_name }}" - export RELEASE_VERSION="${GITHUB_REF_NAME#v}" - export RELEASE_MAJOR_VERSION="${RELEASE_VERSION%%.*}" - export remaining_version="${RELEASE_VERSION#*.}" - export RELEASE_MINOR_VERSION="${RELEASE_MAJOR_VERSION}.${remaining_version%%.*}" - - echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT - echo "RELEASE_MINOR_VERSION=$RELEASE_MINOR_VERSION" >> $GITHUB_OUTPUT - echo "RELEASE_MAJOR_VERSION=$RELEASE_MAJOR_VERSION" >> $GITHUB_OUTPUT + if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: $RELEASE_TAG" >&2 + exit 1 + fi + + RELEASE_VERSION="${RELEASE_TAG#v}" + RELEASE_MAJOR_VERSION="${RELEASE_VERSION%%.*}" + remaining_version="${RELEASE_VERSION#*.}" + RELEASE_MINOR_VERSION="${RELEASE_MAJOR_VERSION}.${remaining_version%%.*}" + SOURCE_REF="$GITHUB_SHA" + if [[ "$GITHUB_REF_TYPE" == "tag" && "$GITHUB_REF_NAME" == "$RELEASE_TAG" ]]; then + SOURCE_REF="$RELEASE_TAG" + fi + + echo "RELEASE_TAG=$RELEASE_TAG" >> "$GITHUB_OUTPUT" + echo "SOURCE_REF=$SOURCE_REF" >> "$GITHUB_OUTPUT" + echo "RELEASE_VERSION=$RELEASE_VERSION" >> "$GITHUB_OUTPUT" + echo "RELEASE_MINOR_VERSION=$RELEASE_MINOR_VERSION" >> "$GITHUB_OUTPUT" + echo "RELEASE_MAJOR_VERSION=$RELEASE_MAJOR_VERSION" >> "$GITHUB_OUTPUT" outputs: + RELEASE_TAG: ${{ steps.get-version-tag.outputs.RELEASE_TAG }} + SOURCE_REF: ${{ steps.get-version-tag.outputs.SOURCE_REF }} RELEASE_VERSION: ${{ steps.get-version-tag.outputs.RELEASE_VERSION }} RELEASE_MINOR_VERSION: ${{ steps.get-version-tag.outputs.RELEASE_MINOR_VERSION }} RELEASE_MAJOR_VERSION: ${{ steps.get-version-tag.outputs.RELEASE_MAJOR_VERSION }} @@ -30,190 +59,115 @@ jobs: runs-on: ubuntu-latest needs: version steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v5 + with: + ref: ${{ needs.version.outputs.SOURCE_REF }} - name: Setup .NET - uses: actions/setup-dotnet@v4.0.0 + uses: actions/setup-dotnet@v6.0.0 with: global-json-file: global.json - name: Test run: dotnet test --verbosity normal release_zip: - if: ${{ github.ref_type == 'tag' }} - needs: [version, verify] + if: ${{ github.repository == 'oahsiao/M2_BaGetter' && (github.ref_type == 'tag' || inputs.tag != '') }} + needs: [version, verify, release_docker_image] name: Release BaGetter.zip to GitHub runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v5 with: fetch-depth: 0 + ref: ${{ needs.version.outputs.SOURCE_REF }} - name: Setup .NET - uses: actions/setup-dotnet@v4.0.0 + uses: actions/setup-dotnet@v6.0.0 with: global-json-file: global.json - name: Publish run: | dotnet publish src/BaGetter --configuration Release --output artifacts --property:Version=${{ needs.version.outputs.RELEASE_VERSION }} - echo ${{ github.sha }} > artifacts/ReleaseSha.txt + echo "${{ github.sha }}" > artifacts/ReleaseSha.txt 7z a -tzip bagetter-${{ needs.version.outputs.RELEASE_VERSION }}.zip ./artifacts/* + - name: Download Docker image archives + uses: actions/download-artifact@v8.0.1 + with: + name: docker-images + path: docker-images - name: Generate changelog with git-cliff uses: tj-actions/git-cliff@v2.2.0 with: args: --latest --strip all - output: "CHANGELOG.md" + output: CHANGELOG.md - name: Create release with ncipollo/release-action - uses: ncipollo/release-action@v1.14.0 + uses: ncipollo/release-action@v1.21.0 with: - bodyFile: "CHANGELOG.md" - artifacts: "bagetter-${{ needs.version.outputs.RELEASE_VERSION }}.zip" + bodyFile: CHANGELOG.md + artifacts: bagetter-${{ needs.version.outputs.RELEASE_VERSION }}.zip,docker-images/*.docker.tar.gz + commit: ${{ github.sha }} name: ${{ needs.version.outputs.RELEASE_VERSION }} - prerelease: ${{ contains(github.ref_name, '-') }} - - release_packages: - if: ${{ github.ref_type == 'tag' }} - needs: [version, verify] - name: Release packages to nuget.org - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.1 - - name: Setup .NET - uses: actions/setup-dotnet@v4.0.0 - with: - global-json-file: global.json - - name: Pack - run: | - export PackageVersion=${{ needs.version.outputs.RELEASE_VERSION }} - export PackageSource=${PackageSource:="https://api.nuget.org/v3/index.json"} - echo "PackageSource=${PackageSource}" >> $GITHUB_ENV - dotnet pack --configuration Release --output artifacts --property:Version=${{ needs.version.outputs.RELEASE_VERSION }} - - name: Push - run: dotnet nuget push "*" -s ${PackageSource} -k ${{secrets.NUGET_API_KEY}} - working-directory: artifacts + tag: ${{ needs.version.outputs.RELEASE_TAG }} + allow_updates: true + prerelease: ${{ contains(needs.version.outputs.RELEASE_VERSION, '-') }} release_docker_image: - if: ${{ github.ref_type == 'tag' }} + if: ${{ github.repository == 'oahsiao/M2_BaGetter' && (github.ref_type == 'tag' || inputs.tag != '') }} needs: [version, verify] - name: Release Docker image to Docker Hub + name: Release Docker image to GitHub Container Registry runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - - uses: actions/checkout@v4.1.1 - - name: Login to Docker Hub - uses: docker/login-action@v3.0.0 + - uses: actions/checkout@v5 + with: + ref: ${{ needs.version.outputs.SOURCE_REF }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v4.6.0 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set release version - run: echo "PackageVersion=${{ needs.version.outputs.RELEASE_VERSION }}" >> $GITHUB_ENV + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.4.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4.4.1 + - name: Set Docker metadata + id: metadata + uses: docker/metadata-action@v6.2.0 + with: + images: ghcr.io/oahsiao/bagetter + tags: | + type=raw,value=${{ needs.version.outputs.RELEASE_VERSION }} + type=raw,value=${{ needs.version.outputs.RELEASE_MINOR_VERSION }},enable=${{ !contains(needs.version.outputs.RELEASE_VERSION, '-') }} + type=raw,value=${{ needs.version.outputs.RELEASE_MAJOR_VERSION }},enable=${{ !contains(needs.version.outputs.RELEASE_VERSION, '-') }} + type=raw,value=latest,enable=${{ !contains(needs.version.outputs.RELEASE_VERSION, '-') }} - name: Build and push Docker image - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v7.4.0 with: context: . platforms: linux/amd64,linux/arm64 push: true - tags: | - bagetter/bagetter:latest - bagetter/bagetter:${{ needs.version.outputs.RELEASE_VERSION }} - bagetter/bagetter:${{ needs.version.outputs.RELEASE_MINOR_VERSION }} - bagetter/bagetter:${{ needs.version.outputs.RELEASE_MAJOR_VERSION }} + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} build-args: | Version=${{ needs.version.outputs.RELEASE_VERSION }} - - release_helm_chart: - if: ${{ github.ref_type == 'tag' }} - needs: [version, verify] - name: Release Helm chart to GitHub - runs-on: ubuntu-latest - env: - CHART_DIR: deployment templates/chart - CHART_REPO: bagetter/helm-chart-repo - CHART_REPO_BRANCH: gh-pages - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - steps: - - uses: actions/checkout@v4.1.1 - - name: Update version in helm chart + - name: Export Docker image archives + env: + IMAGE: ghcr.io/oahsiao/bagetter:${{ needs.version.outputs.RELEASE_VERSION }} + VERSION: ${{ needs.version.outputs.RELEASE_VERSION }} run: | - sed -i "s/version: \".*\"/version: \"${{ needs.version.outputs.RELEASE_VERSION }}\"/" "deployment templates/chart/bagetter/Chart.yaml" - sed -i "s/appVersion: \".*\"/appVersion: \"${{ needs.version.outputs.RELEASE_VERSION }}\"/" "deployment templates/chart/bagetter/Chart.yaml" - - name: Generate helm docs - uses: losisin/helm-docs-github-action@v1 + set -euo pipefail + mkdir -p docker-images + docker pull --platform linux/amd64 "$IMAGE" + docker image save "$IMAGE" | gzip > "docker-images/bagetter-${VERSION}-linux-amd64.docker.tar.gz" + docker pull --platform linux/arm64 "$IMAGE" + docker image save "$IMAGE" | gzip > "docker-images/bagetter-${VERSION}-linux-arm64.docker.tar.gz" + - name: Upload Docker image archives + uses: actions/upload-artifact@v7.0.1 with: - chart-search-root: "deployment templates" - - - name: Install yq - portable yaml processor - uses: mikefarah/yq@v4.44.3 - - - name: Collect charts - id: charts - run: | - set -e - find -L '${{ env.CHART_DIR }}' -mindepth 2 -maxdepth 2 -type f \( -name 'Chart.yaml' -o -name 'Chart.yml' \) -exec dirname "{}" \; \ - | sort -u \ - | sed -E 's/^/- /' \ - | yq --no-colors --indent 0 --output-format json '.' \ - | sed -E 's/^/charts=/' >> $GITHUB_OUTPUT - - - name: Install chart releaser - run: | - set -e - arch="$(dpkg --print-architecture)" - curl -s https://api.github.com/repos/helm/chart-releaser/releases/latest \ - | yq --indent 0 --no-colors --input-format json --unwrapScalar \ - ".assets[] | select(.name | test("\""^chart-releaser_.+_linux_${arch}\.tar\.gz$"\"")) | .browser_download_url" \ - | xargs curl -SsL \ - | tar zxf - -C /usr/local/bin - - - name: Install Helm - uses: azure/setup-helm@v4.2.0 - - - name: Helm Deps - run: | - set -ex - echo '${{ steps.charts.outputs.charts }}' \ - | yq --indent 0 --no-colors --input-format json --unwrapScalar '.[]' \ - | while read -r dir; do - helm dependency update "$dir"; - if [ -f "$dir/Chart.lock" ]; then - yq --indent 0 \ - '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' \ - "$dir/Chart.lock" \ - | sh --; - fi - done - - - name: Package charts - id: package - run: | - set -ex - PACKAGES=.cr-release-packages - echo '${{ steps.charts.outputs.charts }}' \ - | yq --indent 0 --no-colors --input-format json --unwrapScalar '.[]' \ - | xargs -d$'\n' cr package --package-path "$PACKAGES" - echo "dir=${PACKAGES}" >> $GITHUB_OUTPUT - - - name: Upload packages - run: | - set -ex - git config --list - owner=$(cut -d '/' -f 1 <<< '${{ env.CHART_REPO }}') - repo=$(cut -d '/' -f 2 <<< '${{ env.CHART_REPO }}') - cr upload --commit '${{ github.sha }}' --git-repo "$repo" --owner "$owner" --token '${{ github.token }}' \ - --package-path '${{ steps.package.outputs.dir }}' --skip-existing - - - name: Update charts index - working-directory: .helm-chart-repo - run: | - set -ex - git config --local user.name "$GITHUB_ACTOR" - git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" - mkdir -p .cr-index - owner=$(cut -d '/' -f 1 <<< '${{ env.CHART_REPO }}') - repo=$(cut -d '/' -f 2 <<< '${{ env.CHART_REPO }}') - cr index --git-repo "$repo" --owner "$owner" --pages-branch '${{ env.CHART_REPO_BRANCH }}' \ - --package-path '../${{ steps.package.outputs.dir }}' \ - --index-path .cr-index --push + name: docker-images + path: docker-images/*.docker.tar.gz + if-no-files-found: error + retention-days: 1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 828b85b53..2adad88de 100644 --- a/.gitignore +++ b/.gitignore @@ -219,6 +219,9 @@ UpgradeLog*.htm *.mdf *.ldf +# Runtime administrative audit logs +**/App_Data/ + # Business Intelligence projects *.rdl.data *.bim.layout diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..78553a4e4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,44 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `src/` contains all production projects. Key modules are `src/BaGetter` (host app), `src/BaGetter.Core` (domain/services), `src/BaGetter.Web` (NuGet API + UI), and `src/BaGetter.Protocol` (client SDK). +- Provider-specific integrations live in `src/BaGetter.Database.*` and `src/BaGetter.{Aws,Azure,Gcp,Aliyun,Tencent}`. +- `tests/` mirrors production areas: `BaGetter.Tests` (integration), `BaGetter.Core.Tests`, `BaGetter.Protocol.Tests`, and `BaGetter.Web.Tests`. +- `docs/` is the Docusaurus documentation site; `deployment templates/` contains Helm chart assets. + +## Build, Test, and Development Commands +- `dotnet restore` restores solution dependencies. +- `dotnet build --no-restore` compiles all projects (same pattern as CI). +- `dotnet test --no-build --verbosity normal` runs all tests. +- `dotnet run --project src/BaGetter` starts the local server. +- `docker build -t bagetter:local .` builds the container image. +- Docs site (from `docs/`): `yarn`, `yarn start`, `yarn build`. + +## Coding Style & Naming Conventions +- Follow `.editorconfig`: UTF-8, CRLF, trim trailing whitespace, final newline. +- Indentation: 4 spaces for C# and most files; 2 spaces for JSON/YAML/XML/web assets. +- C# conventions enforced by analyzers: file-scoped namespaces, braces preferred, `var` preferred when reasonable, `using` outside namespace. +- Naming: use PascalCase for types, members, and `const` fields. + +## Testing Guidelines +- Test stack: xUnit + `Microsoft.NET.Test.Sdk` + `coverlet.collector` (centrally managed in `Directory.Packages.props`). +- Name test files by subject with `Tests`/`Facts` suffix (for example, `PackageServiceTests.cs`, `IndexModelFacts.cs`). +- Keep integration scenarios in `tests/BaGetter.Tests` and focused unit tests in project-specific test folders. + +## Commit & Pull Request Guidelines +- Prefer Conventional Commit-style prefixes already used in history: `fix:`, `docs:`, `chore:`, `feat:`. +- Keep commit messages imperative and scoped to a single change. +- PRs should include: clear summary, linked issue (`#123` when applicable), test evidence (`dotnet test` output), and screenshots for UI/documentation updates. +- Ensure CI passes on both Linux and Windows runners before merge. + +## Security & Configuration Tips +- Do not commit secrets; use environment variables and local `appsettings.*` overrides. +- Keep `nuget.config` and feed credentials out of source control. +- LDAP `BindPassword` must remain a retrievable secret, not a one-way hash. It should be supplied through environment variables, secret files, or another secret store because LDAP bind requires the original password at runtime. +- LDAP uses `System.DirectoryServices.Protocols`, which is cross-platform. When changing LDAP behavior, preserve non-Windows compatibility and document any OS-level LDAPS certificate trust requirements for Linux/macOS deployments. + +## MCP & Documentation Lookup +- Prefer MCP Context7 when the user asks about library/framework docs, API usage, or code examples. +- Use Context7 proactively when it can improve solution quality or reduce ambiguity, even if not explicitly requested. +- Resolve the library ID first, then query docs with focused questions. +- Treat Context7 as the primary source for usage guidance before falling back to memory. diff --git a/Directory.Packages.props b/Directory.Packages.props index 5d7200bed..bd0727338 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,35 +8,36 @@ - - - - - - - - - - + + + + + + + + + + - - + + + - + - - - - + + + + @@ -44,7 +45,8 @@ - + + diff --git a/Dockerfile b/Dockerfile index cf5a988fb..18c79cd3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,65 @@ -ARG Version=1.0.0 - -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-alpine AS build -ARG Version -ARG TARGETARCH -WORKDIR /src - -## Create separate layer for `dotnet restore` to allow for caching; useful for local development -# copy only necessary files -COPY ./Directory.Packages.props ./nuget.config ./src/**/*.csproj ./ -# create folder structure that was lost when copying -RUN for file in $(ls *.csproj); do mkdir -p ${file%.*}/ && mv $file ${file%.*}/; done -# useful for debugging to display all files -#RUN echo $(ls) -# restore packages -RUN dotnet restore BaGetter/BaGetter.csproj --arch $TARGETARCH - -## Publish app (implicitly builds the app) -FROM build AS publish -ARG Version -# copy all files -COPY /src . -RUN dotnet publish BaGetter \ - --configuration Release \ - --output /app \ - --no-restore \ - -p Version=${Version} \ - -p DebugType=none \ - -p DebugSymbols=false \ - -p GenerateDocumentationFile=false \ - -p UseAppHost=false \ - -a $TARGETARCH - -# create default folders -RUN mkdir -p "/data/packages" \ - mkdir -p "/data/symbols" \ - mkdir -p "/data/db" - -## Create final image -FROM mcr.microsoft.com/dotnet/aspnet:9.0-alpine AS base -# install cultures (same approach as Alpine SDK image) -RUN apk add --no-cache icu-libs icu-data-full tzdata -# disable the invariant mode (set in base image) -ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false -# set default configurations; use the `/data` folder for packages, symbols and the SQLite database -ENV Storage__Path "/data" -ENV Search__Type "Database" -ENV Database__Type "Sqlite" -ENV Database__ConnectionString "Data Source=/data/db/bagetter.db" -LABEL org.opencontainers.image.source="https://github.com/bagetter/BaGetter" -# copy default folders -COPY --from=publish /data /data -# copy the published app -WORKDIR /app -COPY --from=publish /app . - -ENTRYPOINT ["dotnet", "BaGetter.dll"] +ARG Version=1.0.0 +ARG NuGetSource=https://api.nuget.org/v3/index.json + +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:10.0-alpine AS build +ARG Version +ARG NuGetSource +ARG TARGETARCH +WORKDIR /src + +## Create separate layer for `dotnet restore` to allow for caching; useful for local development +# copy only necessary files +COPY ./Directory.Packages.props ./nuget.config ./src/**/*.csproj ./ +# create folder structure that was lost when copying +RUN for file in $(ls *.csproj); do mkdir -p ${file%.*}/ && mv $file ${file%.*}/; done +# useful for debugging to display all files +#RUN echo $(ls) +# restore packages +RUN dotnet restore BaGetter/BaGetter.csproj --arch $TARGETARCH --source "$NuGetSource" + +## Publish app (implicitly builds the app) +FROM build AS publish +ARG Version +# copy all files +COPY /src . +RUN dotnet publish BaGetter \ + --configuration Release \ + --output /app \ + --no-restore \ + -m:1 \ + -p Version=${Version} \ + -p DebugType=none \ + -p DebugSymbols=false \ + -p GenerateDocumentationFile=false \ + -p UseAppHost=false \ + -a $TARGETARCH + +# create default folders +RUN mkdir -p "/data/packages" \ + "/data/symbols" \ + "/data/db" \ + "/data/admin" + +## Create final image +FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS base +# install cultures (same approach as Alpine SDK image) +RUN apk add --no-cache icu-libs icu-data-full tzdata openldap +# disable the invariant mode (set in base image) +ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false +# set default configurations; use the `/data` folder for packages, symbols and the SQLite database +ENV Storage__Path=/data +ENV Search__Type=Database +ENV Database__Type=Sqlite +ENV Database__ConnectionString="Data Source=/data/db/bagetter.db" +ENV Admin__AuditLogPath=/data/admin/admin-audit.jsonl +ENV ASPNETCORE_URLS=http://+:3007 +LABEL org.opencontainers.image.source="https://github.com/bagetter/BaGetter" +# copy default folders +COPY --from=publish /data /data +# copy the published app +WORKDIR /app +COPY --from=publish /app . + +EXPOSE 3007 + +ENTRYPOINT ["dotnet", "BaGetter.dll"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..3e88ee34b --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,169 @@ +// CD pipeline for the BaGetter NuGet mirror (the devop fork, vhafdal/BaGetter). +// +// Brings BaGetter into the same Jenkins + Harbor + Helm flow as DevOp.Nexus +// (it previously shipped via GitHub Actions -> GHCR and was deployed to k8s by +// hand). One run: build the image -> push to Harbor (registry.devop.is) -> helm +// upgrade the `bagetter` release in the cluster. +// +// Agent: provisions its OWN pod (the shared Jenkins cloud has no Docker daemon): +// - dind : docker:27-dind daemon (privileged) on tcp://localhost:2375, MTU 1450 +// - build : docker:27-cli -> talks to dind, builds + pushes the image +// - helm : alpine/k8s (helm + kubectl) for the deploy stage +// +// Credentials used (already exist in Jenkins for the platform pipeline): +// devop-registry-credentials (user/pass) — docker login to registry.devop.is (robot$jenkins-push) +// vcube-kubeconfig (secret file, optional) — kube access if the agent SA can't deploy +// +// PREREQUISITES (one-time, see deployment templates/chart/bagetter/README-cd.md): +// - Harbor pull secret in the `bagetter` namespace so the cluster can pull the +// private registry.devop.is/nexus/bagetter image (robot$k8s-pull). +// - The live deployment is NOT currently Helm-managed; the first `helm upgrade +// --install` must adopt or replace the existing resources. Validate on a +// scratch namespace (ENVIRONMENT=dev) before pointing at prod `bagetter`. + +pipeline { + agent { + kubernetes { + defaultContainer 'jnlp' + yaml ''' +apiVersion: v1 +kind: Pod +spec: + serviceAccountName: nexus-deployer + containers: + - name: dind + image: docker:27-dind + securityContext: + privileged: true + args: ["--mtu=1450"] + env: + - name: DOCKER_TLS_CERTDIR + value: "" + resources: + requests: + ephemeral-storage: 4Gi + memory: 1Gi + volumeMounts: + - name: docker-storage + mountPath: /var/lib/docker + - name: build + image: docker:27-cli + command: ["cat"] + tty: true + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: DOCKER_BUILDKIT + value: "1" + - name: helm + image: alpine/k8s:1.30.7 + command: ["cat"] + tty: true + volumes: + - name: docker-storage + emptyDir: {} +''' + } + } + + parameters { + choice(name: 'ENVIRONMENT', choices: ['prod', 'dev'], + description: 'Target: prod -> namespace bagetter; dev -> namespace bagetter-dev (scratch, for validating the chart)') + string(name: 'VERSION_OVERRIDE', defaultValue: '', + description: 'Image tag to build/deploy. Blank = sha-, matching the existing tag style.') + booleanParam(name: 'PUSH_IMAGE', defaultValue: true, description: 'Build and push the image to Harbor') + booleanParam(name: 'DEPLOY', defaultValue: true, description: 'helm upgrade --install against the cluster') + } + + environment { + REGISTRY_HOST = 'registry.devop.is' + IMAGE = 'registry.devop.is/nexus/bagetter' + CHART_DIR = 'deployment templates/chart/bagetter' + } + + stages { + stage('Resolve version') { + steps { + script { + git_safe() + def v = params.VERSION_OVERRIDE?.trim() + if (!v) { + def sha = sh(returnStdout: true, script: "git rev-parse --short HEAD").trim() + v = "sha-${sha}" + } + env.VERSION = v + env.NAMESPACE = (params.ENVIRONMENT == 'dev') ? 'bagetter-dev' : 'bagetter' + echo "BaGetter version=${env.VERSION} env=${params.ENVIRONMENT} ns=${env.NAMESPACE}" + } + } + } + + stage('Build & push image') { + when { expression { params.PUSH_IMAGE } } + steps { + container('build') { + sh ''' + apk add --no-cache git >/dev/null + echo "Waiting for the dind daemon..." + timeout 90 sh -c 'until docker info >/dev/null 2>&1; do sleep 2; done' + ''' + withCredentials([usernamePassword(credentialsId: 'devop-registry-credentials', + usernameVariable: 'REG_USER', passwordVariable: 'REG_PASS')]) { + sh ''' + echo "$REG_PASS" | docker login "$REGISTRY_HOST" -u "$REG_USER" --password-stdin + # The fork Dockerfile takes the build-time version via --build-arg Version. + # BuildKit auto-populates TARGETARCH (the Dockerfile builds for it). + # Image tag is VERSION (sha-* or override); the assembly Version + # build-arg must be valid semver, so fall back to 1.0.0 for sha builds. + docker build \ + --build-arg Version="${VERSION_OVERRIDE:-1.0.0}" \ + -t "$IMAGE:$VERSION" \ + -t "$IMAGE:latest" \ + . + docker push "$IMAGE:$VERSION" + docker push "$IMAGE:latest" + ''' + } + } + } + } + + stage('Helm deploy') { + when { expression { params.DEPLOY } } + steps { + container('helm') { + withKube { deployHelm() } + } + } + } + } + + post { + success { echo "BaGetter ${env.VERSION} deployed to ${params.ENVIRONMENT} (ns ${env.NAMESPACE})." } + always { container('build') { sh 'docker logout "$REGISTRY_HOST" || true' } } + } +} + +def git_safe() { sh "git config --global --add safe.directory '*' 2>/dev/null || true" } + +def withKube(Closure body) { + try { + withCredentials([file(credentialsId: 'vcube-kubeconfig', variable: 'KUBECONFIG')]) { body() } + } catch (ignored) { + echo 'vcube-kubeconfig not set; using the agent in-cluster ServiceAccount.' + body() + } +} + +def deployHelm() { + sh ''' + helm dependency build "$CHART_DIR" + helm upgrade --install bagetter "$CHART_DIR" \ + --namespace "$NAMESPACE" --create-namespace \ + -f "$CHART_DIR/values-prod.yaml" \ + --set controllers.bagetter.containers.bagetter.image.repository="$IMAGE" \ + --set controllers.bagetter.containers.bagetter.image.tag="$VERSION" \ + --wait --timeout 5m + kubectl -n "$NAMESPACE" rollout status deploy/bagetter --timeout=3m + ''' +} diff --git a/deployment templates/chart/bagetter/README-cd.md b/deployment templates/chart/bagetter/README-cd.md new file mode 100644 index 000000000..f51ce7f77 --- /dev/null +++ b/deployment templates/chart/bagetter/README-cd.md @@ -0,0 +1,54 @@ +# BaGetter CD (Jenkins + Harbor + Helm) + +Brings the devop BaGetter mirror (`nuget.devop.is`) onto the same delivery path as +DevOp.Nexus. The repo `Jenkinsfile` builds the fork image, pushes it to Harbor, and +`helm upgrade`s the `bagetter` release in the `vcube` cluster. + +## The job + +Create a Jenkins **Pipeline** job (e.g. `NexerIQ / Infra / BaGetter Deploy`) from SCM +pointing at this repo (GitHub, `vhafdal/BaGetter`, credential: the existing GitHub +Key), script path `Jenkinsfile`. Parameters: + +| Param | Default | Meaning | +|---|---|---| +| `ENVIRONMENT` | `prod` | `prod` → ns `bagetter`; `dev` → ns `bagetter-dev` (scratch, validate the chart here first) | +| `VERSION_OVERRIDE` | _(blank)_ | image tag; blank → `sha-` (matches the current tag style) | +| `PUSH_IMAGE` | true | build + push `registry.devop.is/nexus/bagetter:` | +| `DEPLOY` | true | `helm upgrade --install bagetter` | + +Image lands at `registry.devop.is/nexus/bagetter` (the `nexus` Harbor project, so +`robot$jenkins-push` / `robot$k8s-pull` already have access). + +## One-time prerequisites + +1. **Harbor pull secret in the namespace** — the image is private: + ```bash + kubectl -n bagetter create secret docker-registry nexus-registry \ + --docker-server=registry.devop.is \ + --docker-username='robot$k8s-pull' --docker-password='' + ``` + (`values-prod.yaml` references it via `imagePullSecrets`.) + +2. **Adopt the existing (non-Helm) release.** The live deployment was applied + outside Helm, so the first `helm upgrade --install` will collide with existing + objects. Either validate end-to-end in `bagetter-dev` first, or let Helm adopt + the resources (Helm 3 adopts objects that match name/namespace/GVK once the + `app.kubernetes.io/managed-by=Helm` + release annotations are added). Confirm the + `bagetter-data` PVC, `bagetter-mirror`/`bagetter-secret`/`bagetter-tls` Secrets, + and the ingress survive the cutover. + +3. **DB path** — verify `Database__ConnectionString` in `values-prod.yaml` matches + the running pod so the SQLite cache isn't orphaned (mooted by the Postgres step). + +## Next: PostgreSQL (fixes the concurrency lock) + +BaGetter currently runs SQLite on an NFS PVC; concurrent restore writes fail with +`database is locked`, which is why platform builds throttle to +`maxHttpRequestsPerSource=1`. With this pipeline in place we can ship the fix: + +1. Provision PostgreSQL (in-cluster or managed). +2. `values-prod.yaml`: set `Database__Type: PostgreSql` and + `Database__ConnectionString` to the Postgres DSN (via secret). +3. Deploy through this job; let the mirror re-cache (download stats reset is fine). +4. Remove the `maxHttpRequestsPerSource=1` throttle from `nexus-platform/nuget.config`. diff --git a/deployment templates/chart/bagetter/mirror-secret.example.yaml b/deployment templates/chart/bagetter/mirror-secret.example.yaml new file mode 100644 index 000000000..029a95e0f --- /dev/null +++ b/deployment templates/chart/bagetter/mirror-secret.example.yaml @@ -0,0 +1,49 @@ +# Read-through upstream mirrors for BaGetter (multi-mirror fork). +# +# These env vars bind to BaGetterOptions.Mirrors[] — when set, they take precedence +# over the single "Mirror" setting. On a cache miss BaGetter fetches the package from +# the upstream, stores it locally, and serves every later request (and any upstream +# outage) from the local copy. +# +# This file carries the DevExpress feed key (in the PackageSource URL), so it is a +# Secret and MUST NOT be committed with the real key. Create it in-cluster from a +# sealed/external secret or: +# +# kubectl -n create secret generic bagetter-mirror \ +# --from-literal=Mirrors__0__Enabled=true \ +# --from-literal=Mirrors__0__Legacy=true \ +# --from-literal=Mirrors__0__PackageSource='https://nuget.devexpress.com//api' \ +# --from-literal=Mirrors__0__PackageDownloadTimeoutSeconds=900 +# +apiVersion: v1 +kind: Secret +metadata: + name: bagetter-mirror +type: Opaque +stringData: + # Index 0 — DevExpress (v2 / OData feed: Legacy=true; key is embedded in the URL). + Mirrors__0__Enabled: "true" + Mirrors__0__Legacy: "true" + Mirrors__0__PackageSource: "https://nuget.devexpress.com//api" + Mirrors__0__PackageDownloadTimeoutSeconds: "900" + + # Index 1 — Hangfire Pro (v2 feed; Basic auth). + Mirrors__1__Enabled: "true" + Mirrors__1__Legacy: "true" + Mirrors__1__PackageSource: "https://nuget.hangfire.io/nuget/hangfire-pro/" + Mirrors__1__Authentication__Type: "Basic" + Mirrors__1__Authentication__Username: "" + Mirrors__1__Authentication__Password: "" + + # Index 2 — Hangfire Ace (v2 feed; Basic auth). + Mirrors__2__Enabled: "true" + Mirrors__2__Legacy: "true" + Mirrors__2__PackageSource: "https://nuget.hangfire.io/nuget/hangfire-ace/" + Mirrors__2__Authentication__Type: "Basic" + Mirrors__2__Authentication__Username: "" + Mirrors__2__Authentication__Password: "" + + # Optional — also cache nuget.org (insulate builds from nuget.org outages). Only + # useful if builds route nuget.org restores through this feed. + # Mirrors__3__Enabled: "true" + # Mirrors__3__PackageSource: "https://api.nuget.org/v3/index.json" diff --git a/deployment templates/chart/bagetter/values-prod.yaml b/deployment templates/chart/bagetter/values-prod.yaml new file mode 100644 index 000000000..c0a1b1eae --- /dev/null +++ b/deployment templates/chart/bagetter/values-prod.yaml @@ -0,0 +1,105 @@ +--- +# Production values for the devop BaGetter mirror (nuget.devop.is). Faithfully +# reproduces the live deployment (now on PostgreSQL) so Helm can take it over. +# image.repository/tag are set by the Jenkinsfile via --set. +# +# Required in the namespace (NOT created by this chart): +# secret bagetter-secret : api-key, user-valdi-password, user-nexer-password +# secret bagetter-db : connstring (PostgreSQL DSN) +# secret bagetter-mirror : Mirrors__N__* upstreams (0 hangfire-pro, 1 hangfire-ace, 2 nuget.org) +# secret bagetter-tls : TLS cert for nuget.devop.is +# secret nexus-registry : Harbor pull secret (robot$k8s-pull) +# the Postgres StatefulSet from ../../postgres.yaml +# PVC bagetter-data (existing) + +controllers: + bagetter: + pod: + imagePullSecrets: + - name: nexus-registry + containers: + bagetter: + image: + repository: registry.devop.is/nexus/bagetter + tag: latest + pullPolicy: IfNotPresent + env: + ASPNETCORE_URLS: "http://+:8080" + Storage__Type: FileSystem + Storage__Path: /data/packages + Search__Type: Database + Database__Type: PostgreSql + Database__ConnectionString: + valueFrom: + secretKeyRef: + name: bagetter-db + key: connstring + # Vestigial single-mirror setting; superseded by the Mirrors__* array + # (from the bagetter-mirror secret) but kept to match the live config. + Mirror__Enabled: "true" + Mirror__PackageSource: "https://api.nuget.org/v3/index.json" + ApiKey: + valueFrom: + secretKeyRef: + name: bagetter-secret + key: api-key + Authentication__Credentials__0__Username: valdi + Authentication__Credentials__0__Roles__0: Admin + Authentication__Credentials__0__Password: + valueFrom: + secretKeyRef: + name: bagetter-secret + key: user-valdi-password + Authentication__Credentials__1__Username: nexer.usr + Authentication__Credentials__1__Password: + valueFrom: + secretKeyRef: + name: bagetter-secret + key: user-nexer-password + envFrom: + # Read-through upstream mirrors: slot0 hangfire-pro, slot1 hangfire-ace, + # slot2 nuget.org (all Mirrors__N__* keys load as env vars). + - secretRef: + name: bagetter-mirror + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + +service: + bagetter-srv: + controller: bagetter + ports: + http: + port: 8080 + +ingress: + bagetter-ingress: + enabled: true + className: traefik + hosts: + - host: nuget.devop.is + paths: + - path: / + service: + identifier: bagetter-srv + port: http + tls: + - hosts: + - nuget.devop.is + secretName: bagetter-tls + +persistence: + bagetter-data: + enabled: true + existingClaim: bagetter-data + globalMounts: + - path: /data diff --git a/deployment templates/chart/bagetter/values.yaml b/deployment templates/chart/bagetter/values.yaml index b6a18ee15..57c9bb168 100644 --- a/deployment templates/chart/bagetter/values.yaml +++ b/deployment templates/chart/bagetter/values.yaml @@ -24,6 +24,12 @@ configMaps: #Mirror__Enabled: "true" #Mirror__PackageSource: "https://api.nuget.org/v3/index.json" #PackageDeletionBehavior: "HardDelete" + # + # Multi-mirror (read-through caching) is configured via the `bagetter-mirror` + # Secret (see mirror-secret.example.yaml), wired through envFrom below. The + # DevExpress feed key lives in the PackageSource URL, so it must NOT be put + # here in the (committed) ConfigMap. Mirrors[i] keys: Enabled, PackageSource, + # Legacy (true for v2/OData feeds like DevExpress), PackageDownloadTimeoutSeconds. # -- Configures containers for the chart. # @default -- See values.yaml @@ -41,6 +47,11 @@ controllers: envFrom: - configMapRef: name: bagetter-bagetter-env + # Read-through mirror config (incl. the DevExpress feed key in PackageSource). + # optional: pod still starts if the Secret is absent (mirroring simply stays off). + - secretRef: + name: bagetter-mirror + optional: true resources: requests: cpu: 100m diff --git a/deployment templates/postgres.yaml b/deployment templates/postgres.yaml new file mode 100644 index 000000000..94ac5d8b9 --- /dev/null +++ b/deployment templates/postgres.yaml @@ -0,0 +1,81 @@ +--- +# Minimal PostgreSQL for the BaGetter mirror (namespace: bagetter). +# Replaces SQLite-on-NFS, which serialises all writes and throws +# "database is locked" 500s under concurrent restore. Postgres handles +# concurrent writes, so the build-side maxHttpRequestsPerSource=1 throttle +# can be removed afterwards. +# +# Prereq — create the password secret first (never commit the password): +# kubectl -n bagetter create secret generic bagetter-postgres \ +# --from-literal=password="$(openssl rand -base64 24)" +# +# Apply: kubectl -n bagetter apply -f "deployment templates/postgres.yaml" +apiVersion: v1 +kind: Service +metadata: + name: bagetter-postgres + namespace: bagetter +spec: + selector: + app: bagetter-postgres + ports: + - port: 5432 + targetPort: 5432 + clusterIP: None +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: bagetter-postgres + namespace: bagetter +spec: + serviceName: bagetter-postgres + replicas: 1 + selector: + matchLabels: + app: bagetter-postgres + template: + metadata: + labels: + app: bagetter-postgres + spec: + containers: + - name: postgres + image: postgres:16-alpine + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: bagetter + - name: POSTGRES_USER + value: bagetter + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: bagetter-postgres + key: password + # Keep PGDATA in a subdir so the volume root (lost+found etc.) doesn't clash. + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + readinessProbe: + exec: + command: ["sh", "-c", "pg_isready -U bagetter -d bagetter"] + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..fb47320bb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +services: + bagetter: + build: + context: . + dockerfile: Dockerfile + args: + NuGetSource: ${NUGET_SOURCE:-https://api.nuget.org/v3/index.json} + image: bagetter-admin:local + container_name: bagetter + restart: unless-stopped + ports: + - "3007:3007" + environment: + ASPNETCORE_ENVIRONMENT: Production + Authentication__Credentials__0__Username: ${BAGETTER_ADMIN_USERNAME:-admin} + Authentication__Credentials__0__Password: ${BAGETTER_ADMIN_PASSWORD:?Set BAGETTER_ADMIN_PASSWORD} + Authentication__Credentials__0__Roles__0: Admin + volumes: + - bagetter-data:/data + +volumes: + bagetter-data: diff --git a/docs/docs/Advanced/debugging.md b/docs/docs/Advanced/debugging.md index a63a206a4..3a9ab173f 100644 --- a/docs/docs/Advanced/debugging.md +++ b/docs/docs/Advanced/debugging.md @@ -1,7 +1,103 @@ # Debugging BaGetter -:::warning +## Quick diagnostics -This page is a work in progress! +1. Verify service index: -::: +```text +GET /v3/index.json +``` + +2. Verify health endpoint: + +```text +GET /health +``` + +3. Verify search and metadata: + +```text +GET /v3/search?q=test +GET /v3/registration/{id}/index.json +``` + +## Request telemetry logs + +BaGetter emits per-request telemetry logs including: + +- HTTP method +- request path +- status code +- duration in ms +- endpoint display name +- trace id + +Use these logs to find: + +- high-latency endpoints +- repeated failures (`404`, `401`, `429`, `500`) +- request loops from clients + +## Audit logs for package mutations + +BaGetter emits audit entries for: + +- package upload +- package delete +- package relist + +Audit entries include actor and client IP metadata. + +Use them to answer: + +- who attempted a change +- what package and version were targeted +- whether the action was denied, failed, or successful + +## Caching troubleshooting + +For metadata and content endpoints, inspect: + +- `ETag` +- `Cache-Control` + +Expected: + +- registration endpoints: `must-revalidate` +- versioned package content: `immutable` + +If clients do not revalidate correctly, compare request `If-None-Match` and response `ETag`. + +## Search troubleshooting + +If search results appear stale: + +1. Run a manual reindex: + +```powershell +dotnet run --project src/BaGetter -- reindex search +``` + +2. Consider background reindex configuration: + +- `Reindex.Enabled` +- `Reindex.RunOnStartup` +- `Reindex.IntervalMinutes` + +## IIS deployment troubleshooting + +Common issue: deploying build output instead of publish output. + +Expected IIS deploy folder: + +```text +src/BaGetter/bin/Release/net10.0/publish/ +``` + +Use: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/Publish-IIS.ps1 +``` + +For a full production checklist, see the [Operations Playbook](operations-playbook.md). diff --git a/docs/docs/Advanced/operations-playbook.md b/docs/docs/Advanced/operations-playbook.md new file mode 100644 index 000000000..7fb07f76e --- /dev/null +++ b/docs/docs/Advanced/operations-playbook.md @@ -0,0 +1,122 @@ +--- +sidebar_position: 3 +--- + +# Operations Playbook + +This page is a practical checklist for deploying, validating, and troubleshooting BaGetter in production-like environments. + +## Publish and Deploy (IIS) + +Always publish the host project (`src/BaGetter`), not `src/BaGetter.Web`. + +```powershell +dotnet publish src/BaGetter/BaGetter.csproj -c Release -p:PublishProfile=FolderProfile +``` + +Expected output folder: + +```text +src/BaGetter/bin/Release/net10.0/publish/ +``` + +The publish folder must contain `web.config` for IIS hosting. + +You can also run: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/Publish-IIS.ps1 +``` + +## Post-Deploy Smoke Checks + +Run these checks against your deployed base URL. + +```powershell +$base = "https://your-bagetter-host" +Invoke-WebRequest "$base/v3/index.json" | Out-Null +Invoke-WebRequest "$base/health" | Out-Null +Invoke-WebRequest "$base/v3/search?q=test" | Out-Null +``` + +Or run the bundled smoke test script: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/SmokeTest-BaGetter.ps1 -BaseUrl https://your-bagetter-host +``` + +Check caching headers: + +```powershell +$index = Invoke-WebRequest "$base/v3/registration/TestData/index.json" +$index.Headers.ETag +$index.Headers."Cache-Control" +``` + +Expected behavior: + +- Registration endpoints include `ETag`. +- Registration endpoints use `Cache-Control` with `must-revalidate`. +- Versioned package content endpoints use `Cache-Control` with `immutable`. + +## Troubleshooting: Partial Page Render / Endless Loading + +If the UI partially renders and stays in a loading state: + +1. Open browser network tools and look for repeating `404` on icon paths. +2. Confirm package icon endpoints return valid image content types. +3. Check server logs for repeated `AUDIT` or request telemetry patterns on the same path. + +Useful server-side checks: + +```powershell +Invoke-WebRequest "$base/v3/package/testdata/1.2.3/icon" -Method GET +Invoke-WebRequest "$base/v3/package/testdata/1.2.3/testdata.1.2.3.nupkg" -Method GET +``` + +## Reindex Search + +Manual reindex: + +```powershell +dotnet run --project src/BaGetter -- reindex search +``` + +Background reindex configuration: + +```json +{ + "Reindex": { + "Enabled": true, + "RunOnStartup": true, + "IntervalMinutes": 60, + "BatchSize": 100 + } +} +``` + +Use this when: + +- Search results are stale after migrations or provider changes. +- You changed indexing behavior and need a full rebuild. + +## Audit and Request Logs + +BaGetter emits: + +- request telemetry logs (method, path, status, duration, endpoint, trace id) +- audit logs for upload, delete, and relist actions + +Use these to answer: + +- who performed a package mutation +- what action was attempted +- whether it succeeded, failed, or was denied + +## Safety Checklist + +- Do not store secrets in committed `appsettings.*` files. +- Keep `appsettings.Development.json` local-only. +- Restrict CORS in production (`Cors:AllowAnyOrigin=false` with explicit origins). +- Enable rate limiting for internet-facing deployments. +- Keep security headers enabled. diff --git a/docs/docs/Index.md b/docs/docs/Index.md index 25b935270..4b2c4cb61 100644 --- a/docs/docs/Index.md +++ b/docs/docs/Index.md @@ -17,12 +17,16 @@ BaGetter (pronounced "ba getter") is a lightweight NuGet and symbol server. It i BaGetter supports Filesystem, GCP and AWS S3 buckets, and Azure Blob Storage for package storage, and MySQL, Sqlite, SqlServer and PostgreSQL as database. The current per-package size limit is ~8GB. It can be hosted on IIS, and is also available in a linux [docker image](https://hub.docker.com/r/bagetter/bagetter). +It also supports paged NuGet registration metadata responses to improve performance for packages with large version histories. + +Recent operational capabilities include ETag-based HTTP caching, response compression, configurable rate limiting, security headers, and background/manual search reindexing. ## Run BaGetter You can run BaGetter on your preferred platform: - [On your computer](Installation/local.md) +- [Windows Service](Installation/windows-service.md) - [Docker](Installation/docker.md) - [Azure](Installation/azure.md) - [AWS](Installation/aws.md) @@ -32,3 +36,7 @@ You can run BaGetter on your preferred platform: ## BaGetter SDK You can also use the [`BaGetter.Protocol`](https://www.nuget.org/packages/BaGetter.Protocol) package to interact with a NuGet server. For more information, please refer to the [BaGetter SDK](Advanced/sdk.md) guide. + +## Operations + +For deployment validation and production troubleshooting, use the [Operations Playbook](Advanced/operations-playbook.md). diff --git a/docs/docs/Installation/iis-proxy.md b/docs/docs/Installation/iis-proxy.md index 1d6757d34..dcbcdad6c 100644 --- a/docs/docs/Installation/iis-proxy.md +++ b/docs/docs/Installation/iis-proxy.md @@ -16,6 +16,30 @@ With a few extra steps you can run BaGetter behind a Windows IIS proxy. This has - Choose `BaGetterAppPool` as the application pool - In the Binding area, enter the default BaGetter port of 5000 +## Publish output and web.config + +When deploying to IIS, publish the host project (`src/BaGetter`) and deploy the publish folder output: + +```powershell +dotnet publish src/BaGetter/BaGetter.csproj -c Release -p:PublishProfile=FolderProfile +``` + +Or run the provided script from the repository root: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/Publish-IIS.ps1 +``` + +Expected output: + +```text +src/BaGetter/bin/Release/net10.0/publish/ +``` + +This output should include `web.config`. + +If you are looking at `src/BaGetter/bin/Release/net10.0/` (without `publish/`), that is build output, not publish output. + ## BaGetter folder permissions You **may** need to give special permissions to the top-level BaGetter folder so that the app can persist its state. This is necessary as the Application Pools' identity is a virtual account that isn't recognized by the Windows User Management Console. For more information, please refer to ASP.NET Core's documentation: diff --git a/docs/docs/Installation/local.md b/docs/docs/Installation/local.md index e3ce221b2..6c39546e7 100644 --- a/docs/docs/Installation/local.md +++ b/docs/docs/Installation/local.md @@ -11,6 +11,11 @@ You can modify BaGetter's configurations by editing the `appsettings.json` file. For the full list of configurations, please refer to [BaGetter's configuration](../configuration.md) guide. +If you want to secure package restore/download access, BaGetter supports both local +`Authentication.Credentials` users and LDAP / Active Directory through +`Authentication.Ldap`. See the [Private feeds](../configuration.md#private-feeds) section +for examples. + ## Publish packages Publish your first package with: diff --git a/docs/docs/Installation/windows-service.md b/docs/docs/Installation/windows-service.md new file mode 100644 index 000000000..191d1eed9 --- /dev/null +++ b/docs/docs/Installation/windows-service.md @@ -0,0 +1,86 @@ +--- +sidebar_position: 8 +--- + +# Windows Service + +BaGetter can run as a native Windows Service (without IIS). + +## Prerequisites + +1. Install the [.NET 10 Runtime](https://dotnet.microsoft.com/download) if you publish framework-dependent binaries. +2. Open an elevated PowerShell session (Run as Administrator). + +## Publish + +Publish the host project (`src/BaGetter`). + +Framework-dependent: + +```powershell +dotnet publish src/BaGetter/BaGetter.csproj -c Release -o C:\Services\BaGetter +``` + +Self-contained (no shared runtime dependency): + +```powershell +dotnet publish src/BaGetter/BaGetter.csproj -c Release -r win-x64 --self-contained true -o C:\Services\BaGetter +``` + +## Install service + +From the publish folder: + +```powershell +cd C:\Services\BaGetter +.\BaGetter.exe install service --name BaGetter --start +``` + +Available options: + +- `--name `: Windows service name (default: `BaGetter`) +- `--display-name `: Service display name +- `--urls `: URL bindings passed to Kestrel +- `--start`: Starts the service immediately after registration + +## URL binding behavior + +If `--urls` is omitted, BaGetter resolves URLs in this order: + +1. `Urls` / `urls` from configuration +2. default fallback: `http://0.0.0.0:50561` + +Configuration sources include local `appsettings.json` and `%ProgramData%\BaGetter\AppSettings.json`. + +## Uninstall service + +```powershell +.\BaGetter.exe uninstall service --name BaGetter --stop +``` + +Available options: + +- `--name `: Windows service name (default: `BaGetter`) +- `--stop`: Attempts to stop the service before deleting it + +## Service management + +Check status: + +```powershell +sc.exe query BaGetter +``` + +Start/stop manually: + +```powershell +sc.exe start BaGetter +sc.exe stop BaGetter +``` + +## Notes + +- The install/uninstall commands are Windows-only. +- Running behind IIS is still supported; Windows Service hosting is an alternative deployment mode. +- If `Microsoft.Extensions.Hosting.WindowsServices.dll` is missing in a deployment, BaGetter falls back to normal console/IIS hosting on Windows instead of crashing at startup. +- For configuration details, see [Configuration](../configuration.md). diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index c46580185..f23b1a230 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -5,6 +5,29 @@ import TabItem from '@theme/TabItem'; You can modify BaGetter's configurations by editing the `appsettings.json` file. +BaGetter also loads configuration from an OS-level location when present: + +- Windows: `%ProgramData%\BaGetter\AppSettings.json` +- Linux: `/etc/BaGetter/AppSettings.json` + +You can override the configuration root path using: + +```text +BAGET_CONFIG_ROOT +``` + +## Startup validation rules + +BaGetter validates configuration at startup and fails fast when critical settings are invalid. Current guardrails include: + +- `MaxPackageSizeGiB` must be greater than `0`. +- `PathBase` must start with `/` and must not end with `/` (except `/` itself). +- `Cors.AllowAnyOrigin=true` cannot be combined with `Cors.AllowCredentials=true`. +- If `Cors.AllowAnyOrigin=false`, then `Cors.AllowedOrigins` must contain at least one value. +- `SecurityHeaders.EnableHsts=true` requires `SecurityHeaders.Enabled=true`. + +These checks help catch unsafe or inconsistent runtime settings before serving traffic. + ## Require an API key You can require that users provide a password, called an API key, to publish packages. @@ -38,6 +61,38 @@ You can also use the `ApiKeys` array in order to manage multiple API keys for mu Both `ApiKey` and `ApiKeys` work in conjunction additively eg.: `or` `||` logical operator. +You can also store hashed API keys instead of plaintext values: + +- `ApiKeyHash` for the legacy single key +- `Authentication.ApiKeys[].KeyHash` for multiple keys + +Supported hash format: + +```text +PBKDF2$$$ +``` + +Example: + +```json +{ + "ApiKeyHash": "PBKDF2$100000$$", + "Authentication": { + "ApiKeys": [ + { + "KeyHash": "PBKDF2$100000$$" + } + ] + } +} +``` + +Generate a hash from the CLI: + +```shell +dotnet run --project src/BaGetter -- hash "NUGET-SERVER-API-KEY" +``` + Users will now have to provide the API key to push packages: ```shell @@ -150,6 +205,30 @@ The following `Mirror` setting configures BaGetter to index packages from [nuget +You can also configure multiple upstream mirrors using `Mirrors`. +When `Mirrors` is present and contains one or more entries, it takes precedence over `Mirror`. +BaGetter will try mirrors in order and fall back to the next mirror when a package is not found. +If `Mirrors` is absent or empty, BaGetter uses the legacy `Mirror` setting. + +```json +{ + ... + + "Mirrors": [ + { + "Enabled": true, + "PackageSource": "https://api.nuget.org/v3/index.json" + }, + { + "Enabled": true, + "PackageSource": "https://your-secondary-feed/v3/index.json" + } + ], + + ... +} +``` + :::info @@ -241,7 +320,8 @@ To do so, you can insert the credentials in the `Authentication` section. "Credentials": [ { "Username": "username", - "Password": "password" + "Password": "password", + "Roles": [ "Admin" ] } ] ... @@ -250,8 +330,112 @@ To do so, you can insert the credentials in the `Authentication` section. } ``` +- `Roles` controls role-based permissions (for example `Admin` for administration-only actions in the web UI such as package deletion). + +### Administration dashboard + +Users with the `Admin` role can open `/admin` to manage individual package versions. +The dashboard supports search and listing-status filters, unlist/relist, permanent +deletion, copy to a new package ID, and rename to a new package ID. + +- Permanent deletion removes both package metadata and stored package content. +- Rename copies the package to the new ID and permanently deletes the source only + after the copy succeeds. +- Signed packages cannot be copied or renamed because changing the package ID would + invalidate the signature. +- When `IsReadOnlyMode` is enabled, the dashboard remains visible but all package + mutations are disabled. +- Administrative actions are appended as JSON Lines to the path configured by + `Admin.AuditLogPath` (default: `App_Data/admin-audit.jsonl`). Relative paths + are resolved under the application's content root. The Docker image uses the + persistent path `/data/admin/admin-audit.jsonl`. Each entry + records the UTC time, actor, action, package, result, and client IP address. + +For Docker deployment on port 3007: + +```shell +BAGETTER_ADMIN_PASSWORD="" docker compose up -d --build +``` + +The included Compose configuration stores packages, the SQLite database, symbols, +and the administration audit log in the `bagetter-data` volume. + +The image restores packages from NuGet v3 by default. If that endpoint is blocked +by a proxy or TLS inspection policy, set `NUGET_SOURCE` to an approved NuGet +mirror when building the image. For example: + +```shell +NUGET_SOURCE=https://www.nuget.org/api/v2/ BAGETTER_ADMIN_PASSWORD="" docker compose up -d --build +``` + +You can store a hashed password instead of plaintext: + +```json +{ + "Authentication": { + "Credentials": [ + { + "Username": "username", + "PasswordHash": "PBKDF2$100000$$" + } + ] + } +} +``` + +Generate password hash: + +```shell +dotnet run --project src/BaGetter -- hash "password" +``` + Users will now have to provide the username and password to fetch and download packages. +### LDAP / Active Directory authentication + +BaGetter can also validate package feed credentials against LDAP or Active Directory. +Configure LDAP under `Authentication.Ldap`. You can keep `Authentication.Credentials` +configured at the same time to preserve local fallback accounts for administration or +break-glass access. + +```json +{ + "Authentication": { + "Credentials": [ + { + "Username": "local-admin", + "PasswordHash": "PBKDF2$100000$$", + "Roles": [ "Admin" ] + } + ], + "Ldap": { + "Enabled": true, + "Server": "ad.example.local", + "BaseDn": "dc=example,dc=local", + "BindDn": "cn=bagetter,ou=Service Accounts,dc=example,dc=local", + "BindPassword": "change-me", + "UseSsl": true, + "AllowedGroups": [ "NuGet Readers", "NuGet Admins" ] + } + } +} +``` + +LDAP users can authenticate with: + +- `domain\username` +- `username@domain.example` + +Configuration notes: + +- `Authentication.Ldap.Server` and `Authentication.Ldap.BaseDn` are required when LDAP is enabled. +- `Authentication.Ldap.BindDn` and `Authentication.Ldap.BindPassword` are recommended for Active Directory because most environments do not allow anonymous directory searches. +- BaGetter adds role claims from the LDAP / Active Directory groups returned for the user. Group names are taken from each group's `CN`. +- If `Authentication.Ldap.AllowedGroups` is set, login succeeds only when the user belongs to at least one of those group names. +- If `Authentication.Ldap.AllowedGroups` is omitted or empty, no group filter is applied. +- `Authentication.Ldap.UseSsl` defaults to `true`. Use LDAPS whenever possible. +- BaGetter uses `System.DirectoryServices.Protocols`, which is cross-platform. On Linux and macOS, make sure the host trusts your LDAP server's certificate chain when using LDAPS. + How to add private nuget feed: 1. Download the latest NuGet executable. @@ -281,6 +465,13 @@ The commands are slightly different when using the Package Manager console in Vi dotnet nuget add source "http://localhost:5000/v3/index.json" --name "bagetter" --username "username" --password "password" ``` +The same commands work for LDAP-backed feeds. Replace the username and password with your +directory credentials, for example: + +```shell +dotnet nuget add source "http://localhost:5000/v3/index.json" --name "bagetter" --username "hysing\\valdi" --password "password" +``` + ## Database configuration BaGetter supports multiple database engines for storing package information: @@ -396,6 +587,191 @@ This can be useful if you are hosting a private feed and need to host large pack } ``` +## Registration paging + +BaGetter serves package metadata using the NuGet registration resource. By default, versions are split into pages of 64 entries. +This helps keep registration index payloads smaller for packages with many versions. + +You can configure this using `RegistrationPageSize`: + +```json +{ + ... + + "RegistrationPageSize": 64, + + ... +} +``` + +When paging is active, the registration index (`/v3/registration/{id}/index.json`) returns page references, +and each page can be fetched from: + +```text +/v3/registration/{id}/page/{lower}/{upper}.json +``` + +Example: + +```text +/v3/registration/newtonsoft.json/page/11.0.1/13.0.3.json +``` + +## HTTP caching and compression + +BaGetter now emits `ETag` headers for: + +- registration index/page/leaf endpoints +- package versions endpoint +- package content endpoints (`.nupkg`, `.nuspec`, readme, icon) + +Metadata endpoints use: + +```text +Cache-Control: public, max-age=0, must-revalidate +``` + +Versioned package content endpoints use: + +```text +Cache-Control: public, max-age=31536000, immutable +``` + +BaGetter also enables response compression (Brotli + Gzip) for supported responses. + +## HTTP request telemetry + +BaGetter logs a structured entry per request including method, path, status code, duration, endpoint, and trace identifier. + +It also records in-process metrics using `System.Diagnostics.Metrics`: + +- `bagetter.http.requests` (counter) +- `bagetter.http.request.duration` (histogram, milliseconds) + +## Request rate limiting + +You can enable fixed-window request rate limiting: + +```json +{ + ... + + "RequestRateLimit": { + "Enabled": true, + "PermitLimit": 120, + "WindowSeconds": 60, + "QueueLimit": 0 + }, + + ... +} +``` + +- `Enabled`: turn rate limiting on or off. +- `PermitLimit`: number of requests allowed in each window. +- `WindowSeconds`: window duration in seconds. +- `QueueLimit`: number of requests to queue when the limit is reached. + +When enabled, rejected requests return `429 Too Many Requests`. + +## CORS policy + +By default, BaGetter allows all origins, methods, and headers. You can restrict this policy: + +```json +{ + ... + + "Cors": { + "AllowAnyOrigin": false, + "AllowedOrigins": [ + "https://packages.example.com" + ], + "AllowAnyMethod": true, + "AllowedMethods": [], + "AllowAnyHeader": true, + "AllowedHeaders": [], + "AllowCredentials": false + }, + + ... +} +``` + +If `AllowAnyOrigin` is `true`, `AllowedOrigins` is ignored. + +## Security headers + +BaGetter can emit baseline security headers for all responses: + +- `X-Content-Type-Options: nosniff` +- `X-Frame-Options: SAMEORIGIN` +- `Referrer-Policy: no-referrer` +- `X-Permitted-Cross-Domain-Policies: none` +- `Permissions-Policy: camera=(), microphone=(), geolocation=()` + +Configuration: + +```json +{ + ... + + "SecurityHeaders": { + "Enabled": true, + "EnableHsts": false, + "HstsMaxAgeDays": 365, + "HstsIncludeSubDomains": true, + "HstsPreload": false + }, + + ... +} +``` + +If `EnableHsts` is `true`, BaGetter enables HSTS outside development environments. + +## Search reindexing + +BaGetter supports rebuilding the search index from packages stored in the database. + +Configuration: + +```json +{ + ... + + "Reindex": { + "Enabled": false, + "RunOnStartup": false, + "IntervalMinutes": 0, + "BatchSize": 100 + }, + + ... +} +``` + +- `Enabled`: enables the background reindex hosted service. +- `RunOnStartup`: run one reindex pass when the app starts. +- `IntervalMinutes`: if greater than `0`, run periodic reindex passes at this interval. +- `BatchSize`: number of packages processed per batch. + +You can also trigger reindex manually from the CLI: + +```shell +dotnet run --project src/BaGetter -- reindex search +``` + +## Audit logging + +BaGetter emits audit log entries for package mutation operations: + +- package upload attempts and outcomes +- package delete attempts and outcomes +- package relist attempts and outcomes + +Each event includes actor and client IP metadata to simplify traceability in centralized logs. + ## Statistics On the application's statistics page the currently used services and overall package and version counts are listed. @@ -415,6 +791,24 @@ If you set `ListConfiguredServices` to `false` the currently used services for d } ``` +## Web UI theming + +BaGetter's built-in UI can be restyled by editing: + +- `src/BaGetter.Web/wwwroot/css/site.css` + +The stylesheet uses CSS custom properties in `:root` for the visual system. The most important tokens are: + +- `--font-body`, `--font-heading` +- `--page-bg`, `--page-text`, `--surface`, `--surface-border` +- `--brand-primary`, `--brand-accent`, `--brand-highlight` +- `--link-color`, `--link-hover` +- `--radius-sm`, `--radius-md`, `--radius-lg` + +There is also a `@media (prefers-color-scheme: dark)` block for dark-mode values. + +If you need different branding, you can safely override these tokens while keeping the component selectors unchanged. + ## Load secrets from files diff --git a/global.json b/global.json index 214a6386b..60ac4c5b5 100644 --- a/global.json +++ b/global.json @@ -1,7 +1,8 @@ { "sdk": { - "version": "9.0.306", + "version": "10.0.101", "rollForward": "latestFeature", "allowPrerelease": false } } + diff --git a/readme.md b/readme.md index b9a875922..f32eef4d5 100644 --- a/readme.md +++ b/readme.md @@ -23,12 +23,26 @@ With .NET: 3. Start the service with `dotnet BaGetter.dll` 4. Browse `http://localhost:5000/` in your browser +Manual search reindex command: + +```powershell +dotnet run --project src/BaGetter -- reindex search +``` + With IIS ([official microsoft documentation](https://learn.microsoft.com/aspnet/core/host-and-deploy/iis)): 1. Install the [hosting bundle](https://dotnet.microsoft.com/permalink/dotnetcore-current-windows-runtime-bundle-installer) -2. Download the [zip release](https://github.com/bagetter/BaGetter/releases) of BaGetter -3. Unpack the zip file contents to a folder of your choice -4. Create a new or configure an existing IIS site to point its physical path to the folder where you unpacked the zip file +2. Publish the host project (`src/BaGetter`), not `src/BaGetter.Web`: + - `dotnet publish src/BaGetter/BaGetter.csproj -c Release -p:PublishProfile=FolderProfile` +3. Deploy the `src/BaGetter/bin/Release/net10.0/publish/` folder to your server +4. Ensure your IIS site's physical path points to the publish folder +5. Confirm `web.config` exists in the publish output. If you only see `bin/Release/net10.0/` and not `publish/`, you likely built instead of published. + +You can also use the helper script from the repository root: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/Publish-IIS.ps1 +``` For more information, please refer to the [documentation]. @@ -38,6 +52,11 @@ For more information, please refer to the [documentation]. * **ARM** (64bit) **support**. You can host your NuGets on a device like Raspberry Pi! * **Cloud native**: supports [Docker][Docker doc link], [AWS][AWS doc link], [Google Cloud][GCP doc link], [Alibaba Cloud][Aliyun doc link]，[Tencent Cloud][Tencent doc link] * **Offline support**: [Mirror a NuGet server][Read through caching] to speed up builds and enable offline downloads +* **Private feed authentication**: supports local users, API keys, and LDAP / Active Directory-backed feed access +* **Operational readiness**: supports ETag-based HTTP caching, response compression, request rate limiting, security headers, and audit logging for package mutations +* **Search recovery tooling**: supports manual and background search reindexing + +For deployment checks and troubleshooting workflows, see the [Operations Playbook][Operations Playbook]. ## 🤝 Contributing @@ -78,6 +97,7 @@ Thanks to everyone who helps to make BaGetter better! [BaGetter's latest release]: https://github.com/bagetter/BaGetter/releases [Documentation]: https://www.bagetter.com/ +[Operations Playbook]: https://www.bagetter.com/docs/Advanced/operations-playbook [Read through caching]: https://www.bagetter.com/docs/configuration#enable-read-through-caching [AWS doc link]: https://www.bagetter.com/docs/Installation/aws [GCP doc link]: https://www.bagetter.com/docs/Installation/gcp diff --git a/samples/BaGetter.Protocol.Samples.Tests/BaGetter.Protocol.Samples.Tests.csproj b/samples/BaGetter.Protocol.Samples.Tests/BaGetter.Protocol.Samples.Tests.csproj index ae3faefab..5e627113c 100644 --- a/samples/BaGetter.Protocol.Samples.Tests/BaGetter.Protocol.Samples.Tests.csproj +++ b/samples/BaGetter.Protocol.Samples.Tests/BaGetter.Protocol.Samples.Tests.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 false IDE0007 @@ -23,3 +23,4 @@ + diff --git a/samples/BaGetterWebApplication/BaGetterWebApplication.csproj b/samples/BaGetterWebApplication/BaGetterWebApplication.csproj index ac3e03237..3e6628059 100644 --- a/samples/BaGetterWebApplication/BaGetterWebApplication.csproj +++ b/samples/BaGetterWebApplication/BaGetterWebApplication.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 @@ -10,3 +10,4 @@ + diff --git a/scripts/Publish-IIS.ps1 b/scripts/Publish-IIS.ps1 new file mode 100644 index 000000000..1b25b850d --- /dev/null +++ b/scripts/Publish-IIS.ps1 @@ -0,0 +1,26 @@ +param( + [string]$Configuration = "Release", + [string]$Project = "src/BaGetter/BaGetter.csproj", + [string]$PublishProfile = "FolderProfile" +) + +$ErrorActionPreference = "Stop" + +Write-Host "Publishing $Project with profile $PublishProfile ($Configuration)..." +dotnet publish $Project -c $Configuration -p:PublishProfile=$PublishProfile + +$projectDirectory = Split-Path -Parent $Project +$publishDirectory = Join-Path $projectDirectory "bin\$Configuration\net10.0\publish" +$webConfigPath = Join-Path $publishDirectory "web.config" + +if (-not (Test-Path $publishDirectory)) { + throw "Publish output folder not found: $publishDirectory" +} + +if (-not (Test-Path $webConfigPath)) { + throw "Missing web.config in publish output: $webConfigPath" +} + +Write-Host "Publish output is ready:" +Write-Host " $publishDirectory" +Write-Host "web.config exists and IIS deployment prerequisites are satisfied." diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..66011beab --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,45 @@ +# Scripts + +## `Publish-IIS.ps1` + +Publishes the BaGetter host project for IIS deployments and validates that `web.config` exists in publish output. + +Usage: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/Publish-IIS.ps1 +``` + +Optional parameters: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/Publish-IIS.ps1 ` + -Configuration Release ` + -Project src/BaGetter/BaGetter.csproj ` + -PublishProfile FolderProfile +``` + +Expected publish directory: + +```text +src/BaGetter/bin/Release/net10.0/publish/ +``` + +## `SmokeTest-BaGetter.ps1` + +Runs basic endpoint checks against a deployed BaGetter instance. + +Usage: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/SmokeTest-BaGetter.ps1 -BaseUrl https://your-bagetter-host +``` + +Optional parameters: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/SmokeTest-BaGetter.ps1 ` + -BaseUrl https://your-bagetter-host ` + -PackageId TestData ` + -PackageVersion 1.2.3 +``` diff --git a/scripts/SmokeTest-BaGetter.ps1 b/scripts/SmokeTest-BaGetter.ps1 new file mode 100644 index 000000000..3a842dbbe --- /dev/null +++ b/scripts/SmokeTest-BaGetter.ps1 @@ -0,0 +1,47 @@ +param( + [Parameter(Mandatory = $true)] + [string]$BaseUrl, + [string]$PackageId = "TestData", + [string]$PackageVersion = "1.2.3" +) + +$ErrorActionPreference = "Stop" + +function Test-Endpoint { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [Parameter(Mandatory = $true)] + [string]$Url + ) + + try { + $response = Invoke-WebRequest -Uri $Url -Method GET -UseBasicParsing + Write-Host "[PASS] $Name => $($response.StatusCode) $Url" + } + catch { + $status = $_.Exception.Response.StatusCode.value__ + if ($status) { + Write-Host "[FAIL] $Name => $status $Url" + } + else { + Write-Host "[FAIL] $Name => request error $Url" + } + throw + } +} + +$base = $BaseUrl.TrimEnd('/') + +Write-Host "Running BaGetter smoke tests against: $base" + +Test-Endpoint -Name "Service Index" -Url "$base/v3/index.json" +Test-Endpoint -Name "Health" -Url "$base/health" +Test-Endpoint -Name "Search" -Url "$base/v3/search?q=$PackageId" +Test-Endpoint -Name "Registration Index" -Url "$base/v3/registration/$PackageId/index.json" +Test-Endpoint -Name "Package Versions" -Url "$base/v3/package/$PackageId/index.json" + +$packageUrl = "$base/v3/package/$PackageId/$PackageVersion/$PackageId.$PackageVersion.nupkg" +Test-Endpoint -Name "Package Download" -Url $packageUrl + +Write-Host "Smoke tests completed." diff --git a/src/BaGetter.Aliyun/BaGetter.Aliyun.csproj b/src/BaGetter.Aliyun/BaGetter.Aliyun.csproj index 1eda2bc73..bc6e50b1b 100644 --- a/src/BaGetter.Aliyun/BaGetter.Aliyun.csproj +++ b/src/BaGetter.Aliyun/BaGetter.Aliyun.csproj @@ -2,7 +2,7 @@ NuGet;Alibaba;Cloud The libraries to host BaGetter on Alibaba Cloud (Aliyun). - net9.0 + net10.0 @@ -14,3 +14,4 @@ + diff --git a/src/BaGetter.Aws/BaGetter.Aws.csproj b/src/BaGetter.Aws/BaGetter.Aws.csproj index 6e9e91ca4..d1b354294 100644 --- a/src/BaGetter.Aws/BaGetter.Aws.csproj +++ b/src/BaGetter.Aws/BaGetter.Aws.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 NuGet;Amazon;Cloud The libraries to host BaGetter on AWS. @@ -18,3 +18,4 @@ + diff --git a/src/BaGetter.Azure/BaGetter.Azure.csproj b/src/BaGetter.Azure/BaGetter.Azure.csproj index ce89a11f3..055208502 100644 --- a/src/BaGetter.Azure/BaGetter.Azure.csproj +++ b/src/BaGetter.Azure/BaGetter.Azure.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 NuGet;Azure;Cloud The libraries to host BaGetter on Azure. @@ -23,3 +23,4 @@ + diff --git a/src/BaGetter.Azure/Table/TablePackageDatabase.cs b/src/BaGetter.Azure/Table/TablePackageDatabase.cs index 9bf5341d7..176cfbaad 100644 --- a/src/BaGetter.Azure/Table/TablePackageDatabase.cs +++ b/src/BaGetter.Azure/Table/TablePackageDatabase.cs @@ -152,6 +152,60 @@ public async Task> FindAsync(string id, bool includeUnlis return results.OrderBy(p => p.Version).ToList(); } + public async Task> SearchAsync( + string query, + bool? listed, + int skip, + int take, + CancellationToken cancellationToken) + { + var packages = new List(); + await foreach (var entity in _table.QueryAsync(cancellationToken: cancellationToken)) + { + if (!string.IsNullOrWhiteSpace(query) && + !entity.PartitionKey.Contains(query, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (listed.HasValue && entity.Listed != listed.Value) + { + continue; + } + + packages.Add(entity.AsPackage()); + } + + return packages + .OrderBy(p => p.Id, StringComparer.OrdinalIgnoreCase) + .ThenByDescending(p => p.Published) + .Skip(skip) + .Take(take) + .ToList(); + } + + public async Task CountAsync(string query, bool? listed, CancellationToken cancellationToken) + { + var count = 0; + await foreach (var entity in _table.QueryAsync( + select: ["PartitionKey", "Listed"], + cancellationToken: cancellationToken)) + { + if (!string.IsNullOrWhiteSpace(query) && + !entity.PartitionKey.Contains(query, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!listed.HasValue || entity.Listed == listed.Value) + { + count++; + } + } + + return count; + } + public async Task FindOrNullAsync( string id, NuGetVersion version, diff --git a/src/BaGetter.Core/Authentication/ApiKeyAuthenticationService.cs b/src/BaGetter.Core/Authentication/ApiKeyAuthenticationService.cs index 1a7cfb54b..2ffc0ff1e 100644 --- a/src/BaGetter.Core/Authentication/ApiKeyAuthenticationService.cs +++ b/src/BaGetter.Core/Authentication/ApiKeyAuthenticationService.cs @@ -10,6 +10,7 @@ namespace BaGetter.Core; public class ApiKeyAuthenticationService : IAuthenticationService { private readonly string _apiKey; + private readonly string _apiKeyHash; private readonly ApiKey[] _apiKeys; public ApiKeyAuthenticationService(IOptionsSnapshot options) @@ -17,6 +18,7 @@ public ApiKeyAuthenticationService(IOptionsSnapshot options) ArgumentNullException.ThrowIfNull(options); _apiKey = string.IsNullOrEmpty(options.Value.ApiKey) ? null : options.Value.ApiKey; + _apiKeyHash = string.IsNullOrEmpty(options.Value.ApiKeyHash) ? null : options.Value.ApiKeyHash; _apiKeys = options.Value.Authentication?.ApiKeys ?? []; } @@ -25,9 +27,25 @@ public Task AuthenticateAsync(string apiKey, CancellationToken cancellatio private bool Authenticate(string apiKey) { + var hasAuthKeys = !string.IsNullOrWhiteSpace(_apiKey) || + !string.IsNullOrWhiteSpace(_apiKeyHash) || + _apiKeys.Any(x => !string.IsNullOrWhiteSpace(x.Key) || !string.IsNullOrWhiteSpace(x.KeyHash)); + // No authentication is necessary if there is no required API key. - if (_apiKey == null && (_apiKeys.Length == 0)) return true; + if (!hasAuthKeys) return true; + + if (_apiKey == apiKey) + { + return true; + } + + if (SecretHashing.VerifySecret(apiKey, _apiKeyHash)) + { + return true; + } - return _apiKey == apiKey || _apiKeys.Any(x => x.Key.Equals(apiKey)); + return _apiKeys.Any(x => + (!string.IsNullOrEmpty(x.Key) && x.Key.Equals(apiKey)) || + SecretHashing.VerifySecret(apiKey, x.KeyHash)); } } diff --git a/src/BaGetter.Core/Authentication/AuthenticationConstants.cs b/src/BaGetter.Core/Authentication/AuthenticationConstants.cs index f6ff708ea..2e53f03fd 100644 --- a/src/BaGetter.Core/Authentication/AuthenticationConstants.cs +++ b/src/BaGetter.Core/Authentication/AuthenticationConstants.cs @@ -9,4 +9,6 @@ public static class AuthenticationConstants { public const string NugetBasicAuthenticationScheme = "NugetBasicAuthentication"; public const string NugetUserPolicy = "NuGetUserPolicy"; + public const string AdminPolicy = "AdminPolicy"; + public const string AdminRole = "Admin"; } diff --git a/src/BaGetter.Core/Authentication/ConfiguredCredentialValidator.cs b/src/BaGetter.Core/Authentication/ConfiguredCredentialValidator.cs new file mode 100644 index 000000000..56e6d8683 --- /dev/null +++ b/src/BaGetter.Core/Authentication/ConfiguredCredentialValidator.cs @@ -0,0 +1,42 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; + +namespace BaGetter.Core; + +public sealed class ConfiguredCredentialValidator : INugetCredentialValidator +{ + private readonly NugetCredentials[] _credentials; + + public ConfiguredCredentialValidator(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _credentials = options.Value.Authentication?.Credentials ?? []; + } + + public Task ValidateAsync( + string username, + string password, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrEmpty(password)) + { + return Task.FromResult(null); + } + + var credential = _credentials.FirstOrDefault(a => + !string.IsNullOrWhiteSpace(a.Username) && + a.Username.Equals(username, StringComparison.OrdinalIgnoreCase) && + ( + (!string.IsNullOrEmpty(a.PasswordHash) && SecretHashing.VerifySecret(password, a.PasswordHash)) || + a.Password == password + )); + + return Task.FromResult( + credential == null + ? null + : new NugetCredentialValidationResult(credential.Username, credential.Roles)); + } +} diff --git a/src/BaGetter.Core/Authentication/INugetCredentialValidator.cs b/src/BaGetter.Core/Authentication/INugetCredentialValidator.cs new file mode 100644 index 000000000..ddecd10ae --- /dev/null +++ b/src/BaGetter.Core/Authentication/INugetCredentialValidator.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace BaGetter.Core; + +public interface INugetCredentialValidator +{ + Task ValidateAsync( + string username, + string password, + CancellationToken cancellationToken); +} diff --git a/src/BaGetter.Core/Authentication/LdapCredentialValidator.cs b/src/BaGetter.Core/Authentication/LdapCredentialValidator.cs new file mode 100644 index 000000000..67c4df2ac --- /dev/null +++ b/src/BaGetter.Core/Authentication/LdapCredentialValidator.cs @@ -0,0 +1,275 @@ +using System; +using System.DirectoryServices.Protocols; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace BaGetter.Core; + +public sealed class LdapCredentialValidator : INugetCredentialValidator +{ + private readonly IOptions _options; + private readonly ILogger _logger; + + public LdapCredentialValidator( + IOptions options, + ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task ValidateAsync( + string username, + string password, + CancellationToken cancellationToken) + { + var ldap = _options.Value.Authentication?.Ldap; + if (!IsEnabled(ldap) || string.IsNullOrWhiteSpace(username) || string.IsNullOrEmpty(password)) + { + return Task.FromResult(null); + } + + try + { + var identifier = LdapUserIdentifier.Parse(username); + var userEntry = FindUserEntry(ldap, identifier, username, password); + if (userEntry == null) + { + return Task.FromResult(null); + } + + if (!BindAsUser(ldap, userEntry.DistinguishedName, password)) + { + return Task.FromResult(null); + } + + if (!IsAllowedGroupMember(ldap, userEntry.GroupNames)) + { + return Task.FromResult(null); + } + + var resolvedUsername = !string.IsNullOrWhiteSpace(userEntry.SamAccountName) + ? userEntry.SamAccountName + : !string.IsNullOrWhiteSpace(userEntry.Uid) + ? userEntry.Uid + : username; + + return Task.FromResult(new NugetCredentialValidationResult(resolvedUsername, userEntry.GroupNames)); + } + catch (LdapException ex) + { + _logger.LogInformation(ex, "LDAP authentication failed for user {Username}", username); + return Task.FromResult(null); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unexpected LDAP authentication failure for user {Username}", username); + return Task.FromResult(null); + } + } + + private static bool IsEnabled(LdapAuthenticationOptions ldap) + => ldap?.Enabled == true && + !string.IsNullOrWhiteSpace(ldap.Server) && + !string.IsNullOrWhiteSpace(ldap.BaseDn); + + private LdapUserEntry FindUserEntry( + LdapAuthenticationOptions ldap, + LdapUserIdentifier identifier, + string username, + string password) + { + using var connection = CreateConnection(ldap); + + if (!TryBindSearchConnection(connection, ldap, username, password)) + { + return null; + } + + var searchBase = string.IsNullOrWhiteSpace(ldap.UserSearchBase) ? ldap.BaseDn : ldap.UserSearchBase; + var searchFilter = BuildSearchFilter(ldap, identifier); + + var request = new System.DirectoryServices.Protocols.SearchRequest( + searchBase, + searchFilter, + SearchScope.Subtree, + new[] { "distinguishedName", "sAMAccountName", "uid", "memberOf" }); + + var response = (SearchResponse)connection.SendRequest(request); + var entry = response.Entries.Cast().FirstOrDefault(); + if (entry == null) + { + return null; + } + + return new LdapUserEntry( + entry.DistinguishedName, + entry.Attributes["sAMAccountName"]?[0]?.ToString(), + entry.Attributes["uid"]?[0]?.ToString(), + GetGroupNames(entry.Attributes["memberOf"])); + } + + private bool TryBindSearchConnection( + LdapConnection connection, + LdapAuthenticationOptions ldap, + string username, + string password) + { + if (!string.IsNullOrWhiteSpace(ldap.BindDn)) + { + connection.Bind(new NetworkCredential(ldap.BindDn, ldap.BindPassword)); + return true; + } + + try + { + connection.Bind(); + return true; + } + catch (LdapException ex) + { + _logger.LogDebug(ex, "Anonymous LDAP search bind failed. Falling back to the presented credentials."); + } + + connection.Bind(new NetworkCredential(username, password)); + return true; + } + + private bool BindAsUser(LdapAuthenticationOptions ldap, string distinguishedName, string password) + { + using var connection = CreateConnection(ldap); + connection.Bind(new NetworkCredential(distinguishedName, password)); + return true; + } + + private static LdapConnection CreateConnection(LdapAuthenticationOptions ldap) + { + var identifier = ldap.Port.HasValue + ? new LdapDirectoryIdentifier(ldap.Server, ldap.Port.Value) + : new LdapDirectoryIdentifier(ldap.Server, ldap.UseSsl ? 636 : 389); + + var connection = new LdapConnection(identifier) + { + AuthType = AuthType.Basic, + Timeout = TimeSpan.FromSeconds(15), + }; + + connection.SessionOptions.ProtocolVersion = 3; + connection.SessionOptions.ReferralChasing = ReferralChasingOptions.None; + connection.SessionOptions.SecureSocketLayer = ldap.UseSsl; + + return connection; + } + + private static string BuildSearchFilter(LdapAuthenticationOptions ldap, LdapUserIdentifier identifier) + { + if (!string.IsNullOrWhiteSpace(ldap?.UserSearchFilter)) + { + return ldap.UserSearchFilter.Replace("%s", EscapeFilterValue(identifier.AccountName), StringComparison.Ordinal); + } + + var escapedAccountName = EscapeFilterValue(identifier.AccountName); + if (!string.IsNullOrWhiteSpace(identifier.UserPrincipalName)) + { + return $"(|(sAMAccountName={escapedAccountName})(uid={escapedAccountName})(userPrincipalName={EscapeFilterValue(identifier.UserPrincipalName)}))"; + } + + return $"(|(sAMAccountName={escapedAccountName})(uid={escapedAccountName}))"; + } + + private static string EscapeFilterValue(string value) + { + return value + .Replace("\\", "\\5c", StringComparison.Ordinal) + .Replace("*", "\\2a", StringComparison.Ordinal) + .Replace("(", "\\28", StringComparison.Ordinal) + .Replace(")", "\\29", StringComparison.Ordinal) + .Replace("\0", "\\00", StringComparison.Ordinal); + } + + internal static string[] GetGroupNames(DirectoryAttribute attribute) + { + if (attribute == null || attribute.Count == 0) + { + return []; + } + + var groups = new List(attribute.Count); + foreach (var item in attribute) + { + var distinguishedName = item?.ToString(); + var groupName = ExtractCommonName(distinguishedName); + if (!string.IsNullOrWhiteSpace(groupName)) + { + groups.Add(groupName); + } + } + + return groups + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + internal static string ExtractCommonName(string distinguishedName) + { + if (string.IsNullOrWhiteSpace(distinguishedName)) + { + return null; + } + + foreach (var part in distinguishedName.Split(',')) + { + var trimmed = part.Trim(); + if (trimmed.StartsWith("CN=", StringComparison.OrdinalIgnoreCase) && trimmed.Length > 3) + { + return trimmed[3..]; + } + } + + return null; + } + + internal static bool IsAllowedGroupMember(LdapAuthenticationOptions ldap, string[] groupNames) + { + var allowedGroups = ldap?.AllowedGroups? + .Where(g => !string.IsNullOrWhiteSpace(g)) + .Select(g => g.Trim()) + .ToArray(); + + if (allowedGroups == null || allowedGroups.Length == 0) + { + return true; + } + + if (groupNames == null || groupNames.Length == 0) + { + return false; + } + + return groupNames.Intersect(allowedGroups, StringComparer.OrdinalIgnoreCase).Any(); + } + + private sealed class LdapUserEntry + { + public LdapUserEntry(string distinguishedName, string samAccountName, string uid, string[] groupNames) + { + DistinguishedName = distinguishedName; + SamAccountName = samAccountName; + Uid = uid; + GroupNames = groupNames ?? []; + } + + public string DistinguishedName { get; } + + public string SamAccountName { get; } + + public string Uid { get; } + + public string[] GroupNames { get; } + } +} diff --git a/src/BaGetter.Core/Authentication/LdapUserIdentifier.cs b/src/BaGetter.Core/Authentication/LdapUserIdentifier.cs new file mode 100644 index 000000000..6e63ce5ec --- /dev/null +++ b/src/BaGetter.Core/Authentication/LdapUserIdentifier.cs @@ -0,0 +1,47 @@ +using System; + +namespace BaGetter.Core; + +internal readonly struct LdapUserIdentifier +{ + private LdapUserIdentifier( + string rawUsername, + string accountName, + string userPrincipalName) + { + RawUsername = rawUsername; + AccountName = accountName; + UserPrincipalName = userPrincipalName; + } + + public string RawUsername { get; } + + public string AccountName { get; } + + public string UserPrincipalName { get; } + + public static LdapUserIdentifier Parse(string username) + { + var value = username?.Trim() ?? string.Empty; + + var separatorIndex = value.IndexOf('\\'); + if (separatorIndex > 0 && separatorIndex < value.Length - 1) + { + return new LdapUserIdentifier( + value, + value[(separatorIndex + 1)..], + null); + } + + var atIndex = value.IndexOf('@'); + if (atIndex > 0 && atIndex < value.Length - 1) + { + return new LdapUserIdentifier( + value, + value[..atIndex], + value); + } + + return new LdapUserIdentifier(value, value, null); + } +} diff --git a/src/BaGetter.Core/Authentication/NugetCredentialValidationResult.cs b/src/BaGetter.Core/Authentication/NugetCredentialValidationResult.cs new file mode 100644 index 000000000..876a3d6e5 --- /dev/null +++ b/src/BaGetter.Core/Authentication/NugetCredentialValidationResult.cs @@ -0,0 +1,14 @@ +namespace BaGetter.Core; + +public sealed class NugetCredentialValidationResult +{ + public NugetCredentialValidationResult(string username, string[] roles = null) + { + Username = username; + Roles = roles ?? []; + } + + public string Username { get; } + + public string[] Roles { get; } +} diff --git a/src/BaGetter.Core/Authentication/SecretHashing.cs b/src/BaGetter.Core/Authentication/SecretHashing.cs new file mode 100644 index 000000000..0819e48a8 --- /dev/null +++ b/src/BaGetter.Core/Authentication/SecretHashing.cs @@ -0,0 +1,75 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace BaGetter.Core; + +/// +/// Utilities for hashing and verifying secrets used in configuration. +/// +public static class SecretHashing +{ + private const string Prefix = "PBKDF2"; + private const int DefaultIterations = 100_000; + private const int SaltSize = 16; + private const int HashSize = 32; + + public static string HashSecret(string secret, int iterations = DefaultIterations) + { + ArgumentException.ThrowIfNullOrWhiteSpace(secret); + if (iterations < 10_000) + { + throw new ArgumentOutOfRangeException(nameof(iterations), "Iterations must be >= 10000."); + } + + Span salt = stackalloc byte[SaltSize]; + RandomNumberGenerator.Fill(salt); + + var hash = Rfc2898DeriveBytes.Pbkdf2( + Encoding.UTF8.GetBytes(secret), + salt.ToArray(), + iterations, + HashAlgorithmName.SHA256, + HashSize); + + return $"{Prefix}${iterations}${Convert.ToBase64String(salt)}${Convert.ToBase64String(hash)}"; + } + + public static bool VerifySecret(string secret, string hashValue) + { + if (string.IsNullOrWhiteSpace(secret) || string.IsNullOrWhiteSpace(hashValue)) + { + return false; + } + + var parts = hashValue.Split('$', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 4 || !string.Equals(parts[0], Prefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (!int.TryParse(parts[1], out var iterations) || iterations < 10_000) + { + return false; + } + + try + { + var salt = Convert.FromBase64String(parts[2]); + var expectedHash = Convert.FromBase64String(parts[3]); + + var actualHash = Rfc2898DeriveBytes.Pbkdf2( + Encoding.UTF8.GetBytes(secret), + salt, + iterations, + HashAlgorithmName.SHA256, + expectedHash.Length); + + return CryptographicOperations.FixedTimeEquals(actualHash, expectedHash); + } + catch (FormatException) + { + return false; + } + } +} diff --git a/src/BaGetter.Core/BaGetter.Core.csproj b/src/BaGetter.Core/BaGetter.Core.csproj index 04d25e6da..86fca7c75 100644 --- a/src/BaGetter.Core/BaGetter.Core.csproj +++ b/src/BaGetter.Core/BaGetter.Core.csproj @@ -1,6 +1,6 @@ - net9.0 + net10.0 NuGet The core libraries that power BaGetter. @@ -14,7 +14,7 @@ - + @@ -22,3 +22,4 @@ + diff --git a/src/BaGetter.Core/Configuration/AdminOptions.cs b/src/BaGetter.Core/Configuration/AdminOptions.cs new file mode 100644 index 000000000..dee67215e --- /dev/null +++ b/src/BaGetter.Core/Configuration/AdminOptions.cs @@ -0,0 +1,9 @@ +namespace BaGetter.Core; + +public sealed class AdminOptions +{ + /// + /// Path to the JSON Lines audit log. Relative paths are resolved from the content root. + /// + public string AuditLogPath { get; set; } = "App_Data/admin-audit.jsonl"; +} diff --git a/src/BaGetter.Core/Configuration/ApiKey.cs b/src/BaGetter.Core/Configuration/ApiKey.cs index bb94f661a..e10a98ce5 100644 --- a/src/BaGetter.Core/Configuration/ApiKey.cs +++ b/src/BaGetter.Core/Configuration/ApiKey.cs @@ -8,4 +8,11 @@ namespace BaGetter.Core.Configuration; public class ApiKey { public string Key { get; set; } + + /// + /// Optional API key hash in format: + /// PBKDF2$<iterations>$<base64Salt>$<base64Hash> + /// If set, this is used instead of . + /// + public string KeyHash { get; set; } } diff --git a/src/BaGetter.Core/Configuration/BaGetterOptions.cs b/src/BaGetter.Core/Configuration/BaGetterOptions.cs index 8b2e94648..dbc797d3c 100644 --- a/src/BaGetter.Core/Configuration/BaGetterOptions.cs +++ b/src/BaGetter.Core/Configuration/BaGetterOptions.cs @@ -1,9 +1,12 @@ using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; using BaGetter.Core.Configuration; namespace BaGetter.Core; -public class BaGetterOptions +public class BaGetterOptions : IValidatableObject { /// /// The API Key required to authenticate package @@ -11,6 +14,13 @@ public class BaGetterOptions /// public string ApiKey { get; set; } + /// + /// Optional hash for the legacy in format: + /// PBKDF2$<iterations>$<base64Salt>$<base64Hash>. + /// If set, this is used in addition to . + /// + public string ApiKeyHash { get; set; } + /// /// The application root URL for usage in reverse proxy scenarios. /// @@ -50,6 +60,13 @@ public class BaGetterOptions /// public uint MaxPackageSizeGiB { get; set; } = 8; + /// + /// The number of package versions to include in a single registration page. + /// If a package has more versions than this value, the registration index will return paged entries. + /// + [Range(1, int.MaxValue)] + public int RegistrationPageSize { get; set; } = 64; + /// /// If this is set to a value, it will limit the number of versions that can be pushed for a package. /// the older versions will be deleted. @@ -68,9 +85,155 @@ public class BaGetterOptions public MirrorOptions Mirror { get; set; } + /// + /// Multiple mirrors to use for read-through caching. When this list is set and non-empty, + /// it takes precedence over . + /// + public IList Mirrors { get; set; } + public HealthCheckOptions HealthCheck { get; set; } public StatisticsOptions Statistics { get; set; } + public RequestRateLimitOptions RequestRateLimit { get; set; } = new(); + + public CorsPolicyOptions Cors { get; set; } = new(); + + public SecurityHeadersOptions SecurityHeaders { get; set; } = new(); + + public LocalizationOptions Localization { get; set; } = new(); + + public SearchReindexOptions Reindex { get; set; } = new(); + public NugetAuthenticationOptions Authentication { get; set; } + + public AdminOptions Admin { get; set; } = new(); + + public IEnumerable Validate(ValidationContext validationContext) + { + if (MaxPackageSizeGiB == 0) + { + yield return new ValidationResult( + $"{nameof(MaxPackageSizeGiB)} must be greater than 0.", + new[] { nameof(MaxPackageSizeGiB) }); + } + + if (!string.IsNullOrEmpty(PathBase)) + { + if (!PathBase.StartsWith('/')) + { + yield return new ValidationResult( + $"{nameof(PathBase)} must start with '/'.", + new[] { nameof(PathBase) }); + } + + if (PathBase.Length > 1 && PathBase.EndsWith('/')) + { + yield return new ValidationResult( + $"{nameof(PathBase)} must not end with '/' unless it is '/'.", + new[] { nameof(PathBase) }); + } + } + + if (Cors != null) + { + if (Cors.AllowAnyOrigin && Cors.AllowCredentials) + { + yield return new ValidationResult( + $"{nameof(Cors)}.{nameof(CorsPolicyOptions.AllowCredentials)} cannot be true when {nameof(Cors)}.{nameof(CorsPolicyOptions.AllowAnyOrigin)} is true.", + new[] { nameof(Cors) }); + } + + if (!Cors.AllowAnyOrigin && (Cors.AllowedOrigins == null || Cors.AllowedOrigins.Length == 0)) + { + yield return new ValidationResult( + $"{nameof(Cors)}.{nameof(CorsPolicyOptions.AllowedOrigins)} must contain at least one origin when {nameof(Cors)}.{nameof(CorsPolicyOptions.AllowAnyOrigin)} is false.", + new[] { nameof(Cors) }); + } + } + + if (SecurityHeaders is { EnableHsts: true, Enabled: false }) + { + yield return new ValidationResult( + $"{nameof(SecurityHeaders)}.{nameof(SecurityHeadersOptions.EnableHsts)} requires {nameof(SecurityHeaders)}.{nameof(SecurityHeadersOptions.Enabled)} to be true.", + new[] { nameof(SecurityHeaders) }); + } + + if (Authentication?.Ldap?.Enabled == true) + { + if (string.IsNullOrWhiteSpace(Authentication.Ldap.Server)) + { + yield return new ValidationResult( + $"{nameof(Authentication)}.{nameof(NugetAuthenticationOptions.Ldap)}.{nameof(LdapAuthenticationOptions.Server)} is required when LDAP authentication is enabled.", + [nameof(Authentication)]); + } + + if (string.IsNullOrWhiteSpace(Authentication.Ldap.BaseDn)) + { + yield return new ValidationResult( + $"{nameof(Authentication)}.{nameof(NugetAuthenticationOptions.Ldap)}.{nameof(LdapAuthenticationOptions.BaseDn)} is required when LDAP authentication is enabled.", + [nameof(Authentication)]); + } + + var hasBindDn = !string.IsNullOrWhiteSpace(Authentication.Ldap.BindDn); + var hasBindPassword = !string.IsNullOrWhiteSpace(Authentication.Ldap.BindPassword); + if (hasBindDn != hasBindPassword) + { + yield return new ValidationResult( + $"{nameof(Authentication)}.{nameof(NugetAuthenticationOptions.Ldap)} requires both {nameof(LdapAuthenticationOptions.BindDn)} and {nameof(LdapAuthenticationOptions.BindPassword)} when either is configured.", + [nameof(Authentication)]); + } + + if (Authentication.Ldap.Port is <= 0 or > 65535) + { + yield return new ValidationResult( + $"{nameof(Authentication)}.{nameof(NugetAuthenticationOptions.Ldap)}.{nameof(LdapAuthenticationOptions.Port)} must be between 1 and 65535 when set.", + [nameof(Authentication)]); + } + } + + var mirrors = GetConfiguredMirrors(); + var useMirrorList = Mirrors is { Count: > 0 }; + for (var i = 0; i < mirrors.Count; i++) + { + var mirror = mirrors[i]; + var prefix = useMirrorList ? $"{nameof(Mirrors)}[{i}]" : nameof(Mirror); + + if (mirror == null) + { + yield return new ValidationResult( + $"{prefix} must not be null.", + [prefix]); + continue; + } + + var mirrorValidationResults = new List(); + Validator.TryValidateObject( + mirror, + new ValidationContext(mirror), + mirrorValidationResults, + validateAllProperties: true); + + foreach (var validationResult in mirrorValidationResults) + { + var members = validationResult.MemberNames?.Any() == true + ? validationResult.MemberNames.Select(m => $"{prefix}.{m}") + : [prefix]; + + yield return new ValidationResult( + $"{prefix}: {validationResult.ErrorMessage}", + members); + } + } + } + + public IReadOnlyList GetConfiguredMirrors() + { + if (Mirrors is { Count: > 0 }) + { + return Mirrors.ToList(); + } + + return Mirror == null ? Array.Empty() : [Mirror]; + } } diff --git a/src/BaGetter.Core/Configuration/CorsPolicyOptions.cs b/src/BaGetter.Core/Configuration/CorsPolicyOptions.cs new file mode 100644 index 000000000..69c612f52 --- /dev/null +++ b/src/BaGetter.Core/Configuration/CorsPolicyOptions.cs @@ -0,0 +1,20 @@ +using System; + +namespace BaGetter.Core; + +public class CorsPolicyOptions +{ + public bool AllowAnyOrigin { get; set; } = true; + + public string[] AllowedOrigins { get; set; } = Array.Empty(); + + public bool AllowAnyMethod { get; set; } = true; + + public string[] AllowedMethods { get; set; } = Array.Empty(); + + public bool AllowAnyHeader { get; set; } = true; + + public string[] AllowedHeaders { get; set; } = Array.Empty(); + + public bool AllowCredentials { get; set; } = false; +} diff --git a/src/BaGetter.Core/Configuration/LdapAuthenticationOptions.cs b/src/BaGetter.Core/Configuration/LdapAuthenticationOptions.cs new file mode 100644 index 000000000..a83cba53f --- /dev/null +++ b/src/BaGetter.Core/Configuration/LdapAuthenticationOptions.cs @@ -0,0 +1,24 @@ +namespace BaGetter.Core; + +public sealed class LdapAuthenticationOptions +{ + public bool Enabled { get; set; } + + public string Server { get; set; } + + public int? Port { get; set; } + + public string BaseDn { get; set; } + + public string UserSearchBase { get; set; } + + public string UserSearchFilter { get; set; } + + public string BindDn { get; set; } + + public string BindPassword { get; set; } + + public bool UseSsl { get; set; } = true; + + public string[] AllowedGroups { get; set; } = []; +} diff --git a/src/BaGetter.Core/Configuration/LocalizationOptions.cs b/src/BaGetter.Core/Configuration/LocalizationOptions.cs new file mode 100644 index 000000000..212d37705 --- /dev/null +++ b/src/BaGetter.Core/Configuration/LocalizationOptions.cs @@ -0,0 +1,10 @@ +namespace BaGetter.Core.Configuration; + +public class LocalizationOptions +{ + public static readonly string[] SupportedCultures = ["en-US", "is-IS", "da-DK", "sv-SE", "nb-NO", "pl-PL", "fi-FI", "es-ES", "pt-PT", "it-IT", "zh-CN", "fr-FR"]; + + public bool Enabled { get; set; } = false; + + public string DefaultCulture { get; set; } = "en-US"; +} diff --git a/src/BaGetter.Core/Configuration/NugetAuthenticationOptions.cs b/src/BaGetter.Core/Configuration/NugetAuthenticationOptions.cs index 4d608fd8b..ad72d5982 100644 --- a/src/BaGetter.Core/Configuration/NugetAuthenticationOptions.cs +++ b/src/BaGetter.Core/Configuration/NugetAuthenticationOptions.cs @@ -13,4 +13,9 @@ public sealed class NugetAuthenticationOptions /// Api keys for pushing packages into the feed. /// public ApiKey[] ApiKeys { get; set; } + + /// + /// LDAP authentication settings for downloading packages. + /// + public LdapAuthenticationOptions Ldap { get; set; } } diff --git a/src/BaGetter.Core/Configuration/NugetCredentials.cs b/src/BaGetter.Core/Configuration/NugetCredentials.cs index 5cd5579bf..f9d113192 100644 --- a/src/BaGetter.Core/Configuration/NugetCredentials.cs +++ b/src/BaGetter.Core/Configuration/NugetCredentials.cs @@ -5,4 +5,16 @@ public sealed class NugetCredentials public string Username { get; set; } public string Password { get; set; } + + /// + /// Optional list of roles assigned to this user (for example: "Admin"). + /// + public string[] Roles { get; set; } = []; + + /// + /// Optional password hash in format: + /// PBKDF2$<iterations>$<base64Salt>$<base64Hash> + /// If set, this is used instead of . + /// + public string PasswordHash { get; set; } } diff --git a/src/BaGetter.Core/Configuration/RequestRateLimitOptions.cs b/src/BaGetter.Core/Configuration/RequestRateLimitOptions.cs new file mode 100644 index 000000000..706b43e26 --- /dev/null +++ b/src/BaGetter.Core/Configuration/RequestRateLimitOptions.cs @@ -0,0 +1,17 @@ +using System.ComponentModel.DataAnnotations; + +namespace BaGetter.Core; + +public class RequestRateLimitOptions +{ + public bool Enabled { get; set; } = false; + + [Range(1, int.MaxValue)] + public int PermitLimit { get; set; } = 120; + + [Range(1, int.MaxValue)] + public int WindowSeconds { get; set; } = 60; + + [Range(0, int.MaxValue)] + public int QueueLimit { get; set; } = 0; +} diff --git a/src/BaGetter.Core/Configuration/SearchReindexOptions.cs b/src/BaGetter.Core/Configuration/SearchReindexOptions.cs new file mode 100644 index 000000000..091a7a053 --- /dev/null +++ b/src/BaGetter.Core/Configuration/SearchReindexOptions.cs @@ -0,0 +1,16 @@ +using System.ComponentModel.DataAnnotations; + +namespace BaGetter.Core; + +public class SearchReindexOptions +{ + public bool Enabled { get; set; } = false; + + public bool RunOnStartup { get; set; } = false; + + [Range(0, int.MaxValue)] + public int IntervalMinutes { get; set; } = 0; + + [Range(1, int.MaxValue)] + public int BatchSize { get; set; } = 100; +} diff --git a/src/BaGetter.Core/Configuration/SecurityHeadersOptions.cs b/src/BaGetter.Core/Configuration/SecurityHeadersOptions.cs new file mode 100644 index 000000000..acb890290 --- /dev/null +++ b/src/BaGetter.Core/Configuration/SecurityHeadersOptions.cs @@ -0,0 +1,17 @@ +using System.ComponentModel.DataAnnotations; + +namespace BaGetter.Core; + +public class SecurityHeadersOptions +{ + public bool Enabled { get; set; } = true; + + public bool EnableHsts { get; set; } = false; + + [Range(1, 3650)] + public int HstsMaxAgeDays { get; set; } = 365; + + public bool HstsIncludeSubDomains { get; set; } = true; + + public bool HstsPreload { get; set; } = false; +} diff --git a/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs b/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs index e4d71e85e..4f5d23998 100644 --- a/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs +++ b/src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -9,6 +10,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace BaGetter.Core; @@ -72,6 +74,10 @@ private static void AddConfiguration(this IServiceCollection services) services.AddBaGetterOptions(nameof(BaGetterOptions.Search)); services.AddBaGetterOptions(nameof(BaGetterOptions.Storage)); services.AddBaGetterOptions(nameof(BaGetterOptions.Statistics)); + services.AddBaGetterOptions(nameof(BaGetterOptions.RequestRateLimit)); + services.AddBaGetterOptions(nameof(BaGetterOptions.Cors)); + services.AddBaGetterOptions(nameof(BaGetterOptions.SecurityHeaders)); + services.AddBaGetterOptions(nameof(BaGetterOptions.Reindex)); } private static void AddBaGetServices(this IServiceCollection services) @@ -93,6 +99,8 @@ private static void AddBaGetServices(this IServiceCollection services) services.TryAddScoped(); services.TryAddTransient(); + services.TryAddEnumerable(ServiceDescriptor.Transient()); + services.TryAddEnumerable(ServiceDescriptor.Transient()); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); @@ -100,6 +108,7 @@ private static void AddBaGetServices(this IServiceCollection services) services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); @@ -107,7 +116,12 @@ private static void AddBaGetServices(this IServiceCollection services) services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); - services.TryAddTransient(); + services.TryAddTransient(provider => + { + var options = provider.GetRequiredService>().Value; + var logger = provider.GetRequiredService>(); + return new V2UpstreamClient(options, logger); + }); services.TryAddTransient(); services.TryAddTransient(); services.TryAddSingleton(); @@ -225,13 +239,97 @@ private static NuGetClientFactory NuGetClientFactoryFactory(IServiceProvider pro private static IUpstreamClient UpstreamClientFactory(IServiceProvider provider) { - var options = provider.GetRequiredService>(); + var rootOptions = provider.GetRequiredService>().Value; + if (rootOptions.Mirrors is not { Count: > 0 }) + { + var legacyOptions = provider.GetRequiredService>().Value; + return legacyOptions.Enabled switch + { + false => provider.GetRequiredService(), + true when legacyOptions.Legacy => provider.GetRequiredService(), + _ => provider.GetRequiredService() + }; + } - return options.Value.Enabled switch + var enabledMirrors = new List(); + foreach (var mirror in rootOptions.Mirrors) { - false => provider.GetRequiredService(), - true when options.Value.Legacy => provider.GetRequiredService(), - _ => provider.GetRequiredService() + if (mirror?.Enabled != true) + { + continue; + } + + enabledMirrors.Add(CreateUpstreamClient(provider, mirror)); + } + + return enabledMirrors.Count switch + { + 0 => provider.GetRequiredService(), + 1 => enabledMirrors[0], + _ => new FallbackUpstreamClient( + enabledMirrors, + provider.GetRequiredService>()) }; } + + private static IUpstreamClient CreateUpstreamClient(IServiceProvider provider, MirrorOptions mirror) + { + if (mirror.Legacy) + { + return new V2UpstreamClient( + mirror, + provider.GetRequiredService>()); + } + + var nuGetClient = new NuGetClient(CreateNuGetClientFactory(provider, mirror)); + return new V3UpstreamClient( + nuGetClient, + provider.GetRequiredService>()); + } + + private static NuGetClientFactory CreateNuGetClientFactory(IServiceProvider provider, MirrorOptions mirror) + { + var httpClient = CreateMirrorHttpClient(mirror); + + if (mirror.Authentication is { } auth) + { + switch (auth.Type) + { + case MirrorAuthenticationType.Basic: + var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{auth.Username}:{auth.Password}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials); + break; + + case MirrorAuthenticationType.Bearer: + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", auth.Token); + break; + + case MirrorAuthenticationType.Custom: + foreach (var (header, value) in auth.CustomHeaders) + { + httpClient.DefaultRequestHeaders.Add(header, value); + } + break; + } + } + + return new NuGetClientFactory(httpClient, mirror.PackageSource.ToString()); + } + + private static HttpClient CreateMirrorHttpClient(MirrorOptions options) + { + var assembly = Assembly.GetEntryAssembly(); + var assemblyName = assembly.GetName().Name; + var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; + + var client = new HttpClient(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); + + client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); + client.Timeout = TimeSpan.FromSeconds(options.PackageDownloadTimeoutSeconds); + + return client; + } } diff --git a/src/BaGetter.Core/IPackageDatabase.cs b/src/BaGetter.Core/IPackageDatabase.cs index d855b06ec..02dde2c0f 100644 --- a/src/BaGetter.Core/IPackageDatabase.cs +++ b/src/BaGetter.Core/IPackageDatabase.cs @@ -42,6 +42,21 @@ Task FindOrNullAsync( /// The packages found. Always non-null. Task> FindAsync(string id, bool includeUnlisted, CancellationToken cancellationToken); + /// + /// Find package versions for administrative management. + /// + Task> SearchAsync( + string query, + bool? listed, + int skip, + int take, + CancellationToken cancellationToken); + + /// + /// Count package versions for administrative management. + /// + Task CountAsync(string query, bool? listed, CancellationToken cancellationToken); + /// /// Determine whether a package exists in the database (even if the package is unlisted). /// diff --git a/src/BaGetter.Core/Indexing/PackageIndexingService.cs b/src/BaGetter.Core/Indexing/PackageIndexingService.cs index 9dcfae19e..c15ebba70 100644 --- a/src/BaGetter.Core/Indexing/PackageIndexingService.cs +++ b/src/BaGetter.Core/Indexing/PackageIndexingService.cs @@ -96,10 +96,22 @@ public async Task IndexAsync(Stream packageStream, string } // The package is well-formed. Ensure this is a new package. - if (await _packages.ExistsAsync(package.Id, package.Version, cancellationToken)) + var existingPackage = await _packages.FindOrNullAsync( + package.Id, + package.Version, + includeUnlisted: true, + cancellationToken); + + if (existingPackage != null) { - if (_options.Value.AllowPackageOverwrites == PackageOverwriteAllowed.False || - (_options.Value.AllowPackageOverwrites == PackageOverwriteAllowed.PrereleaseOnly && !package.IsPrerelease)) + var canOverwrite = + _options.Value.AllowPackageOverwrites == PackageOverwriteAllowed.True || + (_options.Value.AllowPackageOverwrites == PackageOverwriteAllowed.PrereleaseOnly && package.IsPrerelease); + + // Allow re-push of soft-deleted (unlisted) packages by replacing them. + var canReplaceUnlisted = !existingPackage.Listed; + + if (!canOverwrite && !canReplaceUnlisted) { return PackageIndexingResult.PackageAlreadyExists; } diff --git a/src/BaGetter.Core/Metadata/BaGetterRegistrationPageResponse.cs b/src/BaGetter.Core/Metadata/BaGetterRegistrationPageResponse.cs new file mode 100644 index 000000000..aa8ecf0de --- /dev/null +++ b/src/BaGetter.Core/Metadata/BaGetterRegistrationPageResponse.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; +using BaGetter.Protocol.Models; + +namespace BaGetter.Core; + +/// +/// BaGetter's extensions to a registration page response. +/// +/// Extends . +public class BaGetterRegistrationPageResponse : RegistrationPageResponse +{ + /// + /// The registration leafs in this page. + /// + /// This was modified to use BaGetter's extended registration index page item model. + [JsonPropertyName("items")] + [JsonPropertyOrder(int.MaxValue)] + public new IReadOnlyList ItemsOrNull { get; set; } +} diff --git a/src/BaGetter.Core/Metadata/DefaultPackageMetadataService.cs b/src/BaGetter.Core/Metadata/DefaultPackageMetadataService.cs index 32ea1be70..3cb5b8074 100644 --- a/src/BaGetter.Core/Metadata/DefaultPackageMetadataService.cs +++ b/src/BaGetter.Core/Metadata/DefaultPackageMetadataService.cs @@ -37,6 +37,27 @@ public async Task GetRegistrationIndexOrNullA packages)); } + /// + public async Task GetRegistrationPageOrNullAsync( + string packageId, + NuGetVersion lower, + NuGetVersion upper, + CancellationToken cancellationToken = default) + { + var packages = await _packages.FindPackagesAsync(packageId, cancellationToken); + if (!packages.Any()) + { + return null; + } + + return _builder.BuildPage( + new PackageRegistration( + packageId, + packages), + lower, + upper); + } + /// public async Task GetRegistrationLeafOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken = default) { diff --git a/src/BaGetter.Core/Metadata/IPackageMetadataService.cs b/src/BaGetter.Core/Metadata/IPackageMetadataService.cs index 433c9f40d..d4514a9d7 100644 --- a/src/BaGetter.Core/Metadata/IPackageMetadataService.cs +++ b/src/BaGetter.Core/Metadata/IPackageMetadataService.cs @@ -20,6 +20,16 @@ public interface IPackageMetadataService /// The package's registration index, or if the package does not exist. Task GetRegistrationIndexOrNullAsync(string packageId, CancellationToken cancellationToken = default); + /// + /// Get the metadata for several versions of a package, if the package exists. + /// + /// The package's id. + /// The inclusive lower version in the page. + /// The inclusive upper version in the page. + /// A token to cancel the task. + /// The registration page, or if the package does not exist or no packages exist in the range. + Task GetRegistrationPageOrNullAsync(string packageId, NuGetVersion lower, NuGetVersion upper, CancellationToken cancellationToken = default); + /// /// Get the metadata for a single package version, if the package exists. /// diff --git a/src/BaGetter.Core/Metadata/RegistrationBuilder.cs b/src/BaGetter.Core/Metadata/RegistrationBuilder.cs index db0388525..b5fb783cb 100644 --- a/src/BaGetter.Core/Metadata/RegistrationBuilder.cs +++ b/src/BaGetter.Core/Metadata/RegistrationBuilder.cs @@ -2,18 +2,27 @@ using System.Collections.Generic; using System.Linq; using BaGetter.Protocol.Models; +using Microsoft.Extensions.Options; +using NuGet.Versioning; namespace BaGetter.Core; public class RegistrationBuilder { private readonly IUrlGenerator _url; + private readonly int _registrationPageSize; - public RegistrationBuilder(IUrlGenerator url) + public RegistrationBuilder(IUrlGenerator url, IOptions options) { ArgumentNullException.ThrowIfNull(url); + ArgumentNullException.ThrowIfNull(options); _url = url; + _registrationPageSize = options.Value.RegistrationPageSize; + if (_registrationPageSize < 1) + { + throw new ArgumentOutOfRangeException(nameof(options), "RegistrationPageSize must be greater than 0."); + } } public virtual BaGetterRegistrationIndexResponse BuildIndex(PackageRegistration registration) @@ -21,27 +30,65 @@ public virtual BaGetterRegistrationIndexResponse BuildIndex(PackageRegistration ArgumentNullException.ThrowIfNull(registration); var sortedPackages = registration.Packages.OrderBy(p => p.Version).ToList(); + var pagedPackages = sortedPackages.Chunk(_registrationPageSize).ToList(); + var isPaged = pagedPackages.Count > 1; - // TODO: Paging of registration items. - // "Un-paged" example: https://api.nuget.org/v3/registration3/newtonsoft.json/index.json - // Paged example: https://api.nuget.org/v3/registration3/fake/index.json return new BaGetterRegistrationIndexResponse { RegistrationIndexUrl = _url.GetRegistrationIndexUrl(registration.PackageId), Type = RegistrationIndexResponse.DefaultType, - Count = 1, + Count = pagedPackages.Count, TotalDownloads = registration.Packages.Sum(p => p.Downloads), - Pages = new[] - { - new BaGetterRegistrationIndexPage + Pages = pagedPackages + .Select(page => { - RegistrationPageUrl = _url.GetRegistrationIndexUrl(registration.PackageId), - Count = registration.Packages.Count, - Lower = sortedPackages.First().Version.ToNormalizedString().ToLowerInvariant(), - Upper = sortedPackages.Last().Version.ToNormalizedString().ToLowerInvariant(), - ItemsOrNull = sortedPackages.Select(ToRegistrationIndexPageItem).ToList(), - } - } + var lower = page.First().Version; + var upper = page.Last().Version; + + return new BaGetterRegistrationIndexPage + { + RegistrationPageUrl = isPaged + ? _url.GetRegistrationPageUrl(registration.PackageId, lower, upper) + : _url.GetRegistrationIndexUrl(registration.PackageId), + Count = page.Length, + Lower = lower.ToNormalizedString().ToLowerInvariant(), + Upper = upper.ToNormalizedString().ToLowerInvariant(), + ItemsOrNull = isPaged ? null : page.Select(ToRegistrationIndexPageItem).ToList(), + }; + }) + .ToList() + }; + } + + public virtual BaGetterRegistrationPageResponse BuildPage(PackageRegistration registration, NuGetVersion lower, NuGetVersion upper) + { + ArgumentNullException.ThrowIfNull(registration); + ArgumentNullException.ThrowIfNull(lower); + ArgumentNullException.ThrowIfNull(upper); + + if (VersionComparer.VersionRelease.Compare(lower, upper) > 0) + { + return null; + } + + var packages = registration.Packages + .Where(p => VersionComparer.VersionRelease.Compare(p.Version, lower) >= 0 + && VersionComparer.VersionRelease.Compare(p.Version, upper) <= 0) + .OrderBy(p => p.Version, VersionComparer.VersionRelease) + .ToList(); + + if (!packages.Any()) + { + return null; + } + + return new BaGetterRegistrationPageResponse + { + RegistrationPageUrl = _url.GetRegistrationPageUrl(registration.PackageId, lower, upper), + Count = packages.Count, + Lower = lower.ToNormalizedString().ToLowerInvariant(), + Upper = upper.ToNormalizedString().ToLowerInvariant(), + ItemsOrNull = packages.Select(ToRegistrationIndexPageItem).ToList(), }; } diff --git a/src/BaGetter.Core/PackageDatabase.cs b/src/BaGetter.Core/PackageDatabase.cs index f860fdba6..46da67cfd 100644 --- a/src/BaGetter.Core/PackageDatabase.cs +++ b/src/BaGetter.Core/PackageDatabase.cs @@ -67,6 +67,51 @@ public async Task> FindAsync(string id, bool includeUnlis return (await query.AsSingleQuery().ToListAsync(cancellationToken)).AsReadOnly(); } + public async Task> SearchAsync( + string query, + bool? listed, + int skip, + int take, + CancellationToken cancellationToken) + { + var packages = _context.Packages.AsNoTracking(); + + if (!string.IsNullOrWhiteSpace(query)) + { + packages = packages.Where(p => p.Id.Contains(query)); + } + + if (listed.HasValue) + { + packages = packages.Where(p => p.Listed == listed.Value); + } + + return (await packages + .OrderBy(p => p.Id) + .ThenByDescending(p => p.Published) + .Skip(skip) + .Take(take) + .ToListAsync(cancellationToken)) + .AsReadOnly(); + } + + public async Task CountAsync(string query, bool? listed, CancellationToken cancellationToken) + { + var packages = _context.Packages.AsNoTracking(); + + if (!string.IsNullOrWhiteSpace(query)) + { + packages = packages.Where(p => p.Id.Contains(query)); + } + + if (listed.HasValue) + { + packages = packages.Where(p => p.Listed == listed.Value); + } + + return await packages.CountAsync(cancellationToken); + } + public Task FindOrNullAsync( string id, NuGetVersion version, diff --git a/src/BaGetter.Core/Properties/AssemblyInfo.cs b/src/BaGetter.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..2050cd767 --- /dev/null +++ b/src/BaGetter.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("BaGetter.Core.Tests")] diff --git a/src/BaGetter.Core/Search/DatabaseSearchService.cs b/src/BaGetter.Core/Search/DatabaseSearchService.cs index d07e5ce8f..2752c4c0b 100644 --- a/src/BaGetter.Core/Search/DatabaseSearchService.cs +++ b/src/BaGetter.Core/Search/DatabaseSearchService.cs @@ -10,6 +10,9 @@ namespace BaGetter.Core; public class DatabaseSearchService : ISearchService { + private const string TagQueryPrefix = "tag:"; + private const string AuthorQueryPrefix = "author:"; + private readonly IContext _context; private readonly IFrameworkCompatibilityService _frameworks; private readonly ISearchResponseBuilder _searchBuilder; @@ -28,9 +31,10 @@ public DatabaseSearchService(IContext context, IFrameworkCompatibilityService fr public async Task SearchAsync(SearchRequest request, CancellationToken cancellationToken) { var frameworks = GetCompatibleFrameworksOrNull(request.Framework); + var (textQuery, tags, authors) = ParseSearchQuery(request.Query); IQueryable search = _context.Packages; - search = ApplySearchQuery(search, request.Query); + search = ApplySearchTextQuery(search, textQuery); search = ApplySearchFilters( search, request.IncludePrerelease, @@ -38,30 +42,32 @@ public async Task SearchAsync(SearchRequest request, Cancellatio request.PackageType, frameworks); - var packageIds = search - .Select(p => p.Id) - .Distinct() - .OrderBy(id => id) - .Skip(request.Skip) - .Take(request.Take); - - // This query MUST fetch all versions for each package that matches the search, - // otherwise the results for a package's latest version may be incorrect. - // If possible, we'll find all these packages in a single query by matching - // the package IDs in a subquery. Otherwise, run two queries: - // 1. Find the package IDs that match the search - // 2. Find all package versions for these package IDs - if (_context.SupportsLimitInSubqueries) + List packageIdResults; + if (tags.Count > 0 || authors.Count > 0) { - search = _context.Packages.Where(p => packageIds.Contains(p.Id)); + packageIdResults = await GetFilteredPackageIdsForSearchAsync( + search, + tags, + authors, + request.Skip, + request.Take, + cancellationToken); } else { - var packageIdResults = await packageIds.ToListAsync(cancellationToken); - - search = _context.Packages.Where(p => packageIdResults.Contains(p.Id)); + packageIdResults = await search + .Select(p => p.Id) + .Distinct() + .OrderBy(id => id) + .Skip(request.Skip) + .Take(request.Take) + .ToListAsync(cancellationToken); } + // This query MUST fetch all versions for each package that matches the search, + // otherwise the results for a package's latest version may be incorrect. + search = _context.Packages.Where(p => packageIdResults.Contains(p.Id)); + search = ApplySearchFilters( search, request.IncludePrerelease, @@ -80,9 +86,10 @@ public async Task SearchAsync(SearchRequest request, Cancellatio public async Task AutocompleteAsync(AutocompleteRequest request, CancellationToken cancellationToken) { + var (textQuery, tags, authors) = ParseSearchQuery(request.Query); IQueryable search = _context.Packages; - search = ApplySearchQuery(search, request.Query); + search = ApplySearchTextQuery(search, textQuery); search = ApplySearchFilters( search, request.IncludePrerelease, @@ -90,13 +97,27 @@ public async Task AutocompleteAsync(AutocompleteRequest re request.PackageType, frameworks: null); - var packageIds = await search - .OrderByDescending(p => p.Downloads) - .Select(p => p.Id) - .Distinct() - .Skip(request.Skip) - .Take(request.Take) - .ToListAsync(cancellationToken); + List packageIds; + if (tags.Count > 0 || authors.Count > 0) + { + packageIds = await GetFilteredPackageIdsForAutocompleteAsync( + search, + tags, + authors, + request.Skip, + request.Take, + cancellationToken); + } + else + { + packageIds = await search + .OrderByDescending(p => p.Downloads) + .Select(p => p.Id) + .Distinct() + .Skip(request.Skip) + .Take(request.Take) + .ToListAsync(cancellationToken); + } return _searchBuilder.BuildAutocomplete(packageIds); } @@ -143,16 +164,282 @@ public async Task FindDependentsAsync(string packageId, Canc } [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1862:Use the 'StringComparison' method overloads to perform case-insensitive string comparisons", Justification = "Not for EF queries")] - private static IQueryable ApplySearchQuery(IQueryable query, string search) + private static IQueryable ApplySearchTextQuery(IQueryable query, string textQuery) { - if (string.IsNullOrEmpty(search)) + if (string.IsNullOrEmpty(textQuery)) { return query; } - search = search.ToLowerInvariant(); + var normalizedTextQuery = textQuery.ToLowerInvariant(); + + return query.Where(p => p.Id.ToLower().Contains(normalizedTextQuery)); + } + + private static (string TextQuery, IReadOnlyList Tags, IReadOnlyList Authors) ParseSearchQuery(string search) + { + if (string.IsNullOrWhiteSpace(search)) + { + return (null, Array.Empty(), Array.Empty()); + } + + var textTerms = new List(); + var tags = new HashSet(StringComparer.OrdinalIgnoreCase); + var authors = new HashSet(StringComparer.OrdinalIgnoreCase); + var tokens = TokenizeQuery(search); + foreach (var token in tokens) + { + if (token.StartsWith(TagQueryPrefix, StringComparison.OrdinalIgnoreCase)) + { + var tag = token[TagQueryPrefix.Length..].Trim(); + if (!string.IsNullOrEmpty(tag)) + { + tags.Add(tag); + } + continue; + } + + if (token.StartsWith(AuthorQueryPrefix, StringComparison.OrdinalIgnoreCase)) + { + var author = token[AuthorQueryPrefix.Length..].Trim(); + if (!string.IsNullOrEmpty(author)) + { + authors.Add(author); + } + continue; + } + + textTerms.Add(token); + } + + var textQuery = textTerms.Count > 0 ? string.Join(' ', textTerms) : null; + return (textQuery, tags.ToList(), authors.ToList()); + } + + private static List TokenizeQuery(string input) + { + var tokens = new List(); + var i = 0; + var length = input.Length; + + while (i < length) + { + while (i < length && char.IsWhiteSpace(input[i])) + { + i++; + } + + if (i >= length) + { + break; + } + + var isTag = StartsWithAt(input, i, TagQueryPrefix); + var isAuthor = !isTag && StartsWithAt(input, i, AuthorQueryPrefix); + var prefix = isTag ? TagQueryPrefix : (isAuthor ? AuthorQueryPrefix : null); + + if (prefix != null) + { + i += prefix.Length; + + while (i < length && char.IsWhiteSpace(input[i])) + { + i++; + } + + var value = ReadTokenValue(input, ref i); + tokens.Add(prefix + value); + continue; + } + + tokens.Add(ReadTokenValue(input, ref i)); + } + + return tokens; + } - return query.Where(p => p.Id.ToLower().Contains(search)); + private static bool StartsWithAt(string input, int index, string value) + { + return input.AsSpan(index).StartsWith(value, StringComparison.OrdinalIgnoreCase); + } + + private static string ReadTokenValue(string input, ref int index) + { + if (index >= input.Length) + { + return string.Empty; + } + + if (input[index] == '"') + { + index++; + var start = index; + + while (index < input.Length && input[index] != '"') + { + index++; + } + + var value = input.Substring(start, index - start); + + if (index < input.Length && input[index] == '"') + { + index++; + } + + return value; + } + + var unquotedStart = index; + while (index < input.Length && !char.IsWhiteSpace(input[index])) + { + index++; + } + + return input.Substring(unquotedStart, index - unquotedStart); + } + + private static bool HasAllTags(string[] packageTags, IReadOnlyList requiredTags) + { + if (requiredTags == null || requiredTags.Count == 0) + { + return true; + } + + if (packageTags == null || packageTags.Length == 0) + { + return false; + } + + var packageTagSet = new HashSet(packageTags, StringComparer.OrdinalIgnoreCase); + return requiredTags.All(packageTagSet.Contains); + } + + private static bool HasAllAuthors(string[] packageAuthors, IReadOnlyList requiredAuthors) + { + if (requiredAuthors == null || requiredAuthors.Count == 0) + { + return true; + } + + if (packageAuthors == null || packageAuthors.Length == 0) + { + return false; + } + + return requiredAuthors.All(required => + packageAuthors.Any(author => + !string.IsNullOrEmpty(author) && + author.Contains(required, StringComparison.OrdinalIgnoreCase))); + } + + private static async Task> GetFilteredPackageIdsForSearchAsync( + IQueryable query, + IReadOnlyList requiredTags, + IReadOnlyList requiredAuthors, + int skip, + int take, + CancellationToken cancellationToken) + { + if (take <= 0) + { + return new List(); + } + + var results = new List(capacity: take); + var seenPackageIds = new HashSet(StringComparer.OrdinalIgnoreCase); + var matchedCount = 0; + + await foreach (var candidate in query + .AsNoTracking() + .OrderBy(p => p.Id) + .Select(p => new PackageFilterCandidate + { + Id = p.Id, + Tags = p.Tags, + Authors = p.Authors, + }) + .AsAsyncEnumerable() + .WithCancellation(cancellationToken)) + { + if (!seenPackageIds.Add(candidate.Id)) + { + continue; + } + + if (!HasAllTags(candidate.Tags, requiredTags) || !HasAllAuthors(candidate.Authors, requiredAuthors)) + { + continue; + } + + if (matchedCount++ < skip) + { + continue; + } + + results.Add(candidate.Id); + if (results.Count >= take) + { + break; + } + } + + return results; + } + + private static async Task> GetFilteredPackageIdsForAutocompleteAsync( + IQueryable query, + IReadOnlyList requiredTags, + IReadOnlyList requiredAuthors, + int skip, + int take, + CancellationToken cancellationToken) + { + if (take <= 0) + { + return new List(); + } + + var results = new List(capacity: take); + var seenPackageIds = new HashSet(StringComparer.OrdinalIgnoreCase); + var matchedCount = 0; + + await foreach (var candidate in query + .AsNoTracking() + .OrderByDescending(p => p.Downloads) + .ThenBy(p => p.Id) + .Select(p => new PackageFilterCandidate + { + Id = p.Id, + Tags = p.Tags, + Authors = p.Authors, + }) + .AsAsyncEnumerable() + .WithCancellation(cancellationToken)) + { + if (!seenPackageIds.Add(candidate.Id)) + { + continue; + } + + if (!HasAllTags(candidate.Tags, requiredTags) || !HasAllAuthors(candidate.Authors, requiredAuthors)) + { + continue; + } + + if (matchedCount++ < skip) + { + continue; + } + + results.Add(candidate.Id); + if (results.Count >= take) + { + break; + } + } + + return results; } private static IQueryable ApplySearchFilters( @@ -191,4 +478,11 @@ private IReadOnlyList GetCompatibleFrameworksOrNull(string framework) return _frameworks.FindAllCompatibleFrameworks(framework); } + + private sealed class PackageFilterCandidate + { + public string Id { get; init; } + public string[] Tags { get; init; } + public string[] Authors { get; init; } + } } diff --git a/src/BaGetter.Core/Search/ISearchReindexService.cs b/src/BaGetter.Core/Search/ISearchReindexService.cs new file mode 100644 index 000000000..b02830751 --- /dev/null +++ b/src/BaGetter.Core/Search/ISearchReindexService.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace BaGetter.Core; + +public interface ISearchReindexService +{ + Task ReindexAsync(CancellationToken cancellationToken); +} diff --git a/src/BaGetter.Core/Search/SearchReindexService.cs b/src/BaGetter.Core/Search/SearchReindexService.cs new file mode 100644 index 000000000..00bd624f7 --- /dev/null +++ b/src/BaGetter.Core/Search/SearchReindexService.cs @@ -0,0 +1,67 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace BaGetter.Core; + +public class SearchReindexService : ISearchReindexService +{ + private readonly IContext _context; + private readonly ISearchIndexer _indexer; + private readonly IOptions _options; + private readonly ILogger _logger; + + public SearchReindexService( + IContext context, + ISearchIndexer indexer, + IOptions options, + ILogger logger) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + _indexer = indexer ?? throw new ArgumentNullException(nameof(indexer)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ReindexAsync(CancellationToken cancellationToken) + { + var batchSize = _options.Value.Reindex?.BatchSize ?? 100; + var indexedCount = 0; + var lastSeenKey = 0; + + _logger.LogInformation("Starting search reindex (batch size: {BatchSize})", batchSize); + + while (true) + { + var batch = await _context.Packages + .AsNoTracking() + .Where(p => p.Key > lastSeenKey) + .OrderBy(p => p.Key) + .Include(p => p.Dependencies) + .Include(p => p.PackageTypes) + .Include(p => p.TargetFrameworks) + .Take(batchSize) + .ToListAsync(cancellationToken); + + if (batch.Count == 0) + { + break; + } + + foreach (var package in batch) + { + await _indexer.IndexAsync(package, cancellationToken); + indexedCount++; + } + + lastSeenKey = batch[^1].Key; + } + + _logger.LogInformation("Finished search reindex. Indexed {PackageCount} packages.", indexedCount); + return indexedCount; + } +} diff --git a/src/BaGetter.Core/ServiceIndex/BaGetterServiceIndex.cs b/src/BaGetter.Core/ServiceIndex/BaGetterServiceIndex.cs index 4a57f08c8..44685d0cb 100644 --- a/src/BaGetter.Core/ServiceIndex/BaGetterServiceIndex.cs +++ b/src/BaGetter.Core/ServiceIndex/BaGetterServiceIndex.cs @@ -35,10 +35,10 @@ public Task GetAsync(CancellationToken cancellationToken = resources.AddRange(BuildResource("PackagePublish", _url.GetPackagePublishResourceUrl(), "2.0.0")); resources.AddRange(BuildResource("SymbolPackagePublish", _url.GetSymbolPublishResourceUrl(), "4.9.0")); - resources.AddRange(BuildResource("SearchQueryService", _url.GetSearchResourceUrl(), "", "3.0.0-beta", "3.0.0-rc")); - resources.AddRange(BuildResource("RegistrationsBaseUrl", _url.GetPackageMetadataResourceUrl(), "", "3.0.0-rc", "3.0.0-beta")); + resources.AddRange(BuildResource("SearchQueryService", _url.GetSearchResourceUrl(), "", "3.0.0-beta", "3.0.0-rc", "3.4.0", "3.5.0")); + resources.AddRange(BuildResource("RegistrationsBaseUrl", _url.GetPackageMetadataResourceUrl(), "", "3.0.0-rc", "3.0.0-beta", "3.4.0", "3.6.0")); resources.AddRange(BuildResource("PackageBaseAddress", _url.GetPackageContentResourceUrl(), "3.0.0")); - resources.AddRange(BuildResource("SearchAutocompleteService", _url.GetAutocompleteResourceUrl(), "", "3.0.0-rc", "3.0.0-beta")); + resources.AddRange(BuildResource("SearchAutocompleteService", _url.GetAutocompleteResourceUrl(), "", "3.0.0-rc", "3.0.0-beta", "3.5.0")); var result = new ServiceIndexResponse { diff --git a/src/BaGetter.Core/Upstream/Clients/FallbackUpstreamClient.cs b/src/BaGetter.Core/Upstream/Clients/FallbackUpstreamClient.cs new file mode 100644 index 000000000..0702779d3 --- /dev/null +++ b/src/BaGetter.Core/Upstream/Clients/FallbackUpstreamClient.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Versioning; + +namespace BaGetter.Core; + +/// +/// Aggregates multiple upstream clients. Version/metadata listings are UNIONED across +/// every upstream (so a package present on several feeds contributes all its versions +/// and no single feed can shadow another — e.g. nuget.org's stale Hangfire.Throttling +/// 1.0.0-beta1 must not hide the licensed feed's 1.4.3). Downloads use the first +/// upstream that has the requested package. +/// +public sealed class FallbackUpstreamClient : IUpstreamClient, IDisposable +{ + private readonly IReadOnlyList _clients; + private readonly ILogger _logger; + + public FallbackUpstreamClient( + IReadOnlyList clients, + ILogger logger) + { + if (clients == null || clients.Count == 0) + { + throw new ArgumentException("At least one upstream client is required.", nameof(clients)); + } + + _clients = clients; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> ListPackageVersionsAsync(string id, CancellationToken cancellationToken) + { + var versions = new HashSet(); + foreach (var client in _clients) + { + try + { + foreach (var version in await client.ListPackageVersionsAsync(id, cancellationToken)) + { + versions.Add(version); + } + } + catch (Exception e) + { + _logger.LogWarning(e, "Mirror failed while listing versions for {PackageId}", id); + } + } + + return versions.ToList(); + } + + public async Task> ListPackagesAsync(string id, CancellationToken cancellationToken) + { + // Union metadata across all upstreams, deduped by version. Earlier (higher + // priority) upstreams win for a version present on more than one feed. + var byVersion = new Dictionary(); + foreach (var client in _clients) + { + try + { + foreach (var package in await client.ListPackagesAsync(id, cancellationToken)) + { + if (!byVersion.ContainsKey(package.Version)) + { + byVersion[package.Version] = package; + } + } + } + catch (Exception e) + { + _logger.LogWarning(e, "Mirror failed while listing metadata for {PackageId}", id); + } + } + + return byVersion.Values.ToList(); + } + + public async Task DownloadPackageOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken) + { + foreach (var client in _clients) + { + try + { + var stream = await client.DownloadPackageOrNullAsync(id, version, cancellationToken); + if (stream != null) + { + return stream; + } + } + catch (Exception e) + { + _logger.LogWarning(e, "Mirror failed while downloading {PackageId} {PackageVersion}", id, version); + } + } + + return null; + } + + public string GetServiceIndexUrl() + { + return _clients[0].GetServiceIndexUrl(); + } + + public void Dispose() + { + foreach (var disposable in _clients.OfType()) + { + disposable.Dispose(); + } + + GC.SuppressFinalize(this); + } +} diff --git a/src/BaGetter.Core/Upstream/Clients/V2UpstreamClient.cs b/src/BaGetter.Core/Upstream/Clients/V2UpstreamClient.cs index 8cf12fe39..320d226cf 100644 --- a/src/BaGetter.Core/Upstream/Clients/V2UpstreamClient.cs +++ b/src/BaGetter.Core/Upstream/Clients/V2UpstreamClient.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using NuGet.Common; using NuGet.Configuration; using NuGet.Packaging; @@ -31,12 +30,12 @@ public class V2UpstreamClient : IUpstreamClient, IDisposable private static readonly char[] AuthorsSeparators = new[] { ',', ';', '\t', '\n', '\r' }; public V2UpstreamClient( - IOptionsSnapshot options, + MirrorOptions options, ILogger logger) { ArgumentNullException.ThrowIfNull(options); - if (options.Value?.PackageSource?.AbsolutePath == null) + if (options.PackageSource?.AbsolutePath == null) { throw new ArgumentException("No mirror package source has been set."); } @@ -45,8 +44,8 @@ public V2UpstreamClient( _ngLogger = NullLogger.Instance; _cache = new SourceCacheContext(); - var packageSource = new PackageSource(options.Value.PackageSource.AbsoluteUri); - if (options.Value.Authentication is { } auth) + var packageSource = new PackageSource(options.PackageSource.AbsoluteUri); + if (options.Authentication is { } auth) { switch (auth.Type) { diff --git a/src/BaGetter.Core/Validation/ValidateStartupOptions.cs b/src/BaGetter.Core/Validation/ValidateStartupOptions.cs index 80a1421ba..d3e084d30 100644 --- a/src/BaGetter.Core/Validation/ValidateStartupOptions.cs +++ b/src/BaGetter.Core/Validation/ValidateStartupOptions.cs @@ -48,6 +48,13 @@ public bool Validate() _ = _healthCheck.Value; _ = _statistics.Value; + var ldap = _root.Value.Authentication?.Ldap; + if (ldap?.Enabled == true && !OperatingSystem.IsWindows()) + { + _logger.LogWarning( + "LDAP authentication is enabled on a non-Windows host. Ensure LDAPS certificate trust and platform LDAP/TLS configuration are set up correctly for this environment."); + } + return true; } catch (OptionsValidationException e) diff --git a/src/BaGetter.Database.MySql/BaGetter.Database.MySql.csproj b/src/BaGetter.Database.MySql/BaGetter.Database.MySql.csproj index a340335f7..3c5da9c91 100644 --- a/src/BaGetter.Database.MySql/BaGetter.Database.MySql.csproj +++ b/src/BaGetter.Database.MySql/BaGetter.Database.MySql.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 latest NuGet The libraries to host BaGetter on MySQL. @@ -16,3 +16,4 @@ + diff --git a/src/BaGetter.Database.PostgreSql/BaGetter.Database.PostgreSql.csproj b/src/BaGetter.Database.PostgreSql/BaGetter.Database.PostgreSql.csproj index b0a859043..094794d67 100644 --- a/src/BaGetter.Database.PostgreSql/BaGetter.Database.PostgreSql.csproj +++ b/src/BaGetter.Database.PostgreSql/BaGetter.Database.PostgreSql.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 NuGet The libraries to host BaGetter on PostgreSQL. @@ -16,3 +16,4 @@ + diff --git a/src/BaGetter.Database.SqlServer/BaGetter.Database.SqlServer.csproj b/src/BaGetter.Database.SqlServer/BaGetter.Database.SqlServer.csproj index a4314e332..e903d0722 100644 --- a/src/BaGetter.Database.SqlServer/BaGetter.Database.SqlServer.csproj +++ b/src/BaGetter.Database.SqlServer/BaGetter.Database.SqlServer.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 NuGet The libraries to host BaGetter on SQL Server. @@ -18,3 +18,4 @@ + diff --git a/src/BaGetter.Database.Sqlite/BaGetter.Database.Sqlite.csproj b/src/BaGetter.Database.Sqlite/BaGetter.Database.Sqlite.csproj index 87fb4d009..ccf3c273d 100644 --- a/src/BaGetter.Database.Sqlite/BaGetter.Database.Sqlite.csproj +++ b/src/BaGetter.Database.Sqlite/BaGetter.Database.Sqlite.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 enable NuGet The libraries to host BaGetter on SQLite. @@ -16,3 +16,4 @@ + diff --git a/src/BaGetter.Gcp/BaGetter.Gcp.csproj b/src/BaGetter.Gcp/BaGetter.Gcp.csproj index 05c2945a1..fd16596be 100644 --- a/src/BaGetter.Gcp/BaGetter.Gcp.csproj +++ b/src/BaGetter.Gcp/BaGetter.Gcp.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 NuGet;Google;Cloud The libraries to host BaGetter on the Google Cloud Platform. @@ -16,3 +16,4 @@ + diff --git a/src/BaGetter.Protocol/BaGetter.Protocol.csproj b/src/BaGetter.Protocol/BaGetter.Protocol.csproj index c1a4d64da..4d34c96be 100644 --- a/src/BaGetter.Protocol/BaGetter.Protocol.csproj +++ b/src/BaGetter.Protocol/BaGetter.Protocol.csproj @@ -1,10 +1,10 @@ - netstandard2.0 NuGet;Protocol Libraries to interact with NuGet server APIs. + netstandard2.1;net10.0 diff --git a/src/BaGetter.Protocol/Extensions/ServiceIndexModelExtensions.cs b/src/BaGetter.Protocol/Extensions/ServiceIndexModelExtensions.cs index b6f0bed0c..38c8f9b30 100644 --- a/src/BaGetter.Protocol/Extensions/ServiceIndexModelExtensions.cs +++ b/src/BaGetter.Protocol/Extensions/ServiceIndexModelExtensions.cs @@ -13,6 +13,7 @@ public static class ServiceIndexModelExtensions private static readonly string Version200 = "/2.0.0"; private static readonly string Version300beta = "/3.0.0-beta"; private static readonly string Version300 = "/3.0.0"; + private static readonly string Version350 = "/3.5.0"; private static readonly string Version340 = "/3.4.0"; private static readonly string Version360 = "/3.6.0"; private static readonly string Version470 = "/4.7.0"; @@ -21,9 +22,9 @@ public static class ServiceIndexModelExtensions private static readonly string Version510 = "/5.1.0"; private static readonly string[] Catalog = { "Catalog" + Version300 }; - private static readonly string[] SearchQueryService = { "SearchQueryService" + Version340, "SearchQueryService" + Version300beta, "SearchQueryService" }; + private static readonly string[] SearchQueryService = { "SearchQueryService" + Version350, "SearchQueryService" + Version340, "SearchQueryService" + Version300beta, "SearchQueryService" }; private static readonly string[] RegistrationsBaseUrl = { "RegistrationsBaseUrl" + Version360, "RegistrationsBaseUrl" + Version340, "RegistrationsBaseUrl" + Version300beta, "RegistrationsBaseUrl" }; - private static readonly string[] SearchAutocompleteService = { "SearchAutocompleteService", "SearchAutocompleteService" + Version300beta }; + private static readonly string[] SearchAutocompleteService = { "SearchAutocompleteService" + Version350, "SearchAutocompleteService", "SearchAutocompleteService" + Version300beta }; private static readonly string[] ReportAbuse = { "ReportAbuseUriTemplate", "ReportAbuseUriTemplate" + Version300 }; private static readonly string[] PackageDetailsUriTemplate = { "PackageDetailsUriTemplate" + Version510 }; private static readonly string[] LegacyGallery = { "LegacyGallery" + Version200 }; diff --git a/src/BaGetter.Tencent/BaGetter.Tencent.csproj b/src/BaGetter.Tencent/BaGetter.Tencent.csproj index 434765158..a1baf9d8c 100644 --- a/src/BaGetter.Tencent/BaGetter.Tencent.csproj +++ b/src/BaGetter.Tencent/BaGetter.Tencent.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 enable The libraries to host BaGetter on Tencent Cloud. enable @@ -15,3 +15,4 @@ + diff --git a/src/BaGetter.Web/Admin/AdminAuditEntry.cs b/src/BaGetter.Web/Admin/AdminAuditEntry.cs new file mode 100644 index 000000000..be1eb3b6e --- /dev/null +++ b/src/BaGetter.Web/Admin/AdminAuditEntry.cs @@ -0,0 +1,15 @@ +using System; + +namespace BaGetter.Web.Admin; + +public sealed class AdminAuditEntry +{ + public DateTimeOffset TimestampUtc { get; init; } + public string Actor { get; init; } + public string Action { get; init; } + public string PackageId { get; init; } + public string PackageVersion { get; init; } + public string Details { get; init; } + public string Result { get; init; } + public string IpAddress { get; init; } +} diff --git a/src/BaGetter.Web/Admin/AdminPackageOperationResult.cs b/src/BaGetter.Web/Admin/AdminPackageOperationResult.cs new file mode 100644 index 000000000..b6a6204c4 --- /dev/null +++ b/src/BaGetter.Web/Admin/AdminPackageOperationResult.cs @@ -0,0 +1,7 @@ +namespace BaGetter.Web.Admin; + +public sealed record AdminPackageOperationResult(bool Succeeded, string Message) +{ + public static AdminPackageOperationResult Success(string message) => new(true, message); + public static AdminPackageOperationResult Failure(string message) => new(false, message); +} diff --git a/src/BaGetter.Web/Admin/AdminPackageService.cs b/src/BaGetter.Web/Admin/AdminPackageService.cs new file mode 100644 index 000000000..1c64d0b62 --- /dev/null +++ b/src/BaGetter.Web/Admin/AdminPackageService.cs @@ -0,0 +1,204 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using BaGetter.Core; +using NuGet.Packaging; +using NuGet.Versioning; + +namespace BaGetter.Web.Admin; + +public sealed class AdminPackageService : IAdminPackageService +{ + private readonly IPackageDatabase _packages; + private readonly IPackageStorageService _storage; + private readonly IPackageIndexingService _indexer; + private readonly ISearchIndexer _searchIndexer; + + public AdminPackageService( + IPackageDatabase packages, + IPackageStorageService storage, + IPackageIndexingService indexer, + ISearchIndexer searchIndexer) + { + _packages = packages ?? throw new ArgumentNullException(nameof(packages)); + _storage = storage ?? throw new ArgumentNullException(nameof(storage)); + _indexer = indexer ?? throw new ArgumentNullException(nameof(indexer)); + _searchIndexer = searchIndexer ?? throw new ArgumentNullException(nameof(searchIndexer)); + } + + public async Task SetListedAsync( + string id, + NuGetVersion version, + bool listed, + CancellationToken cancellationToken) + { + var package = await _packages.FindOrNullAsync(id, version, includeUnlisted: true, cancellationToken); + if (package == null) + { + return AdminPackageOperationResult.Failure("Package version was not found."); + } + + var changed = listed + ? await _packages.RelistPackageAsync(id, version, cancellationToken) + : await _packages.UnlistPackageAsync(id, version, cancellationToken); + + if (!changed) + { + return AdminPackageOperationResult.Failure("Package version was not found."); + } + + await _searchIndexer.IndexAsync(package, cancellationToken); + return AdminPackageOperationResult.Success( + listed ? "Package version was relisted." : "Package version was unlisted."); + } + + public async Task HardDeleteAsync( + string id, + NuGetVersion version, + CancellationToken cancellationToken) + { + var package = await _packages.FindOrNullAsync(id, version, includeUnlisted: true, cancellationToken); + if (package == null) + { + return AdminPackageOperationResult.Failure("Package version was not found."); + } + + if (!await _packages.HardDeletePackageAsync(id, version, cancellationToken)) + { + return AdminPackageOperationResult.Failure("Package version was not found."); + } + + await _storage.DeleteAsync(id, version, cancellationToken); + await _searchIndexer.IndexAsync(package, cancellationToken); + return AdminPackageOperationResult.Success("Package version was permanently deleted."); + } + + public Task CopyAsync( + string id, + NuGetVersion version, + string targetId, + CancellationToken cancellationToken) + { + return CopyCoreAsync(id, version, targetId, cancellationToken); + } + + public async Task RenameAsync( + string id, + NuGetVersion version, + string targetId, + CancellationToken cancellationToken) + { + var copyResult = await CopyCoreAsync(id, version, targetId, cancellationToken); + if (!copyResult.Succeeded) + { + return copyResult; + } + + var deleteResult = await HardDeleteAsync(id, version, cancellationToken); + if (!deleteResult.Succeeded) + { + return AdminPackageOperationResult.Failure( + $"Copied package to '{targetId}', but could not delete the source: {deleteResult.Message}"); + } + + return AdminPackageOperationResult.Success($"Package version was renamed to '{targetId}'."); + } + + private async Task CopyCoreAsync( + string id, + NuGetVersion version, + string targetId, + CancellationToken cancellationToken) + { + targetId = targetId?.Trim(); + if (string.IsNullOrWhiteSpace(targetId) || !PackageIdValidator.IsValidPackageId(targetId)) + { + return AdminPackageOperationResult.Failure("The target package ID is invalid."); + } + + if (string.Equals(id, targetId, StringComparison.OrdinalIgnoreCase)) + { + return AdminPackageOperationResult.Failure("The target package ID must be different."); + } + + if (await _packages.ExistsAsync(targetId, version, cancellationToken)) + { + return AdminPackageOperationResult.Failure("The target package version already exists."); + } + + var sourcePackage = await _packages.FindOrNullAsync(id, version, includeUnlisted: true, cancellationToken); + if (sourcePackage == null) + { + return AdminPackageOperationResult.Failure("Package version was not found."); + } + + await using var source = await _storage.GetPackageStreamAsync(id, version, cancellationToken); + await using var transformed = new MemoryStream(); + await source.CopyToAsync(transformed, cancellationToken); + transformed.Position = 0; + + using (var archive = new ZipArchive(transformed, ZipArchiveMode.Update, leaveOpen: true)) + { + if (archive.Entries.Any(entry => + string.Equals(entry.FullName, ".signature.p7s", StringComparison.OrdinalIgnoreCase))) + { + return AdminPackageOperationResult.Failure( + "Signed packages cannot be copied or renamed because changing the ID would invalidate the signature."); + } + + var nuspecEntry = archive.Entries.SingleOrDefault(entry => + entry.FullName.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)); + if (nuspecEntry == null) + { + return AdminPackageOperationResult.Failure("The package does not contain a nuspec manifest."); + } + + XDocument document; + await using (var nuspecStream = nuspecEntry.Open()) + { + document = await XDocument.LoadAsync(nuspecStream, LoadOptions.PreserveWhitespace, cancellationToken); + } + + var idElement = document.Descendants().FirstOrDefault(element => element.Name.LocalName == "id"); + if (idElement == null) + { + return AdminPackageOperationResult.Failure("The package manifest does not contain an ID."); + } + + idElement.Value = targetId; + nuspecEntry.Delete(); + + var newNuspec = archive.CreateEntry($"{targetId}.nuspec", CompressionLevel.Optimal); + await using var newNuspecStream = newNuspec.Open(); + await document.SaveAsync(newNuspecStream, SaveOptions.DisableFormatting, cancellationToken); + } + + transformed.Position = 0; + var indexingResult = await _indexer.IndexAsync(transformed, cancellationToken); + if (indexingResult == PackageIndexingResult.Success && !sourcePackage.Listed) + { + var copiedPackage = await _packages.FindOrNullAsync( + targetId, version, includeUnlisted: true, cancellationToken); + await _packages.UnlistPackageAsync(targetId, version, cancellationToken); + if (copiedPackage != null) + { + await _searchIndexer.IndexAsync(copiedPackage, cancellationToken); + } + } + + return indexingResult switch + { + PackageIndexingResult.Success => + AdminPackageOperationResult.Success($"Package version was copied to '{targetId}'."), + PackageIndexingResult.PackageAlreadyExists => + AdminPackageOperationResult.Failure("The target package version already exists."), + PackageIndexingResult.InvalidPackage => + AdminPackageOperationResult.Failure("The copied package failed validation."), + _ => throw new InvalidOperationException($"Unknown indexing result '{indexingResult}'."), + }; + } +} diff --git a/src/BaGetter.Web/Admin/FileAdminAuditLog.cs b/src/BaGetter.Web/Admin/FileAdminAuditLog.cs new file mode 100644 index 000000000..65788bfbf --- /dev/null +++ b/src/BaGetter.Web/Admin/FileAdminAuditLog.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using BaGetter.Core; + +namespace BaGetter.Web.Admin; + +public sealed class FileAdminAuditLog : IAdminAuditLog +{ + private static readonly SemaphoreSlim Gate = new(1, 1); + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private readonly string _path; + private readonly ILogger _logger; + + public FileAdminAuditLog( + IWebHostEnvironment environment, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(environment); + ArgumentNullException.ThrowIfNull(options); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var configuredPath = options.Value.Admin?.AuditLogPath; + if (string.IsNullOrWhiteSpace(configuredPath)) + { + throw new InvalidOperationException("Admin.AuditLogPath must be configured."); + } + + _path = Path.IsPathRooted(configuredPath) + ? configuredPath + : Path.Combine(environment.ContentRootPath, configuredPath); + } + + public async Task WriteAsync(AdminAuditEntry entry, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(entry); + + await Gate.WaitAsync(cancellationToken); + try + { + Directory.CreateDirectory(Path.GetDirectoryName(_path)!); + var line = JsonSerializer.Serialize(entry, JsonOptions); + await File.AppendAllTextAsync(_path, line + Environment.NewLine, cancellationToken); + } + finally + { + Gate.Release(); + } + + _logger.LogInformation( + "ADMIN_AUDIT actor={Actor} action={Action} package={PackageId} version={PackageVersion} result={Result} ip={IpAddress}", + entry.Actor, + entry.Action, + entry.PackageId, + entry.PackageVersion, + entry.Result, + entry.IpAddress); + } + + public async Task> ReadLatestAsync(int count, CancellationToken cancellationToken) + { + if (count <= 0 || !File.Exists(_path)) + { + return []; + } + + await Gate.WaitAsync(cancellationToken); + try + { + var latest = new Queue(count); + await using var stream = new FileStream( + _path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite, + bufferSize: 4096, + useAsync: true); + using var reader = new StreamReader(stream); + + while (await reader.ReadLineAsync(cancellationToken) is { } line) + { + var entry = ParseEntry(line); + if (entry == null) + { + continue; + } + + if (latest.Count == count) + { + latest.Dequeue(); + } + + latest.Enqueue(entry); + } + + return latest.Reverse().ToList(); + } + finally + { + Gate.Release(); + } + } + + private AdminAuditEntry ParseEntry(string line) + { + try + { + return JsonSerializer.Deserialize(line, JsonOptions); + } + catch (JsonException exception) + { + _logger.LogWarning(exception, "Ignoring malformed admin audit log entry"); + return null; + } + } +} diff --git a/src/BaGetter.Web/Admin/IAdminAuditLog.cs b/src/BaGetter.Web/Admin/IAdminAuditLog.cs new file mode 100644 index 000000000..576f157aa --- /dev/null +++ b/src/BaGetter.Web/Admin/IAdminAuditLog.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace BaGetter.Web.Admin; + +public interface IAdminAuditLog +{ + Task WriteAsync(AdminAuditEntry entry, CancellationToken cancellationToken); + Task> ReadLatestAsync(int count, CancellationToken cancellationToken); +} diff --git a/src/BaGetter.Web/Admin/IAdminPackageService.cs b/src/BaGetter.Web/Admin/IAdminPackageService.cs new file mode 100644 index 000000000..b89cb5351 --- /dev/null +++ b/src/BaGetter.Web/Admin/IAdminPackageService.cs @@ -0,0 +1,31 @@ +using System.Threading; +using System.Threading.Tasks; +using NuGet.Versioning; + +namespace BaGetter.Web.Admin; + +public interface IAdminPackageService +{ + Task SetListedAsync( + string id, + NuGetVersion version, + bool listed, + CancellationToken cancellationToken); + + Task HardDeleteAsync( + string id, + NuGetVersion version, + CancellationToken cancellationToken); + + Task CopyAsync( + string id, + NuGetVersion version, + string targetId, + CancellationToken cancellationToken); + + Task RenameAsync( + string id, + NuGetVersion version, + string targetId, + CancellationToken cancellationToken); +} diff --git a/src/BaGetter.Web/Authentication/NugetBasicAuthenticationHandler.cs b/src/BaGetter.Web/Authentication/NugetBasicAuthenticationHandler.cs index ef5031e79..4f2189fb2 100644 --- a/src/BaGetter.Web/Authentication/NugetBasicAuthenticationHandler.cs +++ b/src/BaGetter.Web/Authentication/NugetBasicAuthenticationHandler.cs @@ -9,32 +9,37 @@ using System; using BaGetter.Core; using System.Linq; +using System.Collections.Generic; +using System.Threading; namespace BaGetter.Web.Authentication; public class NugetBasicAuthenticationHandler : AuthenticationHandler { private readonly IOptions bagetterOptions; + private readonly IReadOnlyList credentialValidators; public NugetBasicAuthenticationHandler( IOptionsMonitor options, ILoggerFactory logger, UrlEncoder encoder, - IOptions bagetterOptions) + IOptions bagetterOptions, + IEnumerable credentialValidators) : base(options, logger, encoder) { this.bagetterOptions = bagetterOptions; + this.credentialValidators = credentialValidators?.ToArray() ?? []; } - protected override Task HandleAuthenticateAsync() + protected override async Task HandleAuthenticateAsync() { if (IsAnonymousAllowed()) { - return CreateAnonymousAuthenticatonResult(); + return await CreateAnonymousAuthenticatonResult(); } if (!Request.Headers.TryGetValue("Authorization", out var auth)) - return Task.FromResult(AuthenticateResult.NoResult()); + return AuthenticateResult.NoResult(); string username = null; string password = null; @@ -48,13 +53,14 @@ protected override Task HandleAuthenticateAsync() } catch { - return Task.FromResult(AuthenticateResult.Fail("Invalid Authorization Header")); + return AuthenticateResult.Fail("Invalid Authorization Header"); } - if (!ValidateCredentials(username, password)) - return Task.FromResult(AuthenticateResult.Fail("Invalid Username or Password")); + var validationResult = await ValidateCredentialsAsync(username, password, Context.RequestAborted); + if (validationResult == null) + return AuthenticateResult.Fail("Invalid Username or Password"); - return CreateUserAuthenticatonResult(username); + return await CreateUserAuthenticatonResult(validationResult); } protected override async Task HandleChallengeAsync(AuthenticationProperties properties) @@ -74,9 +80,21 @@ private Task CreateAnonymousAuthenticatonResult() return Task.FromResult(AuthenticateResult.Success(ticket)); } - private Task CreateUserAuthenticatonResult(string username) + private Task CreateUserAuthenticatonResult(NugetCredentialValidationResult validationResult) { - Claim[] claims = [new Claim(ClaimTypes.Name, username)]; + var claims = new List + { + new Claim(ClaimTypes.Name, validationResult.Username) + }; + + foreach (var role in validationResult.Roles ?? []) + { + if (!string.IsNullOrWhiteSpace(role)) + { + claims.Add(new Claim(ClaimTypes.Role, role.Trim())); + } + } + var identity = new ClaimsIdentity(claims, Scheme.Name); var principal = new ClaimsPrincipal(identity); @@ -87,14 +105,31 @@ private Task CreateUserAuthenticatonResult(string username) private bool IsAnonymousAllowed() { - return bagetterOptions.Value.Authentication is null || - bagetterOptions.Value.Authentication.Credentials is null || - bagetterOptions.Value.Authentication.Credentials.Length == 0 || - bagetterOptions.Value.Authentication.Credentials.All(a => string.IsNullOrWhiteSpace(a.Username) && string.IsNullOrWhiteSpace(a.Password)); + var auth = bagetterOptions.Value.Authentication; + var hasConfiguredCredentials = auth?.Credentials?.Any(a => + !string.IsNullOrWhiteSpace(a.Username) && + (!string.IsNullOrWhiteSpace(a.Password) || !string.IsNullOrWhiteSpace(a.PasswordHash))) == true; + var hasLdap = auth?.Ldap?.Enabled == true && + !string.IsNullOrWhiteSpace(auth.Ldap.Server) && + !string.IsNullOrWhiteSpace(auth.Ldap.BaseDn); + + return !hasConfiguredCredentials && !hasLdap; } - private bool ValidateCredentials(string username, string password) + private async Task ValidateCredentialsAsync( + string username, + string password, + CancellationToken cancellationToken) { - return bagetterOptions.Value.Authentication.Credentials.Any(a => a.Username.Equals(username, StringComparison.OrdinalIgnoreCase) && a.Password == password); + foreach (var validator in credentialValidators) + { + var result = await validator.ValidateAsync(username, password, cancellationToken); + if (result != null) + { + return result; + } + } + + return null; } } diff --git a/src/BaGetter.Web/BaGetter.Web.csproj b/src/BaGetter.Web/BaGetter.Web.csproj index ff350457c..ccace707e 100644 --- a/src/BaGetter.Web/BaGetter.Web.csproj +++ b/src/BaGetter.Web/BaGetter.Web.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 NuGet BaGetter's NuGet server implementation @@ -24,3 +24,4 @@ + diff --git a/src/BaGetter.Web/BaGetterEndpointBuilder.cs b/src/BaGetter.Web/BaGetterEndpointBuilder.cs index c35fb4401..8f453f796 100644 --- a/src/BaGetter.Web/BaGetterEndpointBuilder.cs +++ b/src/BaGetter.Web/BaGetterEndpointBuilder.cs @@ -1,5 +1,7 @@ using System.Diagnostics.CodeAnalysis; +using BaGetter.Authentication; using BaGetter.Web; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing.Constraints; @@ -11,7 +13,13 @@ public class BaGetterEndpointBuilder { public void MapEndpoints(IEndpointRouteBuilder endpoints) { - endpoints.MapRazorPages(); + endpoints + .MapRazorPages() + .RequireAuthorization(new AuthorizeAttribute + { + AuthenticationSchemes = AuthenticationConstants.NugetBasicAuthenticationScheme, + Policy = AuthenticationConstants.NugetUserPolicy, + }); MapServiceIndexRoutes(endpoints); MapPackagePublishRoutes(endpoints); @@ -95,6 +103,11 @@ public void MapPackageMetadataRoutes(IEndpointRouteBuilder endpoints) pattern: "v3/registration/{id}/index.json", defaults: new { controller = "PackageMetadata", action = "RegistrationIndex" }); + endpoints.MapControllerRoute( + name: Routes.RegistrationPageRouteName, + pattern: "v3/registration/{id}/page/{lower}/{upper}.json", + defaults: new { controller = "PackageMetadata", action = "RegistrationPage" }); + endpoints.MapControllerRoute( name: Routes.RegistrationLeafRouteName, pattern: "v3/registration/{id}/{version}.json", diff --git a/src/BaGetter.Web/BaGetterUrlGenerator.cs b/src/BaGetter.Web/BaGetterUrlGenerator.cs index 0859bf9db..d795d9b08 100644 --- a/src/BaGetter.Web/BaGetterUrlGenerator.cs +++ b/src/BaGetter.Web/BaGetterUrlGenerator.cs @@ -78,8 +78,15 @@ public string GetRegistrationIndexUrl(string id) public string GetRegistrationPageUrl(string id, NuGetVersion lower, NuGetVersion upper) { - // BaGetter does not support paging the registration resource. - throw new NotImplementedException(); + return _linkGenerator.GetUriByRouteValues( + _httpContextAccessor.HttpContext, + Routes.RegistrationPageRouteName, + values: new + { + Id = id.ToLowerInvariant(), + Lower = lower.ToNormalizedString().ToLowerInvariant(), + Upper = upper.ToNormalizedString().ToLowerInvariant(), + }); } public string GetRegistrationLeafUrl(string id, NuGetVersion version) diff --git a/src/BaGetter.Web/Controllers/PackageContentController.cs b/src/BaGetter.Web/Controllers/PackageContentController.cs index 62e3bfdd1..84e58b916 100644 --- a/src/BaGetter.Web/Controllers/PackageContentController.cs +++ b/src/BaGetter.Web/Controllers/PackageContentController.cs @@ -1,10 +1,13 @@ using System; +using System.IO; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using BaGetter.Authentication; using BaGetter.Core; using BaGetter.Protocol.Models; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using NuGet.Versioning; @@ -35,6 +38,16 @@ public async Task> GetPackageVersionsAsync return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromText(JsonSerializer.Serialize(versions)); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); return versions; } @@ -52,12 +65,22 @@ public async Task DownloadPackageAsync(string id, string version, return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromParts("package", id.ToLowerInvariant(), nugetVersion.ToNormalizedString().ToLowerInvariant()); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + var packageStream = await _content.GetPackageContentStreamOrNullAsync(id, nugetVersion, cancellationToken); if (packageStream == null) { return NotFound(); } + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); return File(packageStream, "application/octet-stream"); } @@ -68,12 +91,22 @@ public async Task DownloadNuspecAsync(string id, string version, return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromParts("nuspec", id.ToLowerInvariant(), nugetVersion.ToNormalizedString().ToLowerInvariant()); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + var nuspecStream = await _content.GetPackageManifestStreamOrNullAsync(id, nugetVersion, cancellationToken); if (nuspecStream == null) { return NotFound(); } + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); return File(nuspecStream, "text/xml"); } @@ -84,12 +117,22 @@ public async Task DownloadReadmeAsync(string id, string version, return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromParts("readme", id.ToLowerInvariant(), nugetVersion.ToNormalizedString().ToLowerInvariant()); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + var readmeStream = await _content.GetPackageReadmeStreamOrNullAsync(id, nugetVersion, cancellationToken); if (readmeStream == null) { return NotFound(); } + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); return File(readmeStream, "text/markdown"); } @@ -100,12 +143,83 @@ public async Task DownloadIconAsync(string id, string version, Ca return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromParts("icon", id.ToLowerInvariant(), nugetVersion.ToNormalizedString().ToLowerInvariant()); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + var iconStream = await _content.GetPackageIconStreamOrNullAsync(id, nugetVersion, cancellationToken); if (iconStream == null) { return NotFound(); } - return File(iconStream, "image/xyz"); + await using var bufferedStream = new MemoryStream(); + await iconStream.CopyToAsync(bufferedStream, cancellationToken); + var iconBytes = bufferedStream.ToArray(); + + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetImmutable(Response); + return File(iconBytes, DetectImageContentType(iconBytes)); + } + + private static string DetectImageContentType(byte[] bytes) + { + if (bytes.Length >= 8 + && bytes[0] == 0x89 + && bytes[1] == 0x50 + && bytes[2] == 0x4E + && bytes[3] == 0x47 + && bytes[4] == 0x0D + && bytes[5] == 0x0A + && bytes[6] == 0x1A + && bytes[7] == 0x0A) + { + return "image/png"; + } + + if (bytes.Length >= 3 + && bytes[0] == 0xFF + && bytes[1] == 0xD8 + && bytes[2] == 0xFF) + { + return "image/jpeg"; + } + + if (bytes.Length >= 6 + && bytes[0] == 0x47 + && bytes[1] == 0x49 + && bytes[2] == 0x46 + && bytes[3] == 0x38 + && (bytes[4] == 0x37 || bytes[4] == 0x39) + && bytes[5] == 0x61) + { + return "image/gif"; + } + + if (bytes.Length >= 2 + && bytes[0] == 0x42 + && bytes[1] == 0x4D) + { + return "image/bmp"; + } + + if (bytes.Length >= 12 + && bytes[0] == 0x52 + && bytes[1] == 0x49 + && bytes[2] == 0x46 + && bytes[3] == 0x46 + && bytes[8] == 0x57 + && bytes[9] == 0x45 + && bytes[10] == 0x42 + && bytes[11] == 0x50) + { + return "image/webp"; + } + + return "application/octet-stream"; } } diff --git a/src/BaGetter.Web/Controllers/PackageMetadataController.cs b/src/BaGetter.Web/Controllers/PackageMetadataController.cs index 914068f5b..f6f9b001c 100644 --- a/src/BaGetter.Web/Controllers/PackageMetadataController.cs +++ b/src/BaGetter.Web/Controllers/PackageMetadataController.cs @@ -1,4 +1,5 @@ using System; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using BaGetter.Authentication; @@ -6,6 +7,7 @@ using BaGetter.Protocol.Models; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Http; using NuGet.Versioning; namespace BaGetter.Web; @@ -35,9 +37,52 @@ public async Task> RegistrationI return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromText(JsonSerializer.Serialize(index)); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); return index; } + // GET v3/registration/{id}/page/{lower}/{upper}.json + [HttpGet] + public async Task> RegistrationPageAsync( + string id, + string lower, + string upper, + CancellationToken cancellationToken) + { + if (!NuGetVersion.TryParse(lower, out var lowerVersion) + || !NuGetVersion.TryParse(upper, out var upperVersion)) + { + return NotFound(); + } + + var page = await _metadata.GetRegistrationPageOrNullAsync(id, lowerVersion, upperVersion, cancellationToken); + if (page == null) + { + return NotFound(); + } + + var etag = HttpCacheUtility.CreateStrongEtagFromText(JsonSerializer.Serialize(page)); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); + return page; + } + // GET v3/registration/{id}/{version}.json [HttpGet] public async Task> RegistrationLeafAsync(string id, string version, CancellationToken cancellationToken) @@ -53,6 +98,16 @@ public async Task> RegistrationLeafAsync( return NotFound(); } + var etag = HttpCacheUtility.CreateStrongEtagFromText(JsonSerializer.Serialize(leaf)); + if (HttpCacheUtility.MatchesIfNoneMatch(Request, etag)) + { + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); + return StatusCode(StatusCodes.Status304NotModified); + } + + HttpCacheUtility.SetEtag(Response, etag); + HttpCacheUtility.SetMustRevalidate(Response); return leaf; } } diff --git a/src/BaGetter.Web/Controllers/PackagePublishController.cs b/src/BaGetter.Web/Controllers/PackagePublishController.cs index 6add0ebf5..ab0683b91 100644 --- a/src/BaGetter.Web/Controllers/PackagePublishController.cs +++ b/src/BaGetter.Web/Controllers/PackagePublishController.cs @@ -37,9 +37,24 @@ public PackagePublishController( // See: https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package public async Task Upload(CancellationToken cancellationToken) { - if (_options.Value.IsReadOnlyMode || - !await _authentication.AuthenticateAsync(Request.GetApiKey(), cancellationToken)) + if (_options.Value.IsReadOnlyMode) + { + _logger.LogWarning( + "AUDIT package_upload_denied reason=read_only actor={Actor} ip={IpAddress}", + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); + + HttpContext.Response.StatusCode = 401; + return; + } + + if (!await _authentication.AuthenticateAsync(Request.GetApiKey(), cancellationToken)) { + _logger.LogWarning( + "AUDIT package_upload_denied reason=unauthorized actor={Actor} ip={IpAddress}", + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); + HttpContext.Response.StatusCode = 401; return; } @@ -49,6 +64,11 @@ public async Task Upload(CancellationToken cancellationToken) using var uploadStream = await Request.GetUploadStreamOrNullAsync(cancellationToken); if (uploadStream == null) { + _logger.LogWarning( + "AUDIT package_upload_failed reason=missing_body actor={Actor} ip={IpAddress}", + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); + HttpContext.Response.StatusCode = 400; return; } @@ -58,14 +78,26 @@ public async Task Upload(CancellationToken cancellationToken) switch (result) { case PackageIndexingResult.InvalidPackage: + _logger.LogInformation( + "AUDIT package_upload_failed reason=invalid_package actor={Actor} ip={IpAddress}", + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); HttpContext.Response.StatusCode = 400; break; case PackageIndexingResult.PackageAlreadyExists: + _logger.LogInformation( + "AUDIT package_upload_failed reason=already_exists actor={Actor} ip={IpAddress}", + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); HttpContext.Response.StatusCode = 409; break; case PackageIndexingResult.Success: + _logger.LogInformation( + "AUDIT package_upload_succeeded actor={Actor} ip={IpAddress}", + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); HttpContext.Response.StatusCode = 201; break; } @@ -83,6 +115,12 @@ public async Task Delete(string id, string version, CancellationT { if (_options.Value.IsReadOnlyMode) { + _logger.LogWarning( + "AUDIT package_delete_denied reason=read_only package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + version, + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return Unauthorized(); } @@ -91,17 +129,37 @@ public async Task Delete(string id, string version, CancellationT return NotFound(); } - if (!await _authentication.AuthenticateAsync(Request.GetApiKey(), cancellationToken)) + var isAdminUser = HttpContext.User?.IsInRole("Admin") == true; + var hasValidApiKey = await _authentication.AuthenticateAsync(Request.GetApiKey(), cancellationToken); + if (!isAdminUser && !hasValidApiKey) { + _logger.LogWarning( + "AUDIT package_delete_denied reason=unauthorized package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + version, + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return Unauthorized(); } if (await _deleteService.TryDeletePackageAsync(id, nugetVersion, cancellationToken)) { + _logger.LogInformation( + "AUDIT package_delete_succeeded package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + nugetVersion.ToNormalizedString(), + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return NoContent(); } else { + _logger.LogInformation( + "AUDIT package_delete_not_found package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + nugetVersion.ToNormalizedString(), + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return NotFound(); } } @@ -111,6 +169,12 @@ public async Task Relist(string id, string version, CancellationT { if (_options.Value.IsReadOnlyMode) { + _logger.LogWarning( + "AUDIT package_relist_denied reason=read_only package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + version, + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return Unauthorized(); } @@ -121,16 +185,39 @@ public async Task Relist(string id, string version, CancellationT if (!await _authentication.AuthenticateAsync(Request.GetApiKey(), cancellationToken)) { + _logger.LogWarning( + "AUDIT package_relist_denied reason=unauthorized package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + version, + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return Unauthorized(); } if (await _packages.RelistPackageAsync(id, nugetVersion, cancellationToken)) { + _logger.LogInformation( + "AUDIT package_relist_succeeded package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + nugetVersion.ToNormalizedString(), + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return Ok(); } else { + _logger.LogInformation( + "AUDIT package_relist_not_found package_id={PackageId} package_version={PackageVersion} actor={Actor} ip={IpAddress}", + id, + nugetVersion.ToNormalizedString(), + GetActor(), + HttpContext.Connection.RemoteIpAddress?.ToString()); return NotFound(); } } + + private string GetActor() + { + return HttpContext.User?.Identity?.Name ?? "anonymous"; + } } diff --git a/src/BaGetter.Web/Extensions/IApplicationBuilderExtensions.cs b/src/BaGetter.Web/Extensions/IApplicationBuilderExtensions.cs index 259a04218..d1ca0d249 100644 --- a/src/BaGetter.Web/Extensions/IApplicationBuilderExtensions.cs +++ b/src/BaGetter.Web/Extensions/IApplicationBuilderExtensions.cs @@ -11,4 +11,18 @@ public static IApplicationBuilder UseOperationCancelledMiddleware(this IApplicat return app.UseMiddleware(); } + + public static IApplicationBuilder UseRequestTelemetryMiddleware(this IApplicationBuilder app) + { + ArgumentNullException.ThrowIfNull(app); + + return app.UseMiddleware(); + } + + public static IApplicationBuilder UseSecurityHeadersMiddleware(this IApplicationBuilder app) + { + ArgumentNullException.ThrowIfNull(app); + + return app.UseMiddleware(); + } } diff --git a/src/BaGetter.Web/Extensions/IServiceCollectionExtensions.cs b/src/BaGetter.Web/Extensions/IServiceCollectionExtensions.cs index 2b08aecee..c4a356c80 100644 --- a/src/BaGetter.Web/Extensions/IServiceCollectionExtensions.cs +++ b/src/BaGetter.Web/Extensions/IServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using BaGetter.Web; using BaGetter.Web.Authentication; using BaGetter.Web.Helper; +using BaGetter.Web.Admin; using Microsoft.AspNetCore.Authorization; using Microsoft.Extensions.DependencyInjection; @@ -25,10 +26,16 @@ public static IServiceCollection AddBaGetterWebApplication( options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; }); - services.AddRazorPages(); + services.AddLocalization(options => options.ResourcesPath = "Resources"); + + services + .AddRazorPages() + .AddViewLocalization(); services.AddHttpContextAccessor(); services.AddTransient(); + services.AddTransient(); + services.AddSingleton(); services.AddSingleton(ApplicationVersionHelper.GetVersion()); diff --git a/src/BaGetter.Web/HttpCacheUtility.cs b/src/BaGetter.Web/HttpCacheUtility.cs new file mode 100644 index 000000000..9fe5e7870 --- /dev/null +++ b/src/BaGetter.Web/HttpCacheUtility.cs @@ -0,0 +1,88 @@ +using System; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Microsoft.AspNetCore.Http; +using Microsoft.Net.Http.Headers; + +namespace BaGetter.Web; + +internal static class HttpCacheUtility +{ + public static string CreateStrongEtagFromBytes(ReadOnlySpan bytes) + { + var hash = SHA256.HashData(bytes); + return $"\"{Convert.ToHexString(hash)}\""; + } + + public static string CreateStrongEtagFromText(string text) + { + ArgumentNullException.ThrowIfNull(text); + return CreateStrongEtagFromBytes(Encoding.UTF8.GetBytes(text)); + } + + public static string CreateStrongEtagFromParts(params string[] parts) + { + ArgumentNullException.ThrowIfNull(parts); + return CreateStrongEtagFromText(string.Join('|', parts.Select(part => part ?? string.Empty))); + } + + public static bool MatchesIfNoneMatch(HttpRequest request, string etag) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(etag); + + var header = request.Headers.IfNoneMatch.ToString(); + if (string.IsNullOrWhiteSpace(header)) + { + return false; + } + + var normalizedCurrent = NormalizeEtag(etag); + var candidates = header.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var candidate in candidates) + { + if (candidate == "*") + { + return true; + } + + if (NormalizeEtag(candidate).Equals(normalizedCurrent, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + public static void SetEtag(HttpResponse response, string etag) + { + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(etag); + response.Headers[HeaderNames.ETag] = etag; + } + + public static void SetMustRevalidate(HttpResponse response) + { + ArgumentNullException.ThrowIfNull(response); + response.Headers[HeaderNames.CacheControl] = "public, max-age=0, must-revalidate"; + } + + public static void SetImmutable(HttpResponse response) + { + ArgumentNullException.ThrowIfNull(response); + response.Headers[HeaderNames.CacheControl] = "public, max-age=31536000, immutable"; + } + + private static string NormalizeEtag(string etag) + { + var trimmed = etag.Trim(); + if (trimmed.StartsWith("W/", StringComparison.Ordinal)) + { + trimmed = trimmed[2..].Trim(); + } + + return trimmed; + } +} diff --git a/src/BaGetter.Web/Localization/SharedResource.cs b/src/BaGetter.Web/Localization/SharedResource.cs new file mode 100644 index 000000000..318046178 --- /dev/null +++ b/src/BaGetter.Web/Localization/SharedResource.cs @@ -0,0 +1,6 @@ +namespace BaGetter.Web; + +public class SharedResource +{ +} + diff --git a/src/BaGetter.Web/Pages/Admin/Index.cshtml b/src/BaGetter.Web/Pages/Admin/Index.cshtml new file mode 100644 index 000000000..2cfb51a80 --- /dev/null +++ b/src/BaGetter.Web/Pages/Admin/Index.cshtml @@ -0,0 +1,270 @@ +@page "/admin" +@model BaGetter.Web.Pages.Admin.IndexModel +@{ + ViewData["Title"] = "Admin dashboard"; + var query = Request.Query["q"].ToString(); + var status = Request.Query["status"].ToString(); +} + +
+
+
+ PACKAGE OPERATIONS +

Admin dashboard

+

Manage package visibility, identity, copies, permanent deletion, and audit history.

+
+
+ + @User.Identity?.Name +
+
+ + @if (!string.IsNullOrWhiteSpace(Model.StatusMessage)) + { +
+ @Model.StatusMessage +
+ } + + @if (Model.IsReadOnly) + { + + } + +
+
+ Total versions + @Model.TotalPackages + All indexed package versions +
+
+ Listed + @Model.ListedPackages + Visible to package clients +
+
+ Unlisted + @Model.UnlistedPackages + Hidden but still downloadable +
+
+ Audit events + @Model.AuditEntries.Count + Latest recorded operations +
+
+ +
+
+
+

Packages

+

@Model.FilteredCount matching versions

+
+
+ + + + + + @if (!string.IsNullOrWhiteSpace(query) || !string.IsNullOrWhiteSpace(status)) + { + Clear + } +
+
+ +
+ + + + + + + + + + + + + @if (Model.Packages.Count == 0) + { + + } + @foreach (var package in Model.Packages) + { + + + + + + + + + } + +
PackageVersionStatusPublishedDownloadsActions
No package versions match the current filters.
+ @package.Id + @if (!string.IsNullOrWhiteSpace(package.Description)) + { + @package.Description + } + @package.Version + + @(package.Listed ? "Listed" : "Unlisted") + + @package.Downloads.ToString("N0") +
+
+ + + + + + + +
+ + + +
+
+
+ + @if (Model.TotalPages > 1) + { + + } +
+ +
+
+
+

Activity log

+

Who changed what, when, and from which address.

+
+
+
+ @if (Model.AuditEntries.Count == 0) + { +

No administrative operations have been recorded.

+ } + @foreach (var entry in Model.AuditEntries) + { +
+
+ +
+
+ @entry.Actor + @BaGetter.Web.Pages.Admin.IndexModel.GetActionLabel(entry.Action) + @entry.PackageId @entry.PackageVersion + @if (!string.IsNullOrWhiteSpace(entry.Details)) + { + @entry.Details + } +
+
+ @entry.Result + + @entry.IpAddress +
+
+ } +
+
+
+ + + +@section Scripts { + +} diff --git a/src/BaGetter.Web/Pages/Admin/Index.cshtml.cs b/src/BaGetter.Web/Pages/Admin/Index.cshtml.cs new file mode 100644 index 000000000..bccd16576 --- /dev/null +++ b/src/BaGetter.Web/Pages/Admin/Index.cshtml.cs @@ -0,0 +1,190 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using BaGetter.Authentication; +using BaGetter.Core; +using BaGetter.Web.Admin; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Logging; +using NuGet.Versioning; + +namespace BaGetter.Web.Pages.Admin; + +[Authorize( + AuthenticationSchemes = AuthenticationConstants.NugetBasicAuthenticationScheme, + Policy = AuthenticationConstants.AdminPolicy)] +public sealed class IndexModel : PageModel +{ + private const int PageSize = 25; + private readonly IPackageDatabase _packages; + private readonly IAdminPackageService _adminPackages; + private readonly IAdminAuditLog _auditLog; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public IndexModel( + IPackageDatabase packages, + IAdminPackageService adminPackages, + IAdminAuditLog auditLog, + IOptionsSnapshot options, + ILogger logger) + { + _packages = packages ?? throw new ArgumentNullException(nameof(packages)); + _adminPackages = adminPackages ?? throw new ArgumentNullException(nameof(adminPackages)); + _auditLog = auditLog ?? throw new ArgumentNullException(nameof(auditLog)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public IReadOnlyList Packages { get; private set; } = []; + public IReadOnlyList AuditEntries { get; private set; } = []; + public int TotalPackages { get; private set; } + public int ListedPackages { get; private set; } + public int UnlistedPackages { get; private set; } + public int FilteredCount { get; private set; } + public int CurrentPage { get; private set; } + public int TotalPages => Math.Max(1, (int)Math.Ceiling(FilteredCount / (double)PageSize)); + public bool IsReadOnly => _options.Value.IsReadOnlyMode; + + public static string GetActionLabel(string action) + { + return action switch + { + "unlist" => "Unlisted", + "relist" => "Relisted", + "copy" => "Copied", + "rename" => "Renamed", + "delete" => "Permanently deleted", + _ => action, + }; + } + + [TempData] + public string StatusMessage { get; set; } + + [TempData] + public bool StatusSucceeded { get; set; } + + public async Task OnGetAsync( + string q, + string status, + int page = 1, + CancellationToken cancellationToken = default) + { + CurrentPage = Math.Max(1, page); + var listed = ParseListedFilter(status); + + TotalPackages = await _packages.CountAsync(null, null, cancellationToken); + ListedPackages = await _packages.CountAsync(null, true, cancellationToken); + UnlistedPackages = TotalPackages - ListedPackages; + FilteredCount = await _packages.CountAsync(q, listed, cancellationToken); + Packages = await _packages.SearchAsync( + q, + listed, + (CurrentPage - 1) * PageSize, + PageSize, + cancellationToken); + AuditEntries = await _auditLog.ReadLatestAsync(50, cancellationToken); + } + + public async Task OnPostManageAsync( + string operation, + string id, + string version, + string targetId, + string confirmation, + string q, + string status, + int page, + CancellationToken cancellationToken) + { + var normalizedOperation = operation?.Trim().ToLowerInvariant() ?? string.Empty; + AdminPackageOperationResult result; + + if (_options.Value.IsReadOnlyMode) + { + result = AdminPackageOperationResult.Failure("The server is in read-only mode."); + } + else if (!NuGetVersion.TryParse(version, out var nugetVersion)) + { + result = AdminPackageOperationResult.Failure("The package version is invalid."); + } + else if (normalizedOperation == "delete" && + !string.Equals(confirmation?.Trim(), id, StringComparison.Ordinal)) + { + result = AdminPackageOperationResult.Failure( + "Permanent delete was cancelled because the confirmation did not match the package ID."); + } + else + { + try + { + result = normalizedOperation switch + { + "unlist" => await _adminPackages.SetListedAsync( + id, nugetVersion, listed: false, cancellationToken), + "relist" => await _adminPackages.SetListedAsync( + id, nugetVersion, listed: true, cancellationToken), + "delete" => await _adminPackages.HardDeleteAsync( + id, nugetVersion, cancellationToken), + "copy" => await _adminPackages.CopyAsync( + id, nugetVersion, targetId, cancellationToken), + "rename" => await _adminPackages.RenameAsync( + id, nugetVersion, targetId, cancellationToken), + _ => AdminPackageOperationResult.Failure("Unknown package operation."), + }; + } + catch (Exception exception) + { + _logger.LogError( + exception, + "Administrative package operation {Operation} failed for {PackageId} {PackageVersion}", + normalizedOperation, + id, + version); + result = AdminPackageOperationResult.Failure( + "The operation failed unexpectedly. Check the server log for details."); + } + } + + var actor = User.Identity?.Name ?? "unknown"; + await _auditLog.WriteAsync( + new AdminAuditEntry + { + TimestampUtc = DateTimeOffset.UtcNow, + Actor = actor, + Action = normalizedOperation, + PackageId = id, + PackageVersion = version, + Details = BuildAuditDetails(targetId, result.Message), + Result = result.Succeeded ? "Succeeded" : "Failed", + IpAddress = HttpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown", + }, + cancellationToken); + + StatusMessage = result.Message; + StatusSucceeded = result.Succeeded; + return RedirectToPage("/Admin/Index", new { q, status, page }); + } + + private static bool? ParseListedFilter(string status) + { + return status?.ToLowerInvariant() switch + { + "listed" => true, + "unlisted" => false, + _ => null, + }; + } + + private static string BuildAuditDetails(string targetId, string message) + { + return string.IsNullOrWhiteSpace(targetId) + ? message + : $"targetId={targetId.Trim()}; {message}"; + } +} diff --git a/src/BaGetter.Web/Pages/Index.cshtml b/src/BaGetter.Web/Pages/Index.cshtml index 4595be5fa..e6b3a8cff 100644 --- a/src/BaGetter.Web/Pages/Index.cshtml +++ b/src/BaGetter.Web/Pages/Index.cshtml @@ -1,8 +1,9 @@ @page @model IndexModel +@inject Microsoft.AspNetCore.Mvc.Localization.IHtmlLocalizer Localizer @{ - ViewData["Title"] = "Packages"; + ViewData["Title"] = Localizer["Packages"]; ViewData["SearchQuery"] = Model.Query; } @@ -28,18 +29,18 @@
- +
@@ -47,12 +48,12 @@
- +
@@ -77,7 +78,7 @@
- + -

Oops, nothing here...

+

@Localizer["Oops, nothing here..."]

- It looks like there's no package here to see. Take a look below for useful links. + @Localizer["It looks like there's no package here to see. Take a look below for useful links."]

-

Upload a package

-

BaGetter documentation

-

BaGetter issues

+

@Localizer["Upload a package"]

+

@Localizer["BaGetter documentation"]

+

@Localizer["BaGetter issues"]

} else @@ -106,8 +107,8 @@ else
@@ -118,27 +119,64 @@ else class="package-title"> @package.PackageId - by: @string.Join(", ", package.Authors) + @{ + var authors = (package.Authors ?? Array.Empty()) + .Where(a => !string.IsNullOrWhiteSpace(a)) + .ToArray(); + } + + @Localizer["by"]: + @if (authors.Length == 0) + { + @Localizer["unknown"] + } + else + { + for (var i = 0; i < authors.Length; i++) + { + if (i > 0) + { + @: , + } + @authors[i] + } + } + + + @if (Model.CanDeletePackages) + { + + }
  • - @package.TotalDownloads.ToMetric() total downloads + @package.TotalDownloads.ToMetric() @Localizer["total downloads"]
  • - Latest version: @package.Version + @Localizer["Latest version"]: @package.Version
    • - @if (package.Tags.Any()) + @if (package.Tags?.Any() == true) {
  • - @string.Join(" ", package.Tags) + @foreach (var tag in package.Tags.Where(t => !string.IsNullOrWhiteSpace(t))) + { + @tag@: + }
    • } @@ -159,18 +197,97 @@ else } } +@section Scripts +{ + +} + @functions { private IReadOnlyDictionary PackageTypes { get; } = new Dictionary() @@ -240,12 +357,14 @@ else }; private string SearchLink( + string query = null, string packageType = null, string framework = null, bool? prerelease = null, int? page = null) { // Use model's value if the argument was skipped. + query = query ?? Model.Query; packageType = packageType ?? Model.PackageType; framework = framework ?? Model.Framework; prerelease = prerelease ?? Model.Prerelease; @@ -259,11 +378,23 @@ else return Url.Page("Index", new { - q = Model.Query, + q = query, p = page, packageType, framework, prerelease, }); } + + private static string FilterQuery(string prefix, string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return $"{prefix}:"; + } + + return value.Any(char.IsWhiteSpace) + ? $"{prefix}:\"{value}\"" + : $"{prefix}:{value}"; + } } diff --git a/src/BaGetter.Web/Pages/Index.cshtml.cs b/src/BaGetter.Web/Pages/Index.cshtml.cs index ee8702689..3c6c7f92c 100644 --- a/src/BaGetter.Web/Pages/Index.cshtml.cs +++ b/src/BaGetter.Web/Pages/Index.cshtml.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Threading; @@ -7,16 +7,21 @@ using BaGetter.Protocol.Models; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.RazorPages; +using Microsoft.Extensions.Options; namespace BaGetter.Web; public class IndexModel : PageModel { private readonly ISearchService _search; + private readonly IOptionsSnapshot _options; - public IndexModel(ISearchService search) + public IndexModel( + ISearchService search, + IOptionsSnapshot options) { _search = search ?? throw new ArgumentNullException(nameof(search)); + _options = options ?? throw new ArgumentNullException(nameof(options)); } public const int ResultsPerPage = 20; @@ -38,6 +43,8 @@ public IndexModel(ISearchService search) public bool Prerelease { get; set; } = true; public IReadOnlyList Packages { get; private set; } + public bool IsReadOnlyMode => _options.Value.IsReadOnlyMode; + public bool CanDeletePackages => !IsReadOnlyMode && User.IsInRole("Admin"); public async Task OnGetAsync(CancellationToken cancellationToken) { diff --git a/src/BaGetter.Web/Pages/Package.cshtml b/src/BaGetter.Web/Pages/Package.cshtml index 61988b631..e7616f7ab 100644 --- a/src/BaGetter.Web/Pages/Package.cshtml +++ b/src/BaGetter.Web/Pages/Package.cshtml @@ -1,5 +1,6 @@ @page "/packages/{id}/{version?}" @model PackageModel +@inject Microsoft.AspNetCore.Mvc.Localization.IHtmlLocalizer Localizer @{ ViewData["Title"] = Model.Found @@ -10,10 +11,10 @@ @if (!Model.Found) {
    -

    Oops, package not found...

    -

    Could not find package '@Model.Package.Id'.

    -

    You can try searching on nuget.org package.

    -

    Think there's a problem? Consider taking a look at our documentation or asking for help on our GitHub project

    +

    @Localizer["OopsPackageNotFound"]

    +

    @Localizer["CouldNotFindPackage", Model.Package.Id]

    +

    @Localizer["TrySearchingOn"] nuget.org @Localizer["PackageSuffix"]

    +

    @Localizer["ThinkTheresAProblemPrefix"] @Localizer["Documentation"] @Localizer["OrAskingForHelpOn"] GitHub

    } else @@ -21,8 +22,8 @@ else
    @@ -61,7 +62,7 @@ else
    @{ - ExpandableSection("Readme", "ms-Icon--Dictionary", expanded: false, + ExpandableSection(Localizer["Readme"].Value, "ms-Icon--Dictionary", expanded: false, @
    @if (Model.Readme == null) { @@ -76,21 +77,21 @@ else @if (!Model.IsDotnetTemplate && !Model.IsDotnetTool) { - ExpandableSection("Used By", "ms-Icon--BranchFork2", expanded: false, + ExpandableSection(Localizer["UsedBy"].Value, "ms-Icon--BranchFork2", expanded: false, @
    @if (!Model.UsedBy.Any()) { -

    No packages depend on @Model.Package.Id.

    +

    @Localizer["NoPackagesDependOn", Model.Package.Id]

    } else { -

    Showing the top 20 packages that depend on @Model.Package.Id.

    +

    @Localizer["ShowingTop20PackagesThatDependOn", Model.Package.Id]

    - - + + @@ -116,15 +117,15 @@ else @if (!string.IsNullOrEmpty(Model.Package.ReleaseNotes)) { - ExpandableSection("Release Notes", "ms-Icon--ReadingMode", expanded: false, + ExpandableSection(Localizer["ReleaseNotes"].Value, "ms-Icon--ReadingMode", expanded: false, @
    @Model.ParsedReleaseNotes
    ); } - @{ ExpandableSection("Dependencies", "ms-Icon--Packages", expanded: false, + @{ ExpandableSection(Localizer["Dependencies"].Value, "ms-Icon--Packages", expanded: false, @
    @if (!Model.DependencyGroups.Any()) { - @:This package has no dependencies. + @Localizer["ThisPackageHasNoDependencies"] } else { @@ -137,7 +138,7 @@ else
      @if (!group.Dependencies.Any()) { -
    • No dependencies.
      • +
    • @Localizer["NoDependencies"]
      • } else { @@ -160,14 +161,18 @@ else
    ); } - @{ ExpandableSection("Versions", "ms-Icon--Stopwatch", expanded: true, + @{ ExpandableSection(Localizer["Versions"].Value, "ms-Icon--Stopwatch", expanded: true, @
    PackagesDownloads@Localizer["Packages"]@Localizer["Downloads"]
    - - - + + + + @if (Model.CanDeletePackages) + { + + } @@ -180,6 +185,19 @@ else + @if (Model.CanDeletePackages) + { + + } } @@ -189,7 +207,7 @@ else { } @@ -199,19 +217,19 @@ else
    VersionDownloadsLast updated@Localizer["Version"]@Localizer["Downloads"]@Localizer["LastUpdated"]@Localizer["Actions"]
    @version.Version.ToNormalizedString() @version.Downloads.ToString("N0") @version.LastUpdated.ToShortDateString() + +