From 10c5f69c8cea5f114821ee879e21c02642c40dfe Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Fri, 28 Aug 2026 15:29:03 -0300 Subject: [PATCH 01/10] Build and publish a signed multi-arch loreserver image to GHCR ## Summary Fills in the publish workflow: each architecture builds on its own native runner and is pushed to GHCR by digest, a merge job stitches the digests into one manifest list, and cosign signs that list keylessly through the job's OIDC token. No secrets are involved. Also drops the Graviton3+ target-cpu from the arm64 container build. The repo tunes aarch64-unknown-linux-gnu for Epic's own deployment, which emits instructions that fault on older arm64 parts and, because the build passes no --target, reaches the proc macros cargo runs on the build host too. A published image has to run anywhere armv8-a runs. ## Test Plan 1. actionlint passes on the workflow. 2. Dispatch from this branch and confirm the manifest list carries linux/amd64 and linux/arm64, and that cosign verify succeeds. 3. Pull the arm64 image on a non-Graviton arm64 host and confirm the server starts. Signed-off-by: Aleksander Arruda --- .../workflows/publish-loreserver-image.yml | 206 +++++++++++++++++- lore-server/DOCKER.md | 11 +- lore-server/Dockerfile | 17 ++ 3 files changed, 221 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index fc558dca8..bdec0de95 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -3,13 +3,29 @@ name: Publish loreserver image -# Stub. Manual trigger only, so the workflow is registered on the default -# branch and can be dispatched against a pull request branch. The multi-arch -# build of lore-server/Dockerfile, the push to -# ghcr.io/epicgames/lore/loreserver and keyless cosign signing land in a -# follow-up pull request. +# Builds lore-server/Dockerfile for linux/amd64 and linux/arm64, publishes a +# multi-arch image to ghcr.io/epicgames/lore/loreserver, and signs it with +# keyless cosign so the community Helm chart can reference a public, +# verifiable tag. +# +# Each architecture builds on its own native runner and is pushed by digest +# alone; the merge job stitches those digests into one manifest list and signs +# that. Native runners rather than QEMU, because a release Rust build under +# emulation is far too slow to be practical. +# +# No secrets are involved: the built-in GITHUB_TOKEN authenticates to GHCR, +# and cosign signs keylessly against Fulcio via the job's OIDC token. on: + push: + tags: ["v*"] workflow_dispatch: + inputs: + tag: + description: >- + Extra tag to publish, for proving the workflow from a branch + (for example "edge"). Blank tags from the ref alone. + type: string + required: false permissions: {} @@ -17,11 +33,181 @@ concurrency: group: publish-loreserver-image-${{ github.ref }} cancel-in-progress: false +env: + IMAGE: ghcr.io/epicgames/lore/loreserver + jobs: - publish: - name: publish + build: + name: build (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + permissions: + contents: read # Check out the source the image is built from + packages: write # Push the per-arch manifest, addressed by digest + strategy: + fail-fast: false + matrix: + include: + - { arch: amd64, platform: linux/amd64, runner: ubuntu-latest } + - { arch: arm64, platform: linux/arm64, runner: ubuntu-24.04-arm } + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # A release build of the workspace does not fit next to the runner + # image's preinstalled toolchains. Reclaim the ones this build never + # touches before BuildKit starts writing layers. + - name: Free disk space + run: | + sudo rm -rf \ + /usr/share/dotnet \ + /usr/local/lib/android \ + /opt/ghc \ + /usr/local/share/boost \ + "${AGENT_TOOLSDIRECTORY:?}" + df -h / + + - id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ${{ env.IMAGE }} + + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: lore-server/Dockerfile + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + # Only the apt and toolchain layers survive a source change. The + # cargo registry and target directory live in BuildKit cache mounts, + # which the gha backend does not carry between runs, so the compile + # itself starts cold every time. + cache-from: type=gha,scope=loreserver-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=loreserver-${{ matrix.arch }} + + # The merge job addresses each architecture by digest. Hand the digests + # over as empty files named after themselves. + - name: Export digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + mkdir -p "${{ runner.temp }}/digests" + touch "${{ runner.temp }}/digests/${DIGEST#sha256:}" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: digests-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: merge and sign + needs: build runs-on: ubuntu-latest + permissions: + packages: write # Push the manifest list and the cosign signature + id-token: write # Federate to Fulcio for keyless signing steps: - # TODO: build, push and sign the multi-arch image. - - name: Publish - run: echo "publish-loreserver-image stub — no build or publish yet" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + + - id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ${{ env.IMAGE }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=${{ inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.tag != '' }} + + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create the multi-arch manifest list + working-directory: ${{ runner.temp }}/digests + run: | + set -euo pipefail + # shellcheck disable=SC2046 # word splitting is how both lists are built + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "${IMAGE}@sha256:%s " *) + + - name: Inspect the manifest list + run: docker buildx imagetools inspect "${IMAGE}:${VERSION}" + env: + VERSION: ${{ steps.meta.outputs.version }} + + - id: digest + name: Resolve the manifest list digest + env: + VERSION: ${{ steps.meta.outputs.version }} + run: | + set -euo pipefail + digest=$(docker buildx imagetools inspect "${IMAGE}:${VERSION}" \ + --format '{{json .Manifest}}' | jq -r .digest) + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # Sign the manifest list by digest, not by tag: a tag can later be moved + # to point at something else, a digest cannot. + - name: Sign the image with cosign + env: + DIGEST: ${{ steps.digest.outputs.digest }} + run: cosign sign --yes "${IMAGE}@${DIGEST}" + + - name: Verify the signature + env: + DIGEST: ${{ steps.digest.outputs.digest }} + run: | + set -euo pipefail + cosign verify "${IMAGE}@${DIGEST}" \ + --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + > /dev/null + echo "signature verified" + + - name: Summarise what was published + env: + DIGEST: ${{ steps.digest.outputs.digest }} + run: | + set -euo pipefail + { + echo "## Published \`${IMAGE}\`" + echo "" + echo "Digest: \`${DIGEST}\`" + echo "" + echo "Platforms: linux/amd64, linux/arm64" + echo "" + echo "Tags:" + jq -r '.tags[] | "- `" + . + "`"' <<< "$DOCKER_METADATA_OUTPUT_JSON" + echo "" + echo "Verify the signature with:" + echo "" + echo '```sh' + echo "cosign verify ${IMAGE}@${DIGEST} \\" + echo " --certificate-identity-regexp '^https://github.com/${GITHUB_REPOSITORY}/' \\" + echo " --certificate-oidc-issuer https://token.actions.githubusercontent.com" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/lore-server/DOCKER.md b/lore-server/DOCKER.md index 688565dd5..28070c21d 100644 --- a/lore-server/DOCKER.md +++ b/lore-server/DOCKER.md @@ -6,17 +6,22 @@ telemetry integration, or replication is configured. ## Prerequisites - Docker with BuildKit support -- On Apple Silicon (M-series Macs), builds must target `linux/amd64` due to Graviton-specific - compiler flags in `.cargo/config.toml` for `aarch64-unknown-linux-gnu` + +Both `linux/amd64` and `linux/arm64` build. The Dockerfile overrides the Graviton-specific +`target-cpu` that `.cargo/config.toml` sets for `aarch64-unknown-linux-gnu`, so the arm64 +image runs on any armv8-a host rather than Graviton3 and newer only. ## Building From the repository root: ```sh -docker build --platform linux/amd64 -f lore-server/Dockerfile -t loreserver . +docker build -f lore-server/Dockerfile -t loreserver . ``` +Pass `--platform linux/amd64` or `--platform linux/arm64` to cross-build; expect it to be slow, +since a release Rust build under emulation is far slower than a native one. + The build compiles the `loreserver` binary and generates self-signed TLS certificates for QUIC using `scripts/server/make-certs.sh`. diff --git a/lore-server/Dockerfile b/lore-server/Dockerfile index 1c3245877..8e3cd4e8a 100644 --- a/lore-server/Dockerfile +++ b/lore-server/Dockerfile @@ -8,6 +8,23 @@ RUN apt-get update && apt-get install -y \ WORKDIR /build COPY . . +# .cargo/config.toml tunes aarch64-unknown-linux-gnu for Epic's own Graviton3+ +# deployment with -C target-cpu=neoverse-512tvb. That emits instructions which +# fault on older arm64 parts, and because this build passes no --target, the +# same flags reach the proc macros cargo then executes on the build host. A +# published image has to run anywhere armv8-a runs, so drop the tuning. +# +# The flags kept below are the ones that block would otherwise supply: a +# [target.*] rustflags table fully replaces the [build] list rather than +# merging with it, and the two --cfg values are load-bearing — the code does +# not compile without them. +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="--cfg tokio_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes --cfg uuid_unstable" + +# Epic's own builds keep full debug symbols in the server binary. An image +# pulled over the network does not want the DWARF; drop it but keep the symbol +# table so backtraces still name their frames. +ENV CARGO_PROFILE_RELEASE_STRIP=debuginfo + RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/target \ From 72afff5d785faa94448f7a1653fb208e14b5ec4c Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Fri, 28 Aug 2026 15:49:05 -0300 Subject: [PATCH 02/10] Drop the disk-reclaim step from the publish workflow The first full run measured 145G on the runner root with 108G still free after both release builds, so deleting unrelated preinstalled toolchains buys nothing and only adds a step that can fail on its own. Signed-off-by: Aleksander Arruda --- .github/workflows/publish-loreserver-image.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index bdec0de95..7ef0dbafe 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -54,19 +54,6 @@ jobs: with: persist-credentials: false - # A release build of the workspace does not fit next to the runner - # image's preinstalled toolchains. Reclaim the ones this build never - # touches before BuildKit starts writing layers. - - name: Free disk space - run: | - sudo rm -rf \ - /usr/share/dotnet \ - /usr/local/lib/android \ - /opt/ghc \ - /usr/local/share/boost \ - "${AGENT_TOOLSDIRECTORY:?}" - df -h / - - id: meta uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: From fb89c97d62cbe865e0d8821570d24f0bdaaba057 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Fri, 28 Aug 2026 16:24:01 -0300 Subject: [PATCH 03/10] Fix the arm64 image faulting with SIGILL, and run the binary in CI ## Summary The previous attempt set CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS to drop the Graviton3+ target-cpu. That does not work: the variable is another source for the same config key, and cargo joins config arrays rather than replacing them, so -C target-cpu=neoverse-512tvb survived and the published arm64 image died with SIGILL on any older arm64 part. Use RUSTFLAGS instead, which is a mutually exclusive source that suppresses [build] and [target.*] rustflags outright, and repeat the flags the build actually needs. Also stop overriding the release strip setting. The workspace asks for line-tables-only debug info on purpose, and the override was discarding the line numbers that make a panic backtrace useful. Finally, run the freshly pushed image on its native runner. Nothing in the workflow executed the binary, which is exactly why a SIGILL shipped past a fully green run. ## Test Plan 1. actionlint passes. 2. The new smoke step fails the arm64 job when the Graviton flag is present and passes once it is gone. 3. Pull the arm64 image on Apple Silicon, which has no SVE, and confirm loreserver --version prints. Signed-off-by: Aleksander Arruda --- .../workflows/publish-loreserver-image.yml | 9 +++++++ lore-server/Dockerfile | 26 +++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index 7ef0dbafe..7ec4fde9c 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -83,6 +83,15 @@ jobs: cache-from: type=gha,scope=loreserver-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=loreserver-${{ matrix.arch }} + # A clean build proves only that the image compiles. Run the binary on + # the native runner as well: a codegen flag aimed at the wrong CPU + # produces an image that builds happily and then dies with SIGILL the + # first time anyone starts it. + - name: Smoke test the pushed image + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: docker run --rm "${IMAGE}@${DIGEST}" --version + # The merge job addresses each architecture by digest. Hand the digests # over as empty files named after themselves. - name: Export digest diff --git a/lore-server/Dockerfile b/lore-server/Dockerfile index 8e3cd4e8a..e8482f8d5 100644 --- a/lore-server/Dockerfile +++ b/lore-server/Dockerfile @@ -9,21 +9,19 @@ WORKDIR /build COPY . . # .cargo/config.toml tunes aarch64-unknown-linux-gnu for Epic's own Graviton3+ -# deployment with -C target-cpu=neoverse-512tvb. That emits instructions which -# fault on older arm64 parts, and because this build passes no --target, the -# same flags reach the proc macros cargo then executes on the build host. A -# published image has to run anywhere armv8-a runs, so drop the tuning. +# deployment with -C target-cpu=neoverse-512tvb, which emits instructions that +# fault on any older arm64 part. A published image has to run anywhere armv8-a +# runs, so that tuning has to go. # -# The flags kept below are the ones that block would otherwise supply: a -# [target.*] rustflags table fully replaces the [build] list rather than -# merging with it, and the two --cfg values are load-bearing — the code does -# not compile without them. -ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="--cfg tokio_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes --cfg uuid_unstable" - -# Epic's own builds keep full debug symbols in the server binary. An image -# pulled over the network does not want the DWARF; drop it but keep the symbol -# table so backtraces still name their frames. -ENV CARGO_PROFILE_RELEASE_STRIP=debuginfo +# It has to go via RUSTFLAGS specifically. CARGO_TARGET__RUSTFLAGS is +# only another source for the same config key, and cargo *joins* config arrays +# rather than replacing them, so setting it leaves the Graviton flag in place +# and the image still dies with SIGILL. RUSTFLAGS is a separate, mutually +# exclusive source: when it is set, cargo ignores [build] and [target.*] +# rustflags entirely. So everything the build needs has to be repeated here. +# The two --cfg values are load-bearing, the code does not compile without +# them; the [build] clippy -W lints are dropped, as they only affect warnings. +ENV RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes" RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ From aaee0685d298fcc1b4ce1d6c5ef00f86cdd562f0 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Tue, 1 Sep 2026 07:41:02 -0300 Subject: [PATCH 04/10] Harden the publish workflow and build the real release profile ## Summary Review of the publish workflow turned up six problems, three of them able to publish or sign the wrong thing. Tags and labels are now resolved once, in a new `meta` job that runs before anything is built. That addresses three of them together: - `latest` was gated only on the ref starting with refs/tags/v, so v1.0.0-rc.1 moved and signed `latest`, and a v-prefixed tag that is not semver at all left `latest` as the only tag the run published. The default `latest=auto` flavor applies it only to a tag that parses as a stable semver release. - A workflow_dispatch from a branch with a blank tag input resolved to no tags at all, and the job then died on `imagetools inspect ghcr.io/...:`, after both arch images had been pushed. A type=ref,event=branch entry gives that path a tag from the ref alone, as the input description already promised, and a gate in the meta job fails the run before any layer reaches GHCR if the list is still empty. - The build job ran its own metadata-action with no tags config, so the labels baked into the image carried the ref-derived version (v1.2.3, or a branch name) while the published tag was 1.2.3. And three smaller ones: - The tag input was interpolated straight into the tags list, where an embedded newline would have smuggled in a further directive. It is held to Docker's tag grammar before it gets there. - imagetools create never received the annotations, so the published index carried none. Building its argument list as an array also removes the word splitting the old command relied on, which would have broken on the spaces in image.description. - The cosign identity regexp was anchored on the repository alone, so it accepted a signature from any workflow here that can ask for an OIDC token. It is anchored on this workflow's path now, and defined once, since the summary hands the same expression to downstream users. Separately, the image was built with --release, which Cargo.toml labels a fast build suitable for local development: debug-assertions, and with them overflow-checks, stay on and there is no LTO. Acceptable for a local convenience image, not for the artifact the Helm chart points at. Build release-lto, "the actual release build", and strip the full DWARF it asks for back out, keeping the symbol table so backtraces still name frames. ## Test Plan 1. actionlint, with the repo config, reports no errors. 2. docker buildx build --check on the Dockerfile is clean. 3. The tag validator accepts edge, 1.2.3 and v1.0.0-rc.1, and rejects an embedded newline, a leading dash, a leading dot, a space, a slash and a 129-character input. 4. The imagetools argument construction, dry-run against a representative metadata-action JSON, keeps image.description as one argument. 5. Still to confirm on the first run: fat LTO links inside the memory and time a 4-core runner has, a prerelease tag does not move latest, and a branch dispatch publishes under the branch name. Signed-off-by: Aleksander Arruda Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/publish-loreserver-image.yml | 134 +++++++++++++----- lore-server/Dockerfile | 27 ++-- 2 files changed, 107 insertions(+), 54 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index 7ec4fde9c..d13132707 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -5,16 +5,15 @@ name: Publish loreserver image # Builds lore-server/Dockerfile for linux/amd64 and linux/arm64, publishes a # multi-arch image to ghcr.io/epicgames/lore/loreserver, and signs it with -# keyless cosign so the community Helm chart can reference a public, -# verifiable tag. +# keyless cosign so the community Helm chart can reference a verifiable tag. # -# Each architecture builds on its own native runner and is pushed by digest -# alone; the merge job stitches those digests into one manifest list and signs -# that. Native runners rather than QEMU, because a release Rust build under -# emulation is far too slow to be practical. +# `meta` resolves tags and labels once, so the later jobs cannot disagree and +# a ref with no tags fails early. Each architecture then builds on its own +# native runner (QEMU is far too slow for a release Rust build) and is pushed +# by digest; `merge` stitches the digests into one manifest list and signs it. # -# No secrets are involved: the built-in GITHUB_TOKEN authenticates to GHCR, -# and cosign signs keylessly against Fulcio via the job's OIDC token. +# No secrets: GITHUB_TOKEN authenticates to GHCR, and cosign signs keylessly +# against Fulcio via the job's OIDC token. on: push: tags: ["v*"] @@ -37,8 +36,67 @@ env: IMAGE: ghcr.io/epicgames/lore/loreserver jobs: + meta: + name: resolve tags + runs-on: ubuntu-latest + permissions: + contents: read # metadata-action reads repository metadata for OCI labels + outputs: + json: ${{ steps.meta.outputs.json }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + primary-tag: ${{ steps.primary.outputs.tag }} + steps: + # The input lands verbatim in the `tags` list below, where a newline + # would smuggle in a further directive. Hold it to Docker's grammar. + - name: Validate the tag input + env: + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + case "${TAG}" in + "") exit 0 ;; + [!A-Za-z0-9_]* | *[!A-Za-z0-9._-]*) + echo "::error::the 'tag' input is not a valid image tag: expected [A-Za-z0-9_][A-Za-z0-9._-]*" + exit 1 + ;; + esac + if [ "${#TAG}" -gt 128 ]; then + echo "::error::the 'tag' input is longer than the 128 characters a tag allows" + exit 1 + fi + + - id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ${{ env.IMAGE }} + # Only a stable semver tag moves `latest`; a prerelease publishes + # its own tag and nothing else. + flavor: latest=auto + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=ref,event=branch + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} + + # An empty list has to stop the run here, not surface as an invalid + # reference in the merge job once both architectures are pushed. + - id: primary + name: Require at least one tag + env: + JSON: ${{ steps.meta.outputs.json }} + run: | + set -euo pipefail + tag=$(jq -r '.tags[0] // empty' <<< "$JSON") + if [ -z "${tag}" ]; then + echo "::error::no image tags resolved for ${GITHUB_REF}; push a semver tag or pass the 'tag' input" + exit 1 + fi + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + build: name: build (${{ matrix.arch }}) + needs: meta runs-on: ${{ matrix.runner }} permissions: contents: read # Check out the source the image is built from @@ -54,11 +112,6 @@ jobs: with: persist-credentials: false - - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ env.IMAGE }} - - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 @@ -73,8 +126,8 @@ jobs: context: . file: lore-server/Dockerfile platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - annotations: ${{ steps.meta.outputs.annotations }} + labels: ${{ needs.meta.outputs.labels }} + annotations: ${{ needs.meta.outputs.annotations }} outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true # Only the apt and toolchain layers survive a source change. The # cargo registry and target directory live in BuildKit cache mounts, @@ -110,11 +163,16 @@ jobs: merge: name: merge and sign - needs: build + needs: [meta, build] runs-on: ubuntu-latest permissions: packages: write # Push the manifest list and the cosign signature id-token: write # Federate to Fulcio for keyless signing + env: + PRIMARY_TAG: ${{ needs.meta.outputs.primary-tag }} + # Anchored on this workflow's path: a bare repository prefix would + # accept a certificate minted by any workflow here. + IDENTITY_REGEXP: '^https://github\.com/${{ github.repository }}/\.github/workflows/publish-loreserver-image\.yml@' steps: - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -122,16 +180,6 @@ jobs: pattern: digests-* merge-multiple: true - - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ env.IMAGE }} - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=${{ inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.tag != '' }} - - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 @@ -142,25 +190,32 @@ jobs: - name: Create the multi-arch manifest list working-directory: ${{ runner.temp }}/digests + env: + JSON: ${{ needs.meta.outputs.json }} run: | set -euo pipefail - # shellcheck disable=SC2046 # word splitting is how both lists are built - docker buildx imagetools create \ - $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf "${IMAGE}@sha256:%s " *) + args=() + while IFS= read -r tag; do + args+=(--tag "${tag}") + done < <(jq -r '.tags[]' <<< "$JSON") + # The build job annotated each per-arch manifest; the index is + # created here, so it needs the same annotations applying to it. + while IFS= read -r annotation; do + args+=(--annotation "index:${annotation}") + done < <(jq -r '.labels | to_entries[] | "\(.key)=\(.value)"' <<< "$JSON") + for digest in *; do + args+=("${IMAGE}@sha256:${digest}") + done + docker buildx imagetools create "${args[@]}" - name: Inspect the manifest list - run: docker buildx imagetools inspect "${IMAGE}:${VERSION}" - env: - VERSION: ${{ steps.meta.outputs.version }} + run: docker buildx imagetools inspect "${PRIMARY_TAG}" - id: digest name: Resolve the manifest list digest - env: - VERSION: ${{ steps.meta.outputs.version }} run: | set -euo pipefail - digest=$(docker buildx imagetools inspect "${IMAGE}:${VERSION}" \ + digest=$(docker buildx imagetools inspect "${PRIMARY_TAG}" \ --format '{{json .Manifest}}' | jq -r .digest) echo "digest=${digest}" >> "$GITHUB_OUTPUT" @@ -179,7 +234,7 @@ jobs: run: | set -euo pipefail cosign verify "${IMAGE}@${DIGEST}" \ - --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/" \ + --certificate-identity-regexp "${IDENTITY_REGEXP}" \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ > /dev/null echo "signature verified" @@ -187,6 +242,7 @@ jobs: - name: Summarise what was published env: DIGEST: ${{ steps.digest.outputs.digest }} + JSON: ${{ needs.meta.outputs.json }} run: | set -euo pipefail { @@ -197,13 +253,13 @@ jobs: echo "Platforms: linux/amd64, linux/arm64" echo "" echo "Tags:" - jq -r '.tags[] | "- `" + . + "`"' <<< "$DOCKER_METADATA_OUTPUT_JSON" + jq -r '.tags[] | "- `" + . + "`"' <<< "$JSON" echo "" echo "Verify the signature with:" echo "" echo '```sh' echo "cosign verify ${IMAGE}@${DIGEST} \\" - echo " --certificate-identity-regexp '^https://github.com/${GITHUB_REPOSITORY}/' \\" + echo " --certificate-identity-regexp '${IDENTITY_REGEXP}' \\" echo " --certificate-oidc-issuer https://token.actions.githubusercontent.com" echo '```' } >> "$GITHUB_STEP_SUMMARY" diff --git a/lore-server/Dockerfile b/lore-server/Dockerfile index e8482f8d5..dbfdbf66c 100644 --- a/lore-server/Dockerfile +++ b/lore-server/Dockerfile @@ -8,26 +8,23 @@ RUN apt-get update && apt-get install -y \ WORKDIR /build COPY . . -# .cargo/config.toml tunes aarch64-unknown-linux-gnu for Epic's own Graviton3+ -# deployment with -C target-cpu=neoverse-512tvb, which emits instructions that -# fault on any older arm64 part. A published image has to run anywhere armv8-a -# runs, so that tuning has to go. -# -# It has to go via RUSTFLAGS specifically. CARGO_TARGET__RUSTFLAGS is -# only another source for the same config key, and cargo *joins* config arrays -# rather than replacing them, so setting it leaves the Graviton flag in place -# and the image still dies with SIGILL. RUSTFLAGS is a separate, mutually -# exclusive source: when it is set, cargo ignores [build] and [target.*] -# rustflags entirely. So everything the build needs has to be repeated here. -# The two --cfg values are load-bearing, the code does not compile without -# them; the [build] clippy -W lints are dropped, as they only affect warnings. +# .cargo/config.toml tunes aarch64 for Graviton3+ with -C +# target-cpu=neoverse-512tvb, which SIGILLs on older arm64 parts. Dropping it +# needs RUSTFLAGS, not CARGO_TARGET__RUSTFLAGS: that one only joins the +# config arrays, RUSTFLAGS replaces [build] and [target.*] outright. Hence the +# repetition; the --cfg values are load-bearing, the clippy lints are not. ENV RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes" +# Not `--release`: that profile is Cargo.toml's local-development build, with +# debug-assertions and so overflow-checks left on. release-lto is the real one. +# It asks for full DWARF, which --strip-debug drops again, keeping the symbol +# table so backtraces still name their frames. RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/target \ - cargo build --release --bin loreserver && \ - cp /build/target/release/loreserver /build/loreserver-bin + cargo build --profile release-lto --bin loreserver && \ + cp /build/target/release-lto/loreserver /build/loreserver-bin && \ + strip --strip-debug /build/loreserver-bin FROM debian:trixie-slim From 1203df25888cf22aa5bcf84b0f8fbcc2ce0c3041 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Tue, 1 Sep 2026 11:35:01 -0300 Subject: [PATCH 05/10] Publish a Graviton-tuned arm64 image alongside the portable default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Lore is tuned for and deployed on Graviton3+, so a tuned arm64 image now ships as well — but as an opt-in `-graviton` tag rather than as the default, because `-C target-cpu=neoverse-512tvb` faults on any older arm64 part and the default tag is what a community chart pulls. X.Y.Z, X.Y, latest arm64 baseline armv8-a X.Y.Z-graviton, latest-graviton arm64 tuned for Graviton3+ The Dockerfile now assembles RUSTFLAGS itself from an ARM64_TARGET_CPU build arg, defaulting to empty. amd64 is baseline in both variants, so it is built once and both manifest lists reference that single digest — the extra arm64 leg runs in parallel and leaves the critical path where it was. OCI cannot express this: platform.variant covers v6/v7/v8, not microarchitecture, so a client cannot be steered to the right arm64 build automatically. Separate tags are the only honest mechanism. ## Also corrects the smoke test's comment The comment claimed the step catches codegen aimed at the wrong CPU. It does not. GitHub's arm64 runner is Neoverse-N2 and reports SVE, so it executes the Graviton-tuned build without complaint — measured, not assumed. The step is still worth keeping for a broken entrypoint or a missing shared library, and the comment now says only that. ## Test Plan 1. actionlint passes. 2. Dispatch from a branch and confirm two manifest lists publish, each carrying linux/amd64 and linux/arm64, each signed and verified. 3. Confirm both lists reference the same amd64 digest. 4. Run the default arm64 image on Apple Silicon, which has no SVE, and confirm loreserver starts; confirm the -graviton arm64 image does not. Signed-off-by: Aleksander Arruda --- .../workflows/publish-loreserver-image.yml | 121 +++++++++++++----- lore-server/DOCKER.md | 25 +++- lore-server/Dockerfile | 33 +++-- 3 files changed, 138 insertions(+), 41 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index d13132707..104c358e0 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -3,14 +3,23 @@ name: Publish loreserver image -# Builds lore-server/Dockerfile for linux/amd64 and linux/arm64, publishes a -# multi-arch image to ghcr.io/epicgames/lore/loreserver, and signs it with +# Builds lore-server/Dockerfile for linux/amd64 and linux/arm64, publishes +# multi-arch images to ghcr.io/epicgames/lore/loreserver, and signs them with # keyless cosign so the community Helm chart can reference a verifiable tag. # +# Two variants ship, differing only in how arm64 is compiled: +# +# :X.Y.Z baseline armv8-a arm64 — runs on any arm64 host +# :X.Y.Z-graviton arm64 tuned for Graviton3+, as Lore is deployed +# +# The default is the portable one, so a community chart works everywhere; the +# tuned build is opt-in for Graviton deployments. amd64 is baseline in both and +# is therefore built once, with both manifest lists pointing at that one digest. +# # `meta` resolves tags and labels once, so the later jobs cannot disagree and -# a ref with no tags fails early. Each architecture then builds on its own -# native runner (QEMU is far too slow for a release Rust build) and is pushed -# by digest; `merge` stitches the digests into one manifest list and signs it. +# a ref with no tags fails early. Each variant then builds on its own native +# runner (QEMU is far too slow for a release Rust build) and is pushed by +# digest; `merge` stitches the digests into manifest lists and signs them. # # No secrets: GITHUB_TOKEN authenticates to GHCR, and cosign signs keylessly # against Fulcio via the job's OIDC token. @@ -34,6 +43,8 @@ concurrency: env: IMAGE: ghcr.io/epicgames/lore/loreserver + # Passed to the Dockerfile for the tuned arm64 leg only. + GRAVITON_TARGET_CPU: neoverse-512tvb jobs: meta: @@ -43,9 +54,11 @@ jobs: contents: read # metadata-action reads repository metadata for OCI labels outputs: json: ${{ steps.meta.outputs.json }} + graviton-json: ${{ steps.meta-graviton.outputs.json }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} primary-tag: ${{ steps.primary.outputs.tag }} + graviton-primary-tag: ${{ steps.primary.outputs.graviton-tag }} steps: # The input lands verbatim in the `tags` list below, where a newline # would smuggle in a further directive. Hold it to Docker's grammar. @@ -79,23 +92,42 @@ jobs: type=ref,event=branch type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} + # The same tag set, suffixed. onlatest carries the suffix onto `latest` + # so the tuned stream has its own moving tag rather than fighting for the + # shared one. + - id: meta-graviton + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ${{ env.IMAGE }} + flavor: | + latest=auto + suffix=-graviton,onlatest=true + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=ref,event=branch + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} + # An empty list has to stop the run here, not surface as an invalid - # reference in the merge job once both architectures are pushed. + # reference in the merge job once every architecture is pushed. - id: primary name: Require at least one tag env: JSON: ${{ steps.meta.outputs.json }} + GRAVITON_JSON: ${{ steps.meta-graviton.outputs.json }} run: | set -euo pipefail tag=$(jq -r '.tags[0] // empty' <<< "$JSON") - if [ -z "${tag}" ]; then + graviton_tag=$(jq -r '.tags[0] // empty' <<< "$GRAVITON_JSON") + if [ -z "${tag}" ] || [ -z "${graviton_tag}" ]; then echo "::error::no image tags resolved for ${GITHUB_REF}; push a semver tag or pass the 'tag' input" exit 1 fi echo "tag=${tag}" >> "$GITHUB_OUTPUT" + echo "graviton-tag=${graviton_tag}" >> "$GITHUB_OUTPUT" build: - name: build (${{ matrix.arch }}) + name: build (${{ matrix.arch }}${{ matrix.variant == 'graviton' && ', graviton' || '' }}) needs: meta runs-on: ${{ matrix.runner }} permissions: @@ -105,8 +137,11 @@ jobs: fail-fast: false matrix: include: - - { arch: amd64, platform: linux/amd64, runner: ubuntu-latest } - - { arch: arm64, platform: linux/arm64, runner: ubuntu-24.04-arm } + # amd64 is baseline for both variants, so it is built once and both + # manifest lists reference this digest. + - { arch: amd64, variant: base, platform: linux/amd64, runner: ubuntu-latest, target_cpu: "" } + - { arch: arm64, variant: base, platform: linux/arm64, runner: ubuntu-24.04-arm, target_cpu: "" } + - { arch: arm64, variant: graviton, platform: linux/arm64, runner: ubuntu-24.04-arm, target_cpu: neoverse-512tvb } steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -126,27 +161,32 @@ jobs: context: . file: lore-server/Dockerfile platforms: ${{ matrix.platform }} + build-args: ARM64_TARGET_CPU=${{ matrix.target_cpu }} labels: ${{ needs.meta.outputs.labels }} annotations: ${{ needs.meta.outputs.annotations }} outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true # Only the apt and toolchain layers survive a source change. The # cargo registry and target directory live in BuildKit cache mounts, # which the gha backend does not carry between runs, so the compile - # itself starts cold every time. - cache-from: type=gha,scope=loreserver-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=loreserver-${{ matrix.arch }} + # itself starts cold every time. Scoped per variant so the two arm64 + # legs, built with different codegen flags, cannot share layers. + cache-from: type=gha,scope=loreserver-${{ matrix.arch }}-${{ matrix.variant }} + cache-to: type=gha,mode=max,scope=loreserver-${{ matrix.arch }}-${{ matrix.variant }} - # A clean build proves only that the image compiles. Run the binary on - # the native runner as well: a codegen flag aimed at the wrong CPU - # produces an image that builds happily and then dies with SIGILL the - # first time anyone starts it. + # A clean build proves only that the image compiles, so start the binary + # too. Know the limit: this arm64 runner is Neoverse-N2 and reports SVE, + # so it executes the Graviton-tuned build quite happily. This catches a + # broken entrypoint or a missing shared library — it does NOT catch + # codegen aimed at a CPU the eventual host lacks, which is exactly how a + # SIGILL-ing arm64 image once passed a fully green run. - name: Smoke test the pushed image env: DIGEST: ${{ steps.build.outputs.digest }} run: docker run --rm "${IMAGE}@${DIGEST}" --version - # The merge job addresses each architecture by digest. Hand the digests - # over as empty files named after themselves. + # The merge job addresses each build by digest. Hand the digests over as + # empty files named after themselves, one artifact per matrix leg so the + # merge job can pick which legs belong in which manifest list. - name: Export digest env: DIGEST: ${{ steps.build.outputs.digest }} @@ -156,29 +196,42 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: digests-${{ matrix.arch }} + name: digests-${{ matrix.arch }}-${{ matrix.variant }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 merge: - name: merge and sign + name: merge and sign (${{ matrix.variant }}) needs: [meta, build] runs-on: ubuntu-latest permissions: packages: write # Push the manifest list and the cosign signature id-token: write # Federate to Fulcio for keyless signing + strategy: + # One variant failing should not withhold the other; both are + # independently useful once published. + fail-fast: false + matrix: + include: + - variant: default + legs: digests-amd64-base digests-arm64-base + - variant: graviton + legs: digests-amd64-base digests-arm64-graviton env: - PRIMARY_TAG: ${{ needs.meta.outputs.primary-tag }} + JSON: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-json || needs.meta.outputs.json }} + PRIMARY_TAG: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-primary-tag || needs.meta.outputs.primary-tag }} + LEGS: ${{ matrix.legs }} # Anchored on this workflow's path: a bare repository prefix would # accept a certificate minted by any workflow here. IDENTITY_REGEXP: '^https://github\.com/${{ github.repository }}/\.github/workflows/publish-loreserver-image\.yml@' steps: + # No merge-multiple: each leg keeps its own subdirectory so the right + # subset can be selected below. - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ${{ runner.temp }}/digests pattern: digests-* - merge-multiple: true - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 @@ -190,8 +243,6 @@ jobs: - name: Create the multi-arch manifest list working-directory: ${{ runner.temp }}/digests - env: - JSON: ${{ needs.meta.outputs.json }} run: | set -euo pipefail args=() @@ -203,8 +254,14 @@ jobs: while IFS= read -r annotation; do args+=(--annotation "index:${annotation}") done < <(jq -r '.labels | to_entries[] | "\(.key)=\(.value)"' <<< "$JSON") - for digest in *; do - args+=("${IMAGE}@sha256:${digest}") + for leg in ${LEGS}; do + if [ ! -d "${leg}" ]; then + echo "::error::${leg} did not produce a digest; refusing to publish a partial manifest list" + exit 1 + fi + for digest in "${leg}"/*; do + args+=("${IMAGE}@sha256:$(basename "${digest}")") + done done docker buildx imagetools create "${args[@]}" @@ -242,15 +299,19 @@ jobs: - name: Summarise what was published env: DIGEST: ${{ steps.digest.outputs.digest }} - JSON: ${{ needs.meta.outputs.json }} + VARIANT: ${{ matrix.variant }} run: | set -euo pipefail { - echo "## Published \`${IMAGE}\`" + echo "## Published \`${IMAGE}\` (${VARIANT})" echo "" echo "Digest: \`${DIGEST}\`" echo "" - echo "Platforms: linux/amd64, linux/arm64" + if [ "${VARIANT}" = "graviton" ]; then + echo "Platforms: linux/amd64 (baseline), linux/arm64 (tuned for Graviton3+ — will not run on older arm64)" + else + echo "Platforms: linux/amd64, linux/arm64 (baseline armv8-a)" + fi echo "" echo "Tags:" jq -r '.tags[] | "- `" + . + "`"' <<< "$JSON" diff --git a/lore-server/DOCKER.md b/lore-server/DOCKER.md index 28070c21d..482fcac08 100644 --- a/lore-server/DOCKER.md +++ b/lore-server/DOCKER.md @@ -7,9 +7,10 @@ telemetry integration, or replication is configured. - Docker with BuildKit support -Both `linux/amd64` and `linux/arm64` build. The Dockerfile overrides the Graviton-specific -`target-cpu` that `.cargo/config.toml` sets for `aarch64-unknown-linux-gnu`, so the arm64 -image runs on any armv8-a host rather than Graviton3 and newer only. +Both `linux/amd64` and `linux/arm64` build. `.cargo/config.toml` pins `aarch64-unknown-linux-gnu` +to Graviton3+ via `-C target-cpu=neoverse-512tvb`, which faults on older arm64 parts, so the +Dockerfile assembles `RUSTFLAGS` itself and leaves that tuning off by default. The arm64 image +therefore runs on any armv8-a host, Apple Silicon included. ## Building @@ -22,6 +23,24 @@ docker build -f lore-server/Dockerfile -t loreserver . Pass `--platform linux/amd64` or `--platform linux/arm64` to cross-build; expect it to be slow, since a release Rust build under emulation is far slower than a native one. +To tune arm64 for Graviton3 and newer, as Lore is deployed, pass the microarchitecture. The +resulting binary will not run on older arm64 hardware: + +```sh +docker build -f lore-server/Dockerfile --build-arg ARM64_TARGET_CPU=neoverse-512tvb -t loreserver . +``` + +## Published images + +The publish workflow ships both variants to `ghcr.io/epicgames/lore/loreserver`: + +| Tag | arm64 build | +| --- | --- | +| `X.Y.Z`, `X.Y`, `latest` | baseline `armv8-a` — runs on any arm64 host | +| `X.Y.Z-graviton`, `latest-graviton` | tuned for Graviton3+ — faults on older arm64 | + +`linux/amd64` is baseline in both, and is the same image in each manifest list. + The build compiles the `loreserver` binary and generates self-signed TLS certificates for QUIC using `scripts/server/make-certs.sh`. diff --git a/lore-server/Dockerfile b/lore-server/Dockerfile index dbfdbf66c..17e938fba 100644 --- a/lore-server/Dockerfile +++ b/lore-server/Dockerfile @@ -8,13 +8,23 @@ RUN apt-get update && apt-get install -y \ WORKDIR /build COPY . . -# .cargo/config.toml tunes aarch64 for Graviton3+ with -C -# target-cpu=neoverse-512tvb, which SIGILLs on older arm64 parts. Dropping it -# needs RUSTFLAGS, not CARGO_TARGET__RUSTFLAGS: that one only joins the -# config arrays, RUSTFLAGS replaces [build] and [target.*] outright. Hence the -# repetition; the --cfg values are load-bearing, the clippy lints are not. -ENV RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes" +# Which arm64 microarchitecture to tune for. Empty is baseline armv8-a, which +# runs anywhere — Apple Silicon, Ampere, any non-Graviton host — and is what +# the default image ships. The publish workflow passes neoverse-512tvb for the +# separate -graviton tag, matching how Lore is built and deployed on +# Graviton3+. Ignored on amd64, which is baseline either way. +ARG TARGETARCH +ARG ARM64_TARGET_CPU="" +# The flags are assembled here rather than left to .cargo/config.toml, whose +# [target.aarch64-unknown-linux-gnu] table pins Graviton3+ unconditionally. +# Overriding that needs RUSTFLAGS specifically, not +# CARGO_TARGET__RUSTFLAGS: the latter is only another source for the +# same config key and cargo *joins* config arrays, so the config's +# -C target-cpu would survive and a baseline build would still SIGILL off +# Graviton. RUSTFLAGS replaces [build] and [target.*] outright, hence the +# repetition. The --cfg values are load-bearing, the clippy lints are not. +# # Not `--release`: that profile is Cargo.toml's local-development build, with # debug-assertions and so overflow-checks left on. release-lto is the real one. # It asks for full DWARF, which --strip-debug drops again, keeping the symbol @@ -22,8 +32,15 @@ ENV RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=y RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/target \ - cargo build --profile release-lto --bin loreserver && \ - cp /build/target/release-lto/loreserver /build/loreserver-bin && \ + set -eu; \ + RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes"; \ + if [ "${TARGETARCH}" = "arm64" ] && [ -n "${ARM64_TARGET_CPU}" ]; then \ + RUSTFLAGS="${RUSTFLAGS} -C target-cpu=${ARM64_TARGET_CPU}"; \ + fi; \ + export RUSTFLAGS; \ + echo "building with RUSTFLAGS=${RUSTFLAGS}"; \ + cargo build --profile release-lto --bin loreserver; \ + cp /build/target/release-lto/loreserver /build/loreserver-bin; \ strip --strip-debug /build/loreserver-bin FROM debian:trixie-slim From 008bcf7c8750b2e71585e5d104afd3cb261527a0 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Tue, 1 Sep 2026 11:56:08 -0300 Subject: [PATCH 06/10] Stop the fat-LTO link getting OOM-killed release-lto sets debug = 2, and a fat-LTO link holds the whole dependency graph's DWARF in memory at once. On a 16 GB runner rustc is SIGKILLed part way through linking loreserver, so both the amd64 and the Graviton arm64 builds died. Build with -C debuginfo=0. strip --strip-debug discarded that DWARF a line later regardless, and tracing's file and line fields come from compile-time macros rather than debug info, so nothing observable is lost. Signed-off-by: Aleksander Arruda --- lore-server/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lore-server/Dockerfile b/lore-server/Dockerfile index 17e938fba..4774d9a24 100644 --- a/lore-server/Dockerfile +++ b/lore-server/Dockerfile @@ -27,13 +27,18 @@ ARG ARM64_TARGET_CPU="" # # Not `--release`: that profile is Cargo.toml's local-development build, with # debug-assertions and so overflow-checks left on. release-lto is the real one. -# It asks for full DWARF, which --strip-debug drops again, keeping the symbol -# table so backtraces still name their frames. +# +# release-lto also asks for `debug = 2`, which a fat-LTO link cannot afford: it +# holds the whole dependency graph's DWARF at once and gets OOM-killed on a +# 16 GB runner. `strip --strip-debug` below would have discarded all of it in +# the next breath, so -C debuginfo=0 costs nothing observable — tracing's +# file/line fields come from compile-time macros, not from DWARF. RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/target \ set -eu; \ RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes"; \ + RUSTFLAGS="${RUSTFLAGS} -C debuginfo=0"; \ if [ "${TARGETARCH}" = "arm64" ] && [ -n "${ARM64_TARGET_CPU}" ]; then \ RUSTFLAGS="${RUSTFLAGS} -C target-cpu=${ARM64_TARGET_CPU}"; \ fi; \ From 9c474d2a1f02ddb4d550ccc7f7b94adca1b39607 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Wed, 2 Sep 2026 10:36:09 -0300 Subject: [PATCH 07/10] Publish each variant independently, and sign before tagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Four ways this workflow could have shipped the wrong thing without saying so. ### A failed graviton leg no longer withholds the default image `merge` declared `needs: [meta, build]`, which waits on the whole matrix *and* requires all of it to pass. So the opt-in graviton leg was a hard dependency of the portable default image — the one a community chart pulls — exactly inverting what this workflow is for. It also made the partial-manifest guard unreachable, since `merge` could never run with a leg missing. The smoke test makes this concrete rather than theoretical: it passes only because GitHub's arm64 pool is Neoverse-N2 and reports SVE. Rotate that pool to an N1 part and the graviton leg SIGILLs, taking the default release with it. `if: !cancelled() && needs.meta.result == 'success'` depends on the legs having finished rather than on all of them having passed. `meta` stays a hard requirement — a variant with no resolved tags has nothing to publish — and the leg-existence guard now decides which variant can actually ship. ### No release tag points at an unsigned digest `imagetools create` moved every tag, `latest` included, and `cosign sign` ran seven steps later. A Fulcio or OIDC failure in between left `latest` published and unverifiable, which undercuts the verifiable tag this workflow exists to produce. The index is now created under a `sha-` staging tag alone, signed, verified, and only then promoted onto the release tags. The staging tag never moves and nothing consumes it, so it is the one thing exposed to that window. Promotion depends on `imagetools create` copying a lone index source through byte for byte. That is proven first, by copying the index onto the staging tag it already occupies: a no-op while the digest is preserved, and if some later buildx stops preserving it, the tag that moves out from under the signature is the staging tag rather than `latest`. Each release tag is re-resolved afterwards as well. `primary-tag` and `graviton-primary-tag` go with it — the staging tag addresses the index now, so those outputs had no remaining reader. The gate step they came from stays, since an empty tag list still has to fail here. ### The -graviton image says so in its own labels The graviton leg was handed the default variant's labels, so its arm64 manifest reported `image.version=X.Y.Z` while the index annotation said `X.Y.Z-graviton`. `docker inspect` on a pulled image could not tell the two arm64 builds apart, which is the first thing anyone debugging a SIGILL on older arm64 hardware would reach for. amd64 is shared between both manifest lists and can only carry one set, which remains the default's. ### One place to retune Graviton `GRAVITON_TARGET_CPU` was never read; the matrix hard-coded the literal instead, so editing the env var changed nothing about what was built. The build arg now derives from that env var via the variant, and the matrix column is gone. `env` is unavailable in `strategy.matrix`, hence resolving it at the step. An untuned build is a perfectly successful one, so a graviton leg that resolved no target CPU would have published a baseline image under the `-graviton` tag with every step green. It now fails before building. ## Test Plan 1. actionlint passes. 2. Dispatch from a branch; confirm both manifest lists publish, each signed and verified, and that every release tag resolves to the signed digest. 3. Confirm the `sha-` tag exists on that same digest, and that the signature verifies against it. 4. Confirm `docker inspect` on the arm64 `-graviton` image reports a `-graviton` `org.opencontainers.image.version`, and the default does not. 5. Force the graviton build leg to fail; confirm the default variant still publishes and signs, and that the graviton merge fails on the leg guard rather than being skipped. 6. Confirm the build log for the graviton leg prints `tuning arm64 for neoverse-512tvb`, and that the baseline legs skip that step. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Aleksander Arruda --- .../workflows/publish-loreserver-image.yml | 139 ++++++++++++++---- 1 file changed, 113 insertions(+), 26 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index 104c358e0..e4ac4060d 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -19,7 +19,10 @@ name: Publish loreserver image # `meta` resolves tags and labels once, so the later jobs cannot disagree and # a ref with no tags fails early. Each variant then builds on its own native # runner (QEMU is far too slow for a release Rust build) and is pushed by -# digest; `merge` stitches the digests into manifest lists and signs them. +# digest; `merge` stitches the digests into a manifest list, signs and verifies +# it under a staging tag, and only then attaches the release tags to the digest +# it just proved. So no release tag is ever left pointing at an unsigned image, +# even if Fulcio is having a bad day. # # No secrets: GITHUB_TOKEN authenticates to GHCR, and cosign signs keylessly # against Fulcio via the job's OIDC token. @@ -57,8 +60,8 @@ jobs: graviton-json: ${{ steps.meta-graviton.outputs.json }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} - primary-tag: ${{ steps.primary.outputs.tag }} - graviton-primary-tag: ${{ steps.primary.outputs.graviton-tag }} + graviton-labels: ${{ steps.meta-graviton.outputs.labels }} + graviton-annotations: ${{ steps.meta-graviton.outputs.annotations }} steps: # The input lands verbatim in the `tags` list below, where a newline # would smuggle in a further directive. Hold it to Docker's grammar. @@ -110,21 +113,18 @@ jobs: # An empty list has to stop the run here, not surface as an invalid # reference in the merge job once every architecture is pushed. - - id: primary - name: Require at least one tag + - name: Require at least one tag env: JSON: ${{ steps.meta.outputs.json }} GRAVITON_JSON: ${{ steps.meta-graviton.outputs.json }} run: | set -euo pipefail - tag=$(jq -r '.tags[0] // empty' <<< "$JSON") - graviton_tag=$(jq -r '.tags[0] // empty' <<< "$GRAVITON_JSON") - if [ -z "${tag}" ] || [ -z "${graviton_tag}" ]; then - echo "::error::no image tags resolved for ${GITHUB_REF}; push a semver tag or pass the 'tag' input" - exit 1 - fi - echo "tag=${tag}" >> "$GITHUB_OUTPUT" - echo "graviton-tag=${graviton_tag}" >> "$GITHUB_OUTPUT" + for json in "$JSON" "$GRAVITON_JSON"; do + if [ -z "$(jq -r '.tags[0] // empty' <<< "${json}")" ]; then + echo "::error::no image tags resolved for ${GITHUB_REF}; push a semver tag or pass the 'tag' input" + exit 1 + fi + done build: name: build (${{ matrix.arch }}${{ matrix.variant == 'graviton' && ', graviton' || '' }}) @@ -139,9 +139,16 @@ jobs: include: # amd64 is baseline for both variants, so it is built once and both # manifest lists reference this digest. - - { arch: amd64, variant: base, platform: linux/amd64, runner: ubuntu-latest, target_cpu: "" } - - { arch: arm64, variant: base, platform: linux/arm64, runner: ubuntu-24.04-arm, target_cpu: "" } - - { arch: arm64, variant: graviton, platform: linux/arm64, runner: ubuntu-24.04-arm, target_cpu: neoverse-512tvb } + - { arch: amd64, variant: base, platform: linux/amd64, runner: ubuntu-latest } + - { arch: arm64, variant: base, platform: linux/arm64, runner: ubuntu-24.04-arm } + - { arch: arm64, variant: graviton, platform: linux/arm64, runner: ubuntu-24.04-arm } + env: + # The graviton leg carries the suffixed tag's metadata, so that `docker + # inspect` on a pulled image distinguishes the two arm64 variants. The + # amd64 leg is shared between both manifest lists and can only carry one + # set, which is the default's. + LABELS: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-labels || needs.meta.outputs.labels }} + ANNOTATIONS: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-annotations || needs.meta.outputs.annotations }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -155,15 +162,36 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # The build argument below resolves from an expression, and an untuned + # build is a perfectly successful one — so were that expression ever to + # come back empty, the -graviton tag would quietly ship a baseline image + # and every later step would still be green. Assert the same expression + # here, where it can only cost a fast failure. + - name: Check the graviton leg is tuned + if: ${{ matrix.variant == 'graviton' }} + env: + TARGET_CPU: ${{ matrix.variant == 'graviton' && env.GRAVITON_TARGET_CPU || '' }} + run: | + if [ -z "${TARGET_CPU}" ]; then + echo "::error::no arm64 target CPU resolved for the graviton leg; it would build baseline armv8-a and publish it as tuned" + exit 1 + fi + echo "tuning arm64 for ${TARGET_CPU}" + - id: build uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . file: lore-server/Dockerfile platforms: ${{ matrix.platform }} - build-args: ARM64_TARGET_CPU=${{ matrix.target_cpu }} - labels: ${{ needs.meta.outputs.labels }} - annotations: ${{ needs.meta.outputs.annotations }} + # Read from the workflow env rather than a matrix column, so retuning + # Graviton is one edit that cannot silently miss the build. `env` is + # unavailable in `strategy.matrix`, hence deriving it here from the + # variant instead. Empty is baseline; the Dockerfile ignores it on + # amd64 either way. + build-args: ARM64_TARGET_CPU=${{ matrix.variant == 'graviton' && env.GRAVITON_TARGET_CPU || '' }} + labels: ${{ env.LABELS }} + annotations: ${{ env.ANNOTATIONS }} outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true # Only the apt and toolchain layers survive a source change. The # cargo registry and target directory live in BuildKit cache mounts, @@ -204,6 +232,13 @@ jobs: merge: name: merge and sign (${{ matrix.variant }}) needs: [meta, build] + # `needs: build` alone waits on the whole matrix *and* requires all of it to + # succeed, which would let the opt-in graviton leg withhold the portable + # default image. Depend on the legs having finished, not on all of them + # having passed; `meta` is still a hard requirement, since a variant with no + # resolved tags has nothing to publish. Which legs a given variant actually + # needs is then the leg-existence check's call, further down. + if: ${{ !cancelled() && needs.meta.result == 'success' }} runs-on: ubuntu-latest permissions: packages: write # Push the manifest list and the cosign signature @@ -220,8 +255,11 @@ jobs: legs: digests-amd64-base digests-arm64-graviton env: JSON: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-json || needs.meta.outputs.json }} - PRIMARY_TAG: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-primary-tag || needs.meta.outputs.primary-tag }} LEGS: ${{ matrix.legs }} + # Where the index lands before it is signed. Commit-addressed, so it never + # moves and nothing has to be pointed away from it later; kept afterwards + # as a record of which commit produced which digest. + STAGING_TAG: sha-${{ github.sha }}${{ matrix.variant == 'graviton' && '-graviton' || '' }} # Anchored on this workflow's path: a bare repository prefix would # accept a certificate minted by any workflow here. IDENTITY_REGEXP: '^https://github\.com/${{ github.repository }}/\.github/workflows/publish-loreserver-image\.yml@' @@ -241,16 +279,20 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # Publishing happens in two phases. Here the index is created under the + # staging tag alone, so a failure between this step and a verified + # signature cannot leave a release tag — `latest` above all — pointing at + # an unsigned digest. The release tags are attached further down, once a + # signature exists and has verified. - name: Create the multi-arch manifest list working-directory: ${{ runner.temp }}/digests run: | set -euo pipefail - args=() - while IFS= read -r tag; do - args+=(--tag "${tag}") - done < <(jq -r '.tags[]' <<< "$JSON") + args=(--tag "${IMAGE}:${STAGING_TAG}") # The build job annotated each per-arch manifest; the index is # created here, so it needs the same annotations applying to it. + # They have to land now: annotations are part of what the digest + # covers, and the promotion below must not alter the digest. while IFS= read -r annotation; do args+=(--annotation "index:${annotation}") done < <(jq -r '.labels | to_entries[] | "\(.key)=\(.value)"' <<< "$JSON") @@ -266,13 +308,13 @@ jobs: docker buildx imagetools create "${args[@]}" - name: Inspect the manifest list - run: docker buildx imagetools inspect "${PRIMARY_TAG}" + run: docker buildx imagetools inspect "${IMAGE}:${STAGING_TAG}" - id: digest name: Resolve the manifest list digest run: | set -euo pipefail - digest=$(docker buildx imagetools inspect "${PRIMARY_TAG}" \ + digest=$(docker buildx imagetools inspect "${IMAGE}:${STAGING_TAG}" \ --format '{{json .Manifest}}' | jq -r .digest) echo "digest=${digest}" >> "$GITHUB_OUTPUT" @@ -296,6 +338,50 @@ jobs: > /dev/null echo "signature verified" + # Promotion below rests on `imagetools create` copying a lone index source + # through byte for byte, which is what makes the release tags resolve to + # the digest just signed. Prove that on the staging tag first, by copying + # the index onto the tag it already occupies: a no-op if the digest is + # preserved, and if some later buildx stops preserving it, the tag that + # moves out from under the signature is this one, which nothing consumes. + # Checking afterwards instead would mean finding out from `latest`. + - name: Prove the copy preserves the digest + env: + DIGEST: ${{ steps.digest.outputs.digest }} + run: | + set -euo pipefail + docker buildx imagetools create --tag "${IMAGE}:${STAGING_TAG}" "${IMAGE}@${DIGEST}" + resolved=$(docker buildx imagetools inspect "${IMAGE}:${STAGING_TAG}" \ + --format '{{json .Manifest}}' | jq -r .digest) + if [ "${resolved}" != "${DIGEST}" ]; then + echo "::error::copying the index changed its digest (${DIGEST} -> ${resolved}); refusing to move the release tags off the signed digest" + exit 1 + fi + echo "copy is digest-preserving" + + # Phase two: the release tags, now that a signature exists over the digest + # and verifies. Each tag is re-resolved afterwards, so a tag pointing at + # anything other than the signed digest fails the run rather than shipping. + - name: Promote the signed digest to the release tags + env: + DIGEST: ${{ steps.digest.outputs.digest }} + run: | + set -euo pipefail + args=() + while IFS= read -r tag; do + args+=(--tag "${tag}") + done < <(jq -r '.tags[]' <<< "$JSON") + docker buildx imagetools create "${args[@]}" "${IMAGE}@${DIGEST}" + while IFS= read -r tag; do + resolved=$(docker buildx imagetools inspect "${tag}" \ + --format '{{json .Manifest}}' | jq -r .digest) + if [ "${resolved}" != "${DIGEST}" ]; then + echo "::error::${tag} resolved to ${resolved}, not the signed ${DIGEST}" + exit 1 + fi + echo "${tag} -> ${resolved}" + done < <(jq -r '.tags[]' <<< "$JSON") + - name: Summarise what was published env: DIGEST: ${{ steps.digest.outputs.digest }} @@ -315,6 +401,7 @@ jobs: echo "" echo "Tags:" jq -r '.tags[] | "- `" + . + "`"' <<< "$JSON" + echo "- \`${IMAGE}:${STAGING_TAG}\` (staging tag the signature was made against)" echo "" echo "Verify the signature with:" echo "" From b6b2d48ce8cefe75a1f3e1ec3a7108ddbc0ec371 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Wed, 2 Sep 2026 10:36:25 -0300 Subject: [PATCH 08/10] Correct the docs the arm64 baseline default left stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Making the arm64 image portable by default contradicted three documents that were not updated with it. `docs/how-to/deploy-local-lore-server.md` is the primary Run with Docker how-to, and it still told Apple Silicon and Windows readers to build and run with `--platform linux/amd64` because the arm64 image targets Graviton3 SVE — the restriction the baseline default removed, and the opposite of what DOCKER.md now says. Following it meant a release Rust build under QEMU and an emulated runtime to work around nothing. The path-chooser bullet carried the same caveat. `contrib/aws/README.md` deploys c8gd.8xlarge Graviton instances and used to inherit `-C target-cpu=neoverse-512tvb` from `.cargo/config.toml`. The Dockerfile assembles RUSTFLAGS itself now, so that command silently shipped an untuned baseline binary to the one place tuning was meant for. It passes `--build-arg ARM64_TARGET_CPU=neoverse-512tvb`. ## Also in DOCKER.md The new Published images heading was inserted above an existing paragraph, which left "generates self-signed TLS certificates for QUIC using scripts/server/make-certs.sh" reading as a claim about the published GHCR images. It was wrong twice over: that script has never existed in this repository, and the Dockerfile generates no certificates. The server generates an ephemeral self-signed one at startup, which is now stated under Running, where someone whose client rejects the certificate will look for it. The `docker.toml` bullet claimed to configure QUIC certificates for the same reason and had the same problem; the file it describes sets two store paths. The tag table was missing `X.Y-graviton`, which `meta-graviton` emits from the same `{{major}}.{{minor}}` pattern the default row already documents, so pinning a minor series on Graviton was undiscoverable. It also now describes the `sha-` tag, which the two-phase signing change publishes alongside each release and which would otherwise look like stray litter in the package list. ## Test Plan 1. docs-lint.sh passes with vale, markdownlint and lychee installed. 2. Build the image on Apple Silicon following the how-to verbatim, with no `--platform` flag, and confirm loreserver starts. 3. Build with the contrib/aws command and confirm the log line reports `-C target-cpu=neoverse-512tvb`. 4. Confirm no document still references `scripts/server/make-certs.sh` as a step in building the Docker image. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Aleksander Arruda --- contrib/aws/README.md | 8 +++++++- docs/how-to/deploy-local-lore-server.md | 6 +++--- lore-server/DOCKER.md | 15 +++++++++++---- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/contrib/aws/README.md b/contrib/aws/README.md index 9490bbd5e..7b9214854 100644 --- a/contrib/aws/README.md +++ b/contrib/aws/README.md @@ -13,9 +13,15 @@ This example uses **c8gd.8xlarge** Graviton instances (32 vCPU, 64 GB RAM, 1.9 T From the Lore repo root: ```sh -docker buildx build --platform linux/arm64 -f lore-server/Dockerfile -t loreserver:v0.8.7 --load . +docker buildx build --platform linux/arm64 \ + --build-arg ARM64_TARGET_CPU=neoverse-512tvb \ + -f lore-server/Dockerfile -t loreserver:v0.8.7 --load . ``` +> `ARM64_TARGET_CPU` tunes codegen for Graviton3+, matching the `c8gd` instances below. Without it +> the build is baseline `armv8-a`, which runs here but leaves performance on the table. The +> resulting binary uses SVE and will not run on older arm64 hardware. + > If building on an x86 host, [register QEMU](https://docs.docker.com/build/building/multi-platform/#qemu) first: > `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes` diff --git a/docs/how-to/deploy-local-lore-server.md b/docs/how-to/deploy-local-lore-server.md index a389639a2..91d5ecdc9 100644 --- a/docs/how-to/deploy-local-lore-server.md +++ b/docs/how-to/deploy-local-lore-server.md @@ -15,7 +15,7 @@ In this guide, you'll deploy local Lore Servers — with durable storage and a c The binary and Docker paths are mutually exclusive, and each is complete on its own — follow one top to bottom. - **[Run from the binary](#run-from-the-binary):** Fewer moving parts and native performance. Pick this to run `loreserver` directly on the host. -- **[Run with Docker](#run-with-docker):** An isolated container. Pick this if you'd rather not put a binary on the host — but note the `linux/amd64` emulation caveat on Apple Silicon in the build step. +- **[Run with Docker](#run-with-docker):** An isolated container. Pick this if you'd rather not put a binary on the host. Builds natively on both `amd64` and `arm64`, Apple Silicon included. ## Run from the binary @@ -196,11 +196,11 @@ The binary and Docker paths are mutually exclusive, and each is complete on its This needs Docker (and WSL2 on Windows) and the Lore repository cloned locally. Building the image compiles the server, so it needs several GB of free RAM. From the repository root: ```bash - docker build --platform linux/amd64 -f lore-server/Dockerfile -t lore-server . + docker build -f lore-server/Dockerfile -t lore-server . ``` > [!NOTE] - > On Apple Silicon or Windows (both arm64 and amd64), build and run with `--platform linux/amd64` as shown. The `linux/arm64` server image targets AWS Graviton3 (SVE), an instruction set those CPUs lack. + > The image builds for your host architecture. On Apple Silicon or Windows on arm64 that is a baseline `armv8-a` build which runs natively — no `--platform` override is needed. To tune arm64 for AWS Graviton3 and newer instead, add `--build-arg ARM64_TARGET_CPU=neoverse-512tvb`; that binary uses SVE and will not run on other arm64 hardware. 2. **Run it with default settings.** diff --git a/lore-server/DOCKER.md b/lore-server/DOCKER.md index 482fcac08..2c07040da 100644 --- a/lore-server/DOCKER.md +++ b/lore-server/DOCKER.md @@ -37,12 +37,15 @@ The publish workflow ships both variants to `ghcr.io/epicgames/lore/loreserver`: | Tag | arm64 build | | --- | --- | | `X.Y.Z`, `X.Y`, `latest` | baseline `armv8-a` — runs on any arm64 host | -| `X.Y.Z-graviton`, `latest-graviton` | tuned for Graviton3+ — faults on older arm64 | +| `X.Y.Z-graviton`, `X.Y-graviton`, `latest-graviton` | tuned for Graviton3+ — faults on older arm64 | `linux/amd64` is baseline in both, and is the same image in each manifest list. -The build compiles the `loreserver` binary and generates self-signed TLS certificates for QUIC -using `scripts/server/make-certs.sh`. +Every tag is signed keylessly with cosign, and the build summary for a release prints the +`cosign verify` invocation for the digest it published. A `sha-` tag appears alongside each +release, on the same digest: the signature is made against it before any release tag is pointed at +that digest, so no release tag is ever briefly unsigned. It stays afterwards as a record of which +commit built which image. ## Running @@ -52,6 +55,10 @@ docker run -p 41337:41337/tcp -p 41337:41337/udp -p 41339:41339 loreserver Both TCP and UDP mappings are required on port 41337 because gRPC uses TCP and QUIC uses UDP. +No QUIC certificate is baked into the image, so the server generates an ephemeral self-signed one +at startup and clients have to be told to trust it. For anything durable, mount a real certificate +and point `[server.quic.certificate]` at it. + ### Persisting data By default, store data is written to `/data` inside the container and is lost when the container @@ -79,7 +86,7 @@ docker run \ The image stores config files in `/etc/lore/config/` (`LORE_CONFIG_PATH`): - `default.toml` — copied from `lore-server/config/default.toml` at image build time. Loaded as the on-disk default layer on top of the compiled-in defaults, so you can mount a custom `default.toml` to override compiled-in values without rebuilding the image. -- `docker.toml` — overrides store paths to `/data` and configures QUIC TLS certificates. Loaded as the `docker` environment layer (`LORE_ENV=docker`). +- `docker.toml` — overrides the immutable and mutable store paths to `/data`. Loaded as the `docker` environment layer (`LORE_ENV=docker`). It configures no QUIC certificate, which is what leaves the server generating an ephemeral self-signed one. Settings can be overridden via environment variables with the `LORE__` prefix and `__` as the separator. For example: From 65c336fbe2e73b4272cd71b2c0d51f0d6c5e8134 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Wed, 2 Sep 2026 12:29:22 -0300 Subject: [PATCH 09/10] Tighten the publish workflow's comments ## Summary Comment-only. The explanations added while fixing this workflow had grown longer than what they explain, so they are cut back to the reasoning that is not recoverable from the code beside them: why `merge` waits on the legs rather than on their success, why the index is staged before it is signed, why the target CPU comes from the workflow env, and why the smoke test does not prove what it looks like it proves. 101 comment lines down to 81. No expression, step, or shell line is touched: diff <(git show HEAD: | grep -vE '^\s*#') \ <(grep -vE '^\s*#' ) is empty, so run 33642540774 still attests to this behaviour and the workflow needs no re-run. ## Test Plan 1. YAML parses and actionlint passes. 2. The non-comment diff against the previous revision is empty. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Aleksander Arruda --- .../workflows/publish-loreserver-image.yml | 124 ++++++++---------- 1 file changed, 52 insertions(+), 72 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index e4ac4060d..5fdfd9bd2 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -12,17 +12,15 @@ name: Publish loreserver image # :X.Y.Z baseline armv8-a arm64 — runs on any arm64 host # :X.Y.Z-graviton arm64 tuned for Graviton3+, as Lore is deployed # -# The default is the portable one, so a community chart works everywhere; the -# tuned build is opt-in for Graviton deployments. amd64 is baseline in both and -# is therefore built once, with both manifest lists pointing at that one digest. +# The default is portable so a community chart works everywhere; the tuned +# build is opt-in. amd64 is baseline in both, so it is built once and both +# manifest lists point at that digest. # -# `meta` resolves tags and labels once, so the later jobs cannot disagree and -# a ref with no tags fails early. Each variant then builds on its own native -# runner (QEMU is far too slow for a release Rust build) and is pushed by -# digest; `merge` stitches the digests into a manifest list, signs and verifies -# it under a staging tag, and only then attaches the release tags to the digest -# it just proved. So no release tag is ever left pointing at an unsigned image, -# even if Fulcio is having a bad day. +# `meta` resolves tags and labels once, so the later jobs cannot disagree. Each +# leg builds on its own native runner (QEMU is far too slow for a release Rust +# build) and pushes by digest. `merge` stitches those into an index, signs and +# verifies it under a staging tag, and only then attaches the release tags — so +# no release tag is ever left pointing at an unsigned image. # # No secrets: GITHUB_TOKEN authenticates to GHCR, and cosign signs keylessly # against Fulcio via the job's OIDC token. @@ -95,9 +93,8 @@ jobs: type=ref,event=branch type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} - # The same tag set, suffixed. onlatest carries the suffix onto `latest` - # so the tuned stream has its own moving tag rather than fighting for the - # shared one. + # The same tags, suffixed. `onlatest` carries the suffix onto `latest`, so + # the tuned stream has its own moving tag instead of fighting for that one. - id: meta-graviton uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: @@ -111,8 +108,8 @@ jobs: type=ref,event=branch type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} - # An empty list has to stop the run here, not surface as an invalid - # reference in the merge job once every architecture is pushed. + # An empty list has to fail here, not as an invalid reference in `merge` + # once every architecture has already been built. - name: Require at least one tag env: JSON: ${{ steps.meta.outputs.json }} @@ -137,16 +134,14 @@ jobs: fail-fast: false matrix: include: - # amd64 is baseline for both variants, so it is built once and both - # manifest lists reference this digest. + # amd64 is baseline for both variants, so it is built once and shared. - { arch: amd64, variant: base, platform: linux/amd64, runner: ubuntu-latest } - { arch: arm64, variant: base, platform: linux/arm64, runner: ubuntu-24.04-arm } - { arch: arm64, variant: graviton, platform: linux/arm64, runner: ubuntu-24.04-arm } env: - # The graviton leg carries the suffixed tag's metadata, so that `docker - # inspect` on a pulled image distinguishes the two arm64 variants. The - # amd64 leg is shared between both manifest lists and can only carry one - # set, which is the default's. + # The graviton leg carries the suffixed tag's metadata, so `docker inspect` + # tells the two arm64 variants apart. The shared amd64 leg can only carry + # one set, which is the default's. LABELS: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-labels || needs.meta.outputs.labels }} ANNOTATIONS: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-annotations || needs.meta.outputs.annotations }} steps: @@ -162,11 +157,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # The build argument below resolves from an expression, and an untuned - # build is a perfectly successful one — so were that expression ever to - # come back empty, the -graviton tag would quietly ship a baseline image - # and every later step would still be green. Assert the same expression - # here, where it can only cost a fast failure. + # An untuned build is a perfectly successful one, so an empty expression + # below would ship a baseline image under the -graviton tag with every + # step green. Assert it here, where it costs only a fast failure. - name: Check the graviton leg is tuned if: ${{ matrix.variant == 'graviton' }} env: @@ -184,37 +177,32 @@ jobs: context: . file: lore-server/Dockerfile platforms: ${{ matrix.platform }} - # Read from the workflow env rather than a matrix column, so retuning - # Graviton is one edit that cannot silently miss the build. `env` is - # unavailable in `strategy.matrix`, hence deriving it here from the - # variant instead. Empty is baseline; the Dockerfile ignores it on - # amd64 either way. + # From the workflow env, not a matrix column, so retuning Graviton is + # one edit. `env` is unavailable in `strategy.matrix`, hence deriving + # it here. Empty is baseline, and amd64 ignores it either way. build-args: ARM64_TARGET_CPU=${{ matrix.variant == 'graviton' && env.GRAVITON_TARGET_CPU || '' }} labels: ${{ env.LABELS }} annotations: ${{ env.ANNOTATIONS }} outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true - # Only the apt and toolchain layers survive a source change. The - # cargo registry and target directory live in BuildKit cache mounts, - # which the gha backend does not carry between runs, so the compile - # itself starts cold every time. Scoped per variant so the two arm64 - # legs, built with different codegen flags, cannot share layers. + # Only the apt and toolchain layers survive a source change: the cargo + # registry and target dir live in cache mounts, which the gha backend + # does not carry between runs, so the compile starts cold every time. + # Scoped per variant so the two arm64 legs cannot share layers. cache-from: type=gha,scope=loreserver-${{ matrix.arch }}-${{ matrix.variant }} cache-to: type=gha,mode=max,scope=loreserver-${{ matrix.arch }}-${{ matrix.variant }} # A clean build proves only that the image compiles, so start the binary - # too. Know the limit: this arm64 runner is Neoverse-N2 and reports SVE, - # so it executes the Graviton-tuned build quite happily. This catches a - # broken entrypoint or a missing shared library — it does NOT catch - # codegen aimed at a CPU the eventual host lacks, which is exactly how a + # too: it catches a broken entrypoint or a missing shared library. It does + # NOT catch codegen for a CPU the host lacks — this runner is Neoverse-N2 + # and reports SVE, so it runs the Graviton build happily, which is how a # SIGILL-ing arm64 image once passed a fully green run. - name: Smoke test the pushed image env: DIGEST: ${{ steps.build.outputs.digest }} run: docker run --rm "${IMAGE}@${DIGEST}" --version - # The merge job addresses each build by digest. Hand the digests over as - # empty files named after themselves, one artifact per matrix leg so the - # merge job can pick which legs belong in which manifest list. + # Hand each digest over as an empty file named after itself, one artifact + # per leg, so `merge` can pick which legs belong in which manifest list. - name: Export digest env: DIGEST: ${{ steps.build.outputs.digest }} @@ -232,20 +220,18 @@ jobs: merge: name: merge and sign (${{ matrix.variant }}) needs: [meta, build] - # `needs: build` alone waits on the whole matrix *and* requires all of it to - # succeed, which would let the opt-in graviton leg withhold the portable - # default image. Depend on the legs having finished, not on all of them - # having passed; `meta` is still a hard requirement, since a variant with no - # resolved tags has nothing to publish. Which legs a given variant actually - # needs is then the leg-existence check's call, further down. + # `needs: build` alone requires the whole matrix to pass, letting the opt-in + # graviton leg withhold the portable default image. Wait for the legs to + # finish, not to pass; `meta` stays required, since a variant with no tags + # has nothing to publish. Which legs a variant needs is the check's call + # below. if: ${{ !cancelled() && needs.meta.result == 'success' }} runs-on: ubuntu-latest permissions: packages: write # Push the manifest list and the cosign signature id-token: write # Federate to Fulcio for keyless signing strategy: - # One variant failing should not withhold the other; both are - # independently useful once published. + # One variant failing should not withhold the other. fail-fast: false matrix: include: @@ -257,8 +243,7 @@ jobs: JSON: ${{ matrix.variant == 'graviton' && needs.meta.outputs.graviton-json || needs.meta.outputs.json }} LEGS: ${{ matrix.legs }} # Where the index lands before it is signed. Commit-addressed, so it never - # moves and nothing has to be pointed away from it later; kept afterwards - # as a record of which commit produced which digest. + # moves; kept afterwards as a record of which commit produced which digest. STAGING_TAG: sha-${{ github.sha }}${{ matrix.variant == 'graviton' && '-graviton' || '' }} # Anchored on this workflow's path: a bare repository prefix would # accept a certificate minted by any workflow here. @@ -279,20 +264,17 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Publishing happens in two phases. Here the index is created under the - # staging tag alone, so a failure between this step and a verified - # signature cannot leave a release tag — `latest` above all — pointing at - # an unsigned digest. The release tags are attached further down, once a - # signature exists and has verified. + # Phase one: the index under the staging tag alone, so a failure before + # the signature verifies cannot leave a release tag — `latest` above all — + # pointing at an unsigned digest. Release tags are attached further down. - name: Create the multi-arch manifest list working-directory: ${{ runner.temp }}/digests run: | set -euo pipefail args=(--tag "${IMAGE}:${STAGING_TAG}") - # The build job annotated each per-arch manifest; the index is - # created here, so it needs the same annotations applying to it. - # They have to land now: annotations are part of what the digest - # covers, and the promotion below must not alter the digest. + # The build job annotated each per-arch manifest; the index is created + # here, so it needs the same. They must land now: annotations are part + # of the digest, and promotion below must not alter it. while IFS= read -r annotation; do args+=(--annotation "index:${annotation}") done < <(jq -r '.labels | to_entries[] | "\(.key)=\(.value)"' <<< "$JSON") @@ -338,13 +320,12 @@ jobs: > /dev/null echo "signature verified" - # Promotion below rests on `imagetools create` copying a lone index source - # through byte for byte, which is what makes the release tags resolve to - # the digest just signed. Prove that on the staging tag first, by copying - # the index onto the tag it already occupies: a no-op if the digest is - # preserved, and if some later buildx stops preserving it, the tag that - # moves out from under the signature is this one, which nothing consumes. - # Checking afterwards instead would mean finding out from `latest`. + # Promotion rests on `imagetools create` copying a lone index source + # through unchanged. Prove that first by copying the index onto the tag it + # already occupies: a no-op while the digest holds, and if some later + # buildx stops preserving it, the tag that moves out from under the + # signature is this one, which nothing consumes. Checking afterwards + # instead would mean learning it from `latest`. - name: Prove the copy preserves the digest env: DIGEST: ${{ steps.digest.outputs.digest }} @@ -359,9 +340,8 @@ jobs: fi echo "copy is digest-preserving" - # Phase two: the release tags, now that a signature exists over the digest - # and verifies. Each tag is re-resolved afterwards, so a tag pointing at - # anything other than the signed digest fails the run rather than shipping. + # Phase two: the release tags, now the signature exists and verifies. Each + # is re-resolved afterwards, so a tag off the signed digest fails the run. - name: Promote the signed digest to the release tags env: DIGEST: ${{ steps.digest.outputs.digest }} From 2935aaffc62b23b5a0fda00ba60c378ff76335ae Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Thu, 3 Sep 2026 15:44:48 -0300 Subject: [PATCH 10/10] Stop shellcheck flagging the tag gate Rewriting the gate as a loop introduced `for json in "$JSON" ...`, a local differing from the environment variable beside it only in case. That is what SC2153 looks for, so actionlint failed the workflow: SC2153:info:2:14: Possible misspelling: JSON may not be assigned. Did you mean json? Name the loop variable `tags_json`. Behaviour is unchanged. Checked the rest of the file for the same pattern: no other script assigns a local whose uppercase form matches an environment variable it reads, and shellcheck now passes on all thirteen `run` blocks. Signed-off-by: Aleksander Arruda Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Aleksander Arruda --- .github/workflows/publish-loreserver-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-loreserver-image.yml b/.github/workflows/publish-loreserver-image.yml index 5fdfd9bd2..ecbef1dd1 100644 --- a/.github/workflows/publish-loreserver-image.yml +++ b/.github/workflows/publish-loreserver-image.yml @@ -116,8 +116,8 @@ jobs: GRAVITON_JSON: ${{ steps.meta-graviton.outputs.json }} run: | set -euo pipefail - for json in "$JSON" "$GRAVITON_JSON"; do - if [ -z "$(jq -r '.tags[0] // empty' <<< "${json}")" ]; then + for tags_json in "$JSON" "$GRAVITON_JSON"; do + if [ -z "$(jq -r '.tags[0] // empty' <<< "${tags_json}")" ]; then echo "::error::no image tags resolved for ${GITHUB_REF}; push a semver tag or pass the 'tag' input" exit 1 fi