xmtplabs · neekolas · Apr 15, 2026
@@ -6,6 +6,7 @@ on:
   push:
     branches:
       - otr-dev
+      - otr-testnet
       - otr-prod
   pull_request:
   workflow_dispatch:
@@ -16,9 +17,10 @@ on:
         type: choice
         options:
           - otr-dev
+          - otr-testnet
           - otr-prod
   repository_dispatch:
-    types: [deploy-otr-prod]
+    types: [deploy-otr-testnet, deploy-otr-prod]
   workflow_call:
     inputs:
       ref:
@@ -138,6 +140,28 @@ jobs:
           variable-value: "ghcr.io/xmtplabs/convos-backend@${{ needs.push_to_registry.outputs.digest }}"
           variable-value-required-prefix: "ghcr.io/xmtplabs/convos-backend@sha256:"
 
+  deploy_otr_testnet:
+    name: Deploy to OTR Testnet
+    runs-on: ubuntu-latest
+    needs: push_to_registry
+    if: (inputs.ref == 'otr-testnet') || (inputs.environment == 'otr-testnet') || (github.ref == 'refs/heads/otr-testnet' && !inputs.ref && !inputs.environment) || (github.event_name == 'repository_dispatch' && github.event.client_payload.ref == 'otr-testnet')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event_name == 'workflow_dispatch' && inputs.environment || github.event_name == 'workflow_call' && inputs.ref || github.ref }}
+      - name: Deploy to OTR Testnet Environment
+        uses: xmtp-labs/terraform-deployer@v1
+        timeout-minutes: 20
+        with:
+          timeout: 20m
+          terraform-token: ${{ secrets.TERRAFORM_TOKEN }}
+          terraform-org: xmtp
+          terraform-workspace: convos-otr-testnet
+          variable-name: api_image
+          variable-value: "ghcr.io/xmtplabs/convos-backend@${{ needs.push_to_registry.outputs.digest }}"
+          variable-value-required-prefix: "ghcr.io/xmtplabs/convos-backend@sha256:"
+
   test_lifecycle_dev:
     name: Test S3 Lifecycle Renewal (Dev)
     runs-on: ubuntu-latest
@@ -174,6 +198,42 @@ jobs:
           fi
           echo "✅ Lifecycle renewal test passed (dev)" >> $GITHUB_STEP_SUMMARY
 
+  test_lifecycle_testnet:
+    name: Test S3 Lifecycle Renewal (Testnet)
+    runs-on: ubuntu-latest
+    needs: deploy_otr_testnet
+    steps:
+      - name: Wait for deployment and verify health
+        run: |
+          echo "Waiting for deployment to stabilize..."
+          for i in {1..10}; do
+            sleep 10
+            if curl -sf "${{ vars.TESTNET_API_URL }}/healthcheck" > /dev/null 2>&1; then
+              echo "Service is healthy after $((i * 10)) seconds"
+              exit 0
+            fi
+            echo "Waiting for service to be ready (attempt $i/10)..."
+          done
+          echo "::error::Service did not become healthy after 100s"
+          exit 1
+
+      - name: Test lifecycle renewal endpoint
+        run: |
+          response=$(curl -s -w "\n%{http_code}" \
+            --max-time 60 \
+            --retry 3 \
+            --retry-connrefused \
+            -X POST "${{ vars.TESTNET_API_URL }}/api/v2/assets/test/lifecycle" \
+            -H "Authorization: Bearer ${{ secrets.LIFECYCLE_TEST_TOKEN }}")
+          http_code=$(echo "$response" | tail -1)
+          body=$(echo "$response" | head -n -1)
+          echo "$body"
+          if [ "$http_code" != "200" ] || ! echo "$body" | jq -e '.success == true' > /dev/null 2>&1; then
+            echo "::error::Lifecycle test failed!"
+            exit 1
+          fi
+          echo "✅ Lifecycle renewal test passed (testnet)" >> $GITHUB_STEP_SUMMARY
+
   test_lifecycle_prod:
     name: Test S3 Lifecycle Renewal (Prod)
     runs-on: ubuntu-latest

@@ -78,6 +78,77 @@ jobs:
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "Status: **healthy**" >> $GITHUB_STEP_SUMMARY
 
+  verify-testnet:
+    name: Verify Lifecycle (Testnet)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verify S3 lifecycle canaries
+        run: |
+          response=$(curl -s -w "\n%{http_code}" \
+            --max-time 60 \
+            --retry 3 \
+            --retry-connrefused \
+            -X POST "${{ vars.TESTNET_API_URL }}/api/v2/assets/test/lifecycle-status" \
+            -H "Authorization: Bearer ${{ secrets.LIFECYCLE_TEST_TOKEN }}")
+          http_code=$(echo "$response" | tail -1)
+          body=$(echo "$response" | head -n -1)
+          echo "Response:"
+          echo "$body" | jq . || echo "$body"
+          if [ "$http_code" != "200" ]; then
+            echo "::error::Lifecycle verification failed with HTTP $http_code"
+            exit 1
+          fi
+          if ! echo "$body" | jq -e '.status == "healthy"' > /dev/null 2>&1; then
+            echo "::error::Lifecycle verification returned unhealthy status"
+            errors=$(echo "$body" | jq -r '.errors[]' 2>/dev/null || echo "Unknown errors")
+            echo "Errors: $errors"
+            exit 1
+          fi
+
+          # Build summary
+          echo "## Lifecycle Verification (Testnet)" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Date: $(echo "$body" | jq -r '.date')" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Created Canaries" >> $GITHUB_STEP_SUMMARY
+          echo "- Delete canary: \`$(echo "$body" | jq -r '.created.deleteCanary')\`" >> $GITHUB_STEP_SUMMARY
+          echo "- Keep canary: \`$(echo "$body" | jq -r '.created.keepCanary')\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "### Renewed" >> $GITHUB_STEP_SUMMARY
+          if [ "$(echo "$body" | jq '.renewed // [] | length')" -eq 0 ]; then
+            echo "- _(none)_" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "$body" | jq -r '.renewed // [] | .[]' | while read -r key; do echo "- \`$key\`"; done >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "### Verified" >> $GITHUB_STEP_SUMMARY
+          echo "**Deleted as expected:**" >> $GITHUB_STEP_SUMMARY
+          if [ "$(echo "$body" | jq '.verified.deletedAsExpected // [] | length')" -eq 0 ]; then
+            echo "- _(none yet - first week of operation)_" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "$body" | jq -r '.verified.deletedAsExpected // [] | .[]' | while read -r key; do echo "- \`$key\`"; done >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "**Exists as expected:**" >> $GITHUB_STEP_SUMMARY
+          if [ "$(echo "$body" | jq '.verified.existsAsExpected // [] | length')" -eq 0 ]; then
+            echo "- _(none yet - first week of operation)_" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "$body" | jq -r '.verified.existsAsExpected // [] | .[]' | while read -r key; do echo "- \`$key\`"; done >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "### Cleaned" >> $GITHUB_STEP_SUMMARY
+          if [ "$(echo "$body" | jq '.cleaned // [] | length')" -eq 0 ]; then
+            echo "- _(none)_" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "$body" | jq -r '.cleaned // [] | .[]' | while read -r key; do echo "- \`$key\`"; done >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Status: **healthy**" >> $GITHUB_STEP_SUMMARY
+
   verify-prod:
     name: Verify Lifecycle (Prod)
     runs-on: ubuntu-latest

@@ -5,19 +5,33 @@
 - Default branch: `otr-dev`
 - PRs merged to `otr-dev` trigger automatic backend deployment via workflow
 
+## Testnet
+
+- Merge `otr-dev` into `otr-testnet` locally
+- Push directly to `otr-testnet` - **deploys to testnet**
+- **Always use fast-forward merge** to maintain identical commit history
+
+```bash
+git checkout otr-testnet
+git merge --ff-only otr-dev
+git push origin otr-testnet
+```
+
 ## Production
 
-- Merge `otr-dev` into `otr-prod` locally
+- Merge `otr-testnet` into `otr-prod` locally after testnet verification
 - Push directly to `otr-prod` - **deploys to production**
 - **Always use fast-forward merge** to maintain identical commit history
 
 ```bash
 git checkout otr-prod
-git merge --ff-only otr-dev
+git merge --ff-only otr-testnet
 git push origin otr-prod
 ```
 
 ## Rules
 
 - **Never push commits to `otr-prod` that aren't on `otr-dev`**
+- **Never push commits to `otr-prod` that aren't on `otr-testnet`**
 - All commits must exist on `otr-dev` first
+- Production commits must pass through `otr-testnet` first
@@ -43,7 +43,8 @@ export const appCheckOnlyMiddleware = async (
     req.log.info("AppCheck verification successful");
     next();
   } catch (error) {
-    req.log.error({ error }, "AppCheck verification failed");
+    const message = error instanceof Error ? error.message : String(error);
+    req.log.error({ error: message }, "AppCheck verification failed");
     res.status(401).json({ error: "Invalid AppCheck token" });
     return;
   }