Zcashd orchard anchor repro on grpc

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ on:
       - zebra-interop-request
       - zaino-interop-request
       - zallet-interop-request
+      - lightwalletd-interop-request
 
 permissions:
   contents: read
@@ -614,6 +615,83 @@ jobs:
           requesting-repository: ${{ steps.repo-ids.outputs.requesting-repository }}
           job-name: "Build zallet on ${{ matrix.platform }}${{ matrix.required_suffix }}"
 
+  build-lightwalletd:
+    name: Build lightwalletd on ${{ matrix.platform }}${{ matrix.required_suffix }}
+    needs: setup
+    runs-on: ${{ matrix.build_os }}
+    container:
+      image: ${{ matrix.container }}
+      env:
+        HOME: /root
+        PATH: /root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
+    continue-on-error: ${{ !matrix.required }}
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.setup.outputs.build_matrix) }}
+
+    steps:
+      - name: Check out integration-tests to access actions
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          path: integration-tests
+          persist-credentials: false
+
+      - name: Compute interop repo ids
+        id: repo-ids
+        uses: ./integration-tests/.github/actions/interop-repo-ids
+
+      - id: start-interop
+        uses: ./integration-tests/.github/actions/start-interop
+        with:
+          status-app-id: ${{ secrets.STATUS_APP_ID }}
+          status-app-private-key: ${{ secrets.STATUS_APP_PRIVATE_KEY }}
+          requesting-owner: ${{ steps.repo-ids.outputs.requesting-owner }}
+          requesting-repository: ${{ steps.repo-ids.outputs.requesting-repository }}
+          job-name: "Build lightwalletd on ${{ matrix.platform }}${{ matrix.required_suffix }}"
+
+      - name: Use specified zcash/lightwalletd commit
+        if: github.event.action == 'lightwalletd-interop-request'
+        shell: sh
+        env:
+          SHA: ${{ github.event.client_payload.sha }}
+        run: echo "LIGHTWALLETD_REF=${SHA}" >> $GITHUB_ENV
+
+      - name: Use zcash/lightwalletd current master
+        if: github.event.action != 'lightwalletd-interop-request'
+        run: echo "LIGHTWALLETD_REF=refs/heads/master" >> $GITHUB_ENV
+
+      - name: Check out zcash/lightwalletd
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: zcash/lightwalletd
+          ref: ${{ env.LIGHTWALLETD_REF }}
+          path: lightwalletd
+
+      - name: Set up Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version-file: lightwalletd/go.mod
+          cache-dependency-path: lightwalletd/go.sum
+
+      - name: Build lightwalletd
+        run: go build -v -o lightwalletd${{ matrix.file_ext }} .
+        working-directory: ./lightwalletd
+
+      - name: Upload lightwalletd
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: lightwalletd-${{ matrix.name }}
+          path: |
+              ${{ format('./lightwalletd/lightwalletd{0}', matrix.file_ext) }}
+
+      - uses: ./integration-tests/.github/actions/finish-interop
+        if: always()
+        with:
+          app-token: ${{ steps.start-interop.outputs.app-token }}
+          requesting-owner: ${{ steps.repo-ids.outputs.requesting-owner }}
+          requesting-repository: ${{ steps.repo-ids.outputs.requesting-repository }}
+          job-name: "Build lightwalletd on ${{ matrix.platform }}${{ matrix.required_suffix }}"
+
   # Not working in Windows
   sec-hard:
     name: sec-hard ${{ matrix.platform }}${{ matrix.required_suffix }}
@@ -622,6 +700,7 @@ jobs:
       - build-zebra
       - build-zaino
       - build-zallet
+      - build-lightwalletd
     runs-on: ${{ matrix.test_os }}
     container:
       image: ${{ matrix.container }}
@@ -682,12 +761,19 @@ jobs:
           name: zallet-${{ matrix.name }}
           path: ./src
 
+      - name: Download lightwalletd artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: lightwalletd-${{ matrix.name }}
+          path: ./src
+
       - name: Make artifact executable
         if: runner.os != 'Windows'
         run: |
           chmod +x ${{ format('./src/zebrad{0}', matrix.file_ext) }}
           chmod +x ${{ format('./src/zainod{0}', matrix.file_ext) }}
           chmod +x ${{ format('./src/zallet{0}', matrix.file_ext) }}
+          chmod +x ${{ format('./src/lightwalletd{0}', matrix.file_ext) }}
 
       - name: Run sec-hard test
         shell: bash
@@ -719,18 +805,19 @@ jobs:
         include: ${{ fromJson(needs.setup.outputs.rpc_test_matrix) }}
 
     steps:
-      - name: Cache Python dependencies for RPC tests
+      - name: Cache uv project environment for RPC tests
         uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
         with:
           enableCrossOsArchive: true
-          path: venv
-          key: test-rpc-venv-${{ matrix.name }}
+          path: .venv
+          key: test-rpc-venv-${{ matrix.name }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+
+      - name: Install uv
+        run: python3 -m pip install uv
 
-      - name: Get Python dependencies for RPC tests
+      - name: Sync Python dependencies for RPC tests
         run: |
-          python3 -m venv ./venv
-          . ./venv/bin/activate
-          pip install zmq asyncio base58 toml
+          uv sync --frozen
 
   test-rpc:
     name: RPC tests ${{ matrix.platform }} ${{ matrix.shard }}${{ matrix.required_suffix }}
@@ -739,6 +826,7 @@ jobs:
       - build-zebra
       - build-zaino
       - build-zallet
+      - build-lightwalletd
       - rpc-depends
     runs-on: ${{ matrix.test_os }}
     container:
@@ -785,18 +873,19 @@ jobs:
           requesting-repository: ${{ steps.repo-ids.outputs.requesting-repository }}
           job-name: "RPC tests ${{ matrix.platform }} ${{ matrix.shard }}${{ matrix.required_suffix }}"
 
-      - name: Cache Python dependencies for RPC tests
+      - name: Cache uv project environment for RPC tests
         uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
         with:
           enableCrossOsArchive: true
-          path: venv
-          key: test-rpc-venv-${{ matrix.name }}
+          path: .venv
+          key: test-rpc-venv-${{ matrix.name }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
 
-      - name: Get Python dependencies for RPC tests if not cached
+      - name: Install uv
+        run: python3 -m pip install uv
+
+      - name: Sync Python dependencies for RPC tests
         run: |
-          python3 -m venv ./venv
-          . ./venv/bin/activate
-          pip install zmq asyncio base58 toml
+          uv sync --frozen
 
       - name: Download zebrad artifact
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -816,12 +905,19 @@ jobs:
           name: zallet-${{ matrix.name }}
           path: ./src
 
+      - name: Download lightwalletd artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: lightwalletd-${{ matrix.name }}
+          path: ./src
+
       - name: Make artifact executable
         if: runner.os != 'Windows'
         run: |
           chmod +x ${{ format('./src/zebrad{0}', matrix.file_ext) }}
           chmod +x ${{ format('./src/zainod{0}', matrix.file_ext) }}
           chmod +x ${{ format('./src/zallet{0}', matrix.file_ext) }}
+          chmod +x ${{ format('./src/lightwalletd{0}', matrix.file_ext) }}
 
       - name: Get Sprout parameters
         uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
@@ -877,8 +973,7 @@ jobs:
           if all_passed == False:
               sys.exit(1)
           EOF
-          . ./venv/bin/activate
-          ZEBRAD=$(pwd)/${{ format('src/zebrad{0}', matrix.file_ext) }} ZAINOD=$(pwd)/${{ format('src/zainod{0}', matrix.file_ext) }} ZALLET=$(pwd)/${{ format('src/zallet{0}', matrix.file_ext) }} SRC_DIR=$(pwd) python3 ./subclass.py
+          ZEBRAD=$(pwd)/${{ format('src/zebrad{0}', matrix.file_ext) }} ZAINOD=$(pwd)/${{ format('src/zainod{0}', matrix.file_ext) }} ZALLET=$(pwd)/${{ format('src/zallet{0}', matrix.file_ext) }} LIGHTWALLETD=$(pwd)/${{ format('src/lightwalletd{0}', matrix.file_ext) }} SRC_DIR=$(pwd) uv run python3 ./subclass.py
 
       - uses: ./.github/actions/finish-interop
         if: always()

.gitignore

@@ -5,10 +5,13 @@
 *.orig
 *.pyc
 .vscode
-
+.claude
+.codex
 qa/pull-tester/tests_config.py
 qa/pull-tester/tests_config.ini
 qa/cache/*
+qa/rpc-tests/cache/grpc_comparison/
+qa/rpc-tests/cache/grpc_comparison_stage1/
 src/*
 
 poetry.lock

README.md

@@ -7,6 +7,9 @@ Zcash ecosystem. The following tests are provided:
 
 - Functional tests in Python of [`zebrad`], [`zainod`], and [`zallet`], using
   regtest mode and primarily their JSON-RPC interfaces.
+- gRPC parity tests that run [`zainod`] and [`lightwalletd`] side-by-side
+  against the same [`zebrad`] node and compare their
+  [lightwallet-protocol] gRPC responses.
 
 The functional tests and CI workflows were originally part of the [`zcashd`]
 codebase, with the Python test framework (and some of the tests) inherited from
@@ -15,37 +18,35 @@ codebase, with the Python test framework (and some of the tests) inherited from
 [`zebrad`]: https://github.com/ZcashFoundation/zebra
 [`zainod`]: https://github.com/zingolabs/zaino
 [`zallet`]: https://github.com/zcash/wallet
+[`lightwalletd`]: https://github.com/zcash/lightwalletd
+[lightwallet-protocol]: https://github.com/zcash/lightwallet-protocol
 [`zcashd`]: https://github.com/zcash/zcash
 [Bitcoin Core]: https://github.com/bitcoin/bitcoin
 <!-- ANCHOR_END: summary -->
 
 ## Getting Started
 
 ### Running the tests locally
+Pre-requisite: See the [`uv` installation instructions](https://docs.astral.sh/uv/getting-started/installation/)
+  if it is not already installed.
 
 - Clone the repository.
 - Build `zebrad`, `zainod`, and `zallet` binaries, and place them in a folder
   `./src/` under the repository root.
-
-#### With uv (recommended)
-
 - `uv sync`
 - `uv run ./qa/zcash/full_test_suite.py`
 
-#### Without uv
+See [the README for the functional tests][qa/README.md] for additional usage
+information.
 
-On Ubuntu or Debian-based distributions:
-- `sudo apt-get install python3-zmq python3-base58 python3-toml`
-- `./qa/zcash/full_test_suite.py`
+### Running the gRPC parity tests
 
-On macOS or other platforms:
-- `python3 -m venv venv`
-- `. venv/bin/activate`
-- `pip3 install pyzmq base58 toml`
-- `./qa/zcash/full_test_suite.py`
+The gRPC parity tests additionally require the `lightwalletd` binary in `./src/`
+(or set `LIGHTWALLETD=/path/to/lightwalletd`).
 
-See [the README for the functional tests][qa/README.md] for additional usage
-information.
+```bash
+uv run ./qa/zcash/grpc_comparison_tests.py
+```
 
 ### Writing tests
 

doc/book/src/SUMMARY.md

@@ -7,6 +7,7 @@
   - [Platform Support](user/platform-support.md)
 - [Developer Documentation](dev.md)
   - [Regtest Mode](dev/regtest.md)
+  - [Bringing `grpc_comparison.py` Live](dev/grpc-comparison.md)
   - [Platform Policy](dev/platform-policy.md)
 - [CI Infrastructure](ci/README.md)
   - [Cross-Repository CI](ci/cross-repo.md)