diff --git a/.github/workflows/gem_release.yaml b/.github/workflows/gem_release.yaml
index eb48168c33..baa69ad13a 100644
--- a/.github/workflows/gem_release.yaml
+++ b/.github/workflows/gem_release.yaml
@@ -22,7 +22,7 @@ jobs:
           ruby-version: 'ruby'
       - name: Build gem
         shell: bash
-        run: gem build --verbose *.gemspec
+        run: bundle exec rake ci:gem_build
       - name: Upload gem to GitHub cache
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -46,6 +46,7 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
+        # gh supports wildcards for release assets, so we can just upload all gem files in one command
         run: gh release create --repo ${{ github.repository }} ${{ github.ref_name }} --generate-notes *.gem
 
   release-to-github:
@@ -60,9 +61,14 @@ jobs:
         with:
           name: gem-artifact
       - name: Publish gem to GitHub packages
-        run: gem push --host https://rubygems.pkg.github.com/${{ github.repository_owner }} *.gem
         env:
           GEM_HOST_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+        # gem push does not support wildcards, so we have to loop through all gem files
+        # each combination of gem name, platform and version is a unique artifact, so we can have multiple gem files with the same name, version but different platforms
+        run: |
+          for gem_file in *.gem; do
+            gem push --host "https://rubygems.pkg.github.com/${{ github.repository_owner }}" "$gem_file"
+          done
 
   release-to-rubygems:
     needs: build-release
@@ -79,7 +85,12 @@ jobs:
       - uses: rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a # v2.1.0
       - name: Publish gem to rubygems.org
         shell: bash
-        run: gem push *.gem
+        # gem push does not support wildcards, so we have to loop through all gem files
+        # each combination of gem name, platform and version is a unique artifact, so we can have multiple gem files with the same name, version but different platforms
+        run: |
+          for gem_file in *.gem; do
+            gem push "$gem_file"
+          done
 
   release-verification:
     name: Check that all releases are done
@@ -101,6 +112,9 @@ jobs:
           ruby-version: 'ruby'
       - name: Wait for release to propagate
         shell: bash
+        # we upload different gem files for different platforms, so we have to wait for all of them to be available.
         run: |
           gem install rubygems-await
-          gem await *.gem
+          for gem_file in *.gem; do
+            gem await "$gem_file"
+          done
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 940b4661fa..2ea8c15e33 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -2,7 +2,9 @@
 name: RSpec tests
 
 on:
-  pull_request: { branches: ['main'] }
+  pull_request:
+    branches:
+      - main
   push:
     branches:
       - main
@@ -13,6 +15,10 @@ permissions:
 env:
   BUNDLE_SET: "without packaging documentation release"
 
+concurrency:
+  group: rspec-tests-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   validate_manpages:
     name: Validate man pages
@@ -87,14 +93,10 @@ jobs:
         with:
           ruby-version: ${{ matrix.cfg.ruby }}
           bundler-cache: true
+
       - name: Output Ruby Environment
         run: bundle env
-      - name: Build gem for all platforms
-        run: bundle exec rake pl_ci:gem_build
-      # Windows tests fail if the gem is in the filesystem
-      - name: Cleanup builds
-        if: runner.os == 'Windows'
-        run: Remove-Item -Path *.gem
+
       - name: Run tests on Windows
         if: runner.os == 'Windows'
         run: |
@@ -175,13 +177,94 @@ jobs:
           bundle exec rake -T | grep vox:upload
         working-directory: packaging
 
+  test_gem_build:
+    name: Test gem build
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install Ruby
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+        with:
+          ruby-version: ruby
+          bundler-cache: true
+
+      - name: Build gem for all platforms
+        run: bundle exec rake ci:gem_build
+
+      - name: Upload gem to GitHub cache
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: gem-artifact
+          path: '*.gem'
+          retention-days: 1
+          compression-level: 0
+
+  test_install:
+    name: Test gem install
+    needs: test_gem_build
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            gem-platform: ruby
+          - os: macos-15
+            gem-platform: universal-darwin
+          - os: windows-2025
+            gem-platform: x64-mingw-ucrt
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Download gem from GitHub cache
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: gem-artifact
+
+      - name: Determine gem file
+        id: gem
+        shell: bash
+        env:
+          GEM_PLATFORM: ${{ matrix.gem-platform }}
+        run: |
+          gem_file="$(
+            ruby -rrubygems -e '
+              spec = Gem::Specification.load("openvox.gemspec")
+              abort "Could not load openvox.gemspec" unless spec
+              spec.platform = ENV.fetch("GEM_PLATFORM") unless ENV["GEM_PLATFORM"] == "ruby"
+              puts spec.file_name
+            '
+          )"
+
+          test -f "$gem_file"
+          echo "file=$gem_file" >> "$GITHUB_OUTPUT"
+
+      - name: Install Ruby
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+        with:
+          ruby-version: ruby
+          bundler-cache: true
+
+      - name: Install gem
+        run: gem install "${{ steps.gem.outputs.file }}" --verbose
+
+      - name: Smoke test
+        shell: bash
+        run: |
+          puppet --version
+          ruby -rpuppet -e 'puts Puppet.version'
+
   tests:
     if: always()
     needs:
-      - validate_manpages
       - checks
-      - rspec_tests
       - rake_checks
+      - rspec_tests
+      - test_gem_build
+      - test_install
+      - validate_manpages
     runs-on: ubuntu-24.04
     name: Test suite
     steps:
diff --git a/Gemfile b/Gemfile
index dfed2740b2..cd3b2fb542 100644
--- a/Gemfile
+++ b/Gemfile
@@ -55,6 +55,8 @@ group(:test) do
   gem 'webmock', '~> 3.0', require: false
   gem 'webrick', '~> 1.7', require: false
   gem 'yard', require: false
+  # The source gemspec has the generic ruby platform, so declare this explicitly for Windows development and CI.
+  gem 'win32ole', '>= 1.8', '< 2.0', require: false, platforms: [:windows]
 
   gem 'rubocop', '~> 1.89.0', require: false, platforms: [:ruby]
   gem 'rubocop-i18n', '~> 3.0', require: false, platforms: [:ruby]
diff --git a/Rakefile b/Rakefile
index fe55f3057d..46f829f5f4 100644
--- a/Rakefile
+++ b/Rakefile
@@ -56,13 +56,12 @@ task :default do
   sh %{rake -T}
 end
 
-namespace :pl_ci do
+namespace :ci do
   desc 'Build openvox gems'
   task :gem_build, [:gemspec] do |t, args|
     args.with_defaults(gemspec: 'openvox.gemspec')
     stdout, stderr, status = Open3.capture3(<<~END)
-      gem build #{args.gemspec} --verbose --strict --platform x86-mingw32 && \
-      gem build #{args.gemspec} --verbose --strict --platform x64-mingw32 && \
+      gem build #{args.gemspec} --verbose --strict --platform x64-mingw-ucrt && \
       gem build #{args.gemspec} --verbose --strict --platform universal-darwin && \
       gem build #{args.gemspec} --verbose --strict
     END
@@ -92,7 +91,7 @@ namespace :pl_ci do
         end
       end
       dst.flush
-      Rake::Task['pl_ci:gem_build'].invoke(dst.path)
+      Rake::Task['ci:gem_build'].invoke(dst.path)
     end
   end
 end
diff --git a/openvox.gemspec b/openvox.gemspec
index efad5f9330..d8701e4e12 100644
--- a/openvox.gemspec
+++ b/openvox.gemspec
@@ -22,7 +22,20 @@ Gem::Specification.new do |spec|
   EOF
   spec.email = "openvox@voxpupuli.org"
   spec.executables = ["puppet"]
-  spec.files = Dir['[A-Z]*'] + Dir['install.rb'] + Dir['bin/*'] + Dir['lib/**/*'] + Dir['conf/*'] + Dir['man/**/*'] + Dir['tasks/*'] + Dir['locales/**/*'] + Dir['ext/**/*'] + Dir['examples/**/*']
+  spec.files = Dir.chdir(__dir__) do
+    Dir.glob(%w[
+      [A-Z]*
+      install.rb
+      bin/*
+      lib/**/*
+      conf/*
+      man/**/*
+      tasks/*
+      locales/**/*
+      ext/**/*
+      examples/**/*
+    ]).reject { |path| File.directory?(path) || File.extname(path) == '.gem' }
+  end
   spec.license = "Apache-2.0"
   spec.homepage = "https://github.com/OpenVoxProject/openvox"
   spec.rdoc_options = ["--title", "OpenVox - Configuration Management", "--main", "README", "--line-numbers"]
@@ -43,16 +56,19 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('racc', '~> 1.5')
   spec.add_runtime_dependency('scanf', '~> 1.0')
   spec.add_runtime_dependency('semantic_puppet', '~> 1.0')
-  spec.add_runtime_dependency('win32ole', '>= 1.8', '< 2.0') if Gem.win_platform?
 
   platform = spec.platform.to_s
+  windows_platforms = %w[x64-mingw-ucrt]
+  windows = windows_platforms.include?(platform)
+
   if platform == 'universal-darwin'
     spec.add_runtime_dependency('CFPropertyList', ['>= 3.0.6', '< 5'])
   end
 
-  if platform == 'x64-mingw32' || platform == 'x86-mingw32'
+  if windows
     # ffi 1.16.0 - 1.16.2 are broken on Windows
     spec.add_runtime_dependency('ffi', '>= 1.15.5', '< 2', '!= 1.16.0', '!= 1.16.1', '!= 1.16.2')
     spec.add_runtime_dependency('minitar', '~> 1.0')
+    spec.add_runtime_dependency('win32ole', '>= 1.8', '< 2.0')
   end
 end