From b723c0638f6356e5817bd5022f80065007528891 Mon Sep 17 00:00:00 2001 From: zhangxingyan1 Date: Tue, 7 Apr 2026 16:01:59 +0800 Subject: [PATCH] Fix FSQ GPU code packing precision bug --- s3tokenizer/model_v2.py | 16 +++++++++----- test/test_fsq_codebook.py | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 test/test_fsq_codebook.py diff --git a/s3tokenizer/model_v2.py b/s3tokenizer/model_v2.py index f8eda98..4279f7e 100644 --- a/s3tokenizer/model_v2.py +++ b/s3tokenizer/model_v2.py @@ -102,13 +102,17 @@ def encode(self, x: torch.Tensor) -> torch.Tensor: h = self.project_down(x).float() h = h.tanh() h = h * 0.9990000128746033 - h = h.round() + 1 - # h = ((self.level - 1) * h).round() # range [-k, k] - powers = torch.pow( - self.level, - torch.arange(2**self.level, device=x.device, dtype=h.dtype)) + h = (h.round() + 1).to(torch.int64) + # Use integer arithmetic for code packing. Floating-point pow() on some + # CUDA stacks produces values like 242.99998 for 3**5, and the final + # int() cast truncates the packed token id by 1. + powers = torch.tensor( + [self.level**i for i in range(2**self.level)], + device=x.device, + dtype=torch.int64, + ) mu = torch.sum(h * powers.unsqueeze(0), dim=-1) - ind = mu.reshape(x_shape[0], x_shape[1]).int() + ind = mu.reshape(x_shape[0], x_shape[1]).to(torch.int32) return ind @torch.inference_mode() diff --git a/test/test_fsq_codebook.py b/test/test_fsq_codebook.py new file mode 100644 index 0000000..980878a --- /dev/null +++ b/test/test_fsq_codebook.py @@ -0,0 +1,46 @@ +import torch +from torch import nn + +from s3tokenizer.model_v2 import FSQCodebook + + +class FakeProjectDown(nn.Module): + + def __init__(self, output: torch.Tensor): + super().__init__() + self.register_buffer("output", output) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + batch, steps = x.shape[:2] + assert self.output.shape[:2] == (batch, steps) + return self.output.to(x.device) + + +def test_fsq_code_packing_returns_exact_token_ids(): + codebook = FSQCodebook(dim=8, level=3) + digits = torch.tensor( + [[[2, 0, 0, 0, 0, 2, 0, 0], [1, 0, 0, 0, 0, 2, 0, 0]]], + dtype=torch.int64, + ) + pre_tanh = torch.where( + digits == 2, + torch.full_like(digits, 20), + torch.where(digits == 1, torch.zeros_like(digits), torch.full_like(digits, -20)), + ).to(torch.float32) + codebook.project_down = FakeProjectDown(pre_tanh) + + dummy_hidden = torch.zeros(1, 2, 8) + packed = codebook.encode(dummy_hidden) + + assert torch.equal(packed, torch.tensor([[488, 487]], dtype=torch.int32)) + + +def test_cuda_float_pow_can_be_non_integer(): + if not torch.cuda.is_available(): + return + + powers = torch.pow(3, torch.arange(8, device="cuda", dtype=torch.float32)).cpu() + + # On some CUDA stacks, 3**5 becomes 242.99998 instead of 243.0. + # This test documents the motivation for avoiding float pow in code packing. + assert powers[5].item() <= 243.0