Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions s3tokenizer/model_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,17 @@ def encode(self, x: torch.Tensor) -> torch.Tensor:
h = self.project_down(x).float()
h = h.tanh()
h = h * 0.9990000128746033
h = h.round() + 1
# h = ((self.level - 1) * h).round() # range [-k, k]
powers = torch.pow(
self.level,
torch.arange(2**self.level, device=x.device, dtype=h.dtype))
h = (h.round() + 1).to(torch.int64)
# Use integer arithmetic for code packing. Floating-point pow() on some
# CUDA stacks produces values like 242.99998 for 3**5, and the final
# int() cast truncates the packed token id by 1.
powers = torch.tensor(
[self.level**i for i in range(2**self.level)],
device=x.device,
dtype=torch.int64,
)
mu = torch.sum(h * powers.unsqueeze(0), dim=-1)
ind = mu.reshape(x_shape[0], x_shape[1]).int()
ind = mu.reshape(x_shape[0], x_shape[1]).to(torch.int32)
return ind

@torch.inference_mode()
Expand Down
46 changes: 46 additions & 0 deletions test/test_fsq_codebook.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import torch
from torch import nn

from s3tokenizer.model_v2 import FSQCodebook


class FakeProjectDown(nn.Module):

def __init__(self, output: torch.Tensor):
super().__init__()
self.register_buffer("output", output)

def forward(self, x: torch.Tensor) -> torch.Tensor:
batch, steps = x.shape[:2]
assert self.output.shape[:2] == (batch, steps)
return self.output.to(x.device)
Comment on lines +13 to +16

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Make the fake match FSQCodebook.project_down()'s actual shape contract.

FSQCodebook.encode() flattens (batch, steps, dim) to ((batch * steps), dim) in preprocess() before it calls project_down(). With dummy_hidden.shape == (1, 2, 8), x.shape here is (2, 8), so this assertion fails immediately and the regression test never reaches the packing logic.

🧪 Proposed fix
 class FakeProjectDown(nn.Module):
@@
     def forward(self, x: torch.Tensor) -> torch.Tensor:
-        batch, steps = x.shape[:2]
-        assert self.output.shape[:2] == (batch, steps)
-        return self.output.to(x.device)
+        flat_output = self.output.reshape(-1, self.output.shape[-1])
+        assert x.shape == flat_output.shape
+        return flat_output.to(x.device)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
def forward(self, x: torch.Tensor) -> torch.Tensor:
batch, steps = x.shape[:2]
assert self.output.shape[:2] == (batch, steps)
return self.output.to(x.device)
def forward(self, x: torch.Tensor) -> torch.Tensor:
flat_output = self.output.reshape(-1, self.output.shape[-1])
assert x.shape == flat_output.shape
return flat_output.to(x.device)
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@test/test_fsq_codebook.py` around lines 13 - 16, The fake forward currently
assumes x is 3D and unflattens it, but FSQCodebook.project_down() receives a
flattened 2D tensor ((batch*steps), dim); update the fake forward to accept x as
2D: read batch_steps = x.shape[0] and dim = x.shape[1], assert
self.output.shape[0] == batch_steps (and optionally self.output.shape[1] == dim)
and return self.output.to(x.device). Change the method referenced in the test
(forward) so its shape contract matches FSQCodebook.project_down().



def test_fsq_code_packing_returns_exact_token_ids():
codebook = FSQCodebook(dim=8, level=3)
digits = torch.tensor(
[[[2, 0, 0, 0, 0, 2, 0, 0], [1, 0, 0, 0, 0, 2, 0, 0]]],
dtype=torch.int64,
)
pre_tanh = torch.where(
digits == 2,
torch.full_like(digits, 20),
torch.where(digits == 1, torch.zeros_like(digits), torch.full_like(digits, -20)),
).to(torch.float32)
codebook.project_down = FakeProjectDown(pre_tanh)

dummy_hidden = torch.zeros(1, 2, 8)
packed = codebook.encode(dummy_hidden)

assert torch.equal(packed, torch.tensor([[488, 487]], dtype=torch.int32))


def test_cuda_float_pow_can_be_non_integer():
if not torch.cuda.is_available():
return

powers = torch.pow(3, torch.arange(8, device="cuda", dtype=torch.float32)).cpu()

# On some CUDA stacks, 3**5 becomes 242.99998 instead of 243.0.
# This test documents the motivation for avoiding float pow in code packing.
assert powers[5].item() <= 243.0
Loading