Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 73 additions & 28 deletions cuFFT/lto_callback_window_1d/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -41,59 +41,104 @@ set(GENCODE_FLAGS_LTO --generate-code arch=compute_${CUDA_LTO_ARCHITECTURE},code

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

add_executable(r2c_c2r_lto_callback_example)
add_executable(r2c_c2r_lto_nvrtc_callback_example)
add_executable(r2c_c2r_legacy_callback_example)
add_executable(r2c_c2r_windowing_lto_callback_example)
add_executable(r2c_c2r_windowing_lto_nvrtc_callback_example)
add_executable(r2c_c2r_windowing_legacy_callback_example)
add_executable(r2c_c2r_padding_lto_callback_example)
add_executable(r2c_c2r_padding_legacy_callback_example)

target_include_directories(r2c_c2r_lto_callback_example
target_include_directories(r2c_c2r_windowing_lto_callback_example
PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
${CMAKE_SOURCE_DIR}/src
${CMAKE_BINARY_DIR})
target_include_directories(r2c_c2r_lto_nvrtc_callback_example
target_include_directories(r2c_c2r_windowing_lto_nvrtc_callback_example
PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
${CMAKE_SOURCE_DIR}/src
${CMAKE_BINARY_DIR})
target_include_directories(r2c_c2r_legacy_callback_example
target_include_directories(r2c_c2r_windowing_legacy_callback_example
PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
${CMAKE_SOURCE_DIR}/src
${CMAKE_BINARY_DIR})
target_include_directories(r2c_c2r_padding_lto_callback_example
PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
${CMAKE_SOURCE_DIR}/src
${CMAKE_BINARY_DIR})
target_include_directories(r2c_c2r_padding_legacy_callback_example
PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
${CMAKE_SOURCE_DIR}/src
${CMAKE_BINARY_DIR})

target_sources(r2c_c2r_windowing_lto_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_windowing_lto_callback_example.cpp
${PROJECT_SOURCE_DIR}/src/common.cpp
${PROJECT_SOURCE_DIR}/src/r2c_c2r_windowing_reference.cu
${CMAKE_BINARY_DIR}/r2c_c2r_windowing_lto_callback_device_fatbin.h)

target_sources(r2c_c2r_windowing_lto_nvrtc_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_windowing_lto_nvrtc_callback_example.cpp
${PROJECT_SOURCE_DIR}/src/common.cpp
${PROJECT_SOURCE_DIR}/src/r2c_c2r_windowing_reference.cu)

target_sources(r2c_c2r_lto_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_lto_callback_example.cpp
target_sources(r2c_c2r_windowing_legacy_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_windowing_legacy_callback_example.cu
${PROJECT_SOURCE_DIR}/src/common.cpp
${PROJECT_SOURCE_DIR}/src/r2c_c2r_reference.cu
${CMAKE_BINARY_DIR}/r2c_c2r_lto_callback_device_fatbin.h)
${PROJECT_SOURCE_DIR}/src/r2c_c2r_windowing_reference.cu)

target_sources(r2c_c2r_lto_nvrtc_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_lto_nvrtc_callback_example.cpp
target_sources(r2c_c2r_padding_lto_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_padding_lto_callback_example.cpp
${PROJECT_SOURCE_DIR}/src/common.cpp
${PROJECT_SOURCE_DIR}/src/r2c_c2r_reference.cu)
${PROJECT_SOURCE_DIR}/src/r2c_c2r_padding_reference.cu
${CMAKE_BINARY_DIR}/r2c_c2r_padding_load_callback_fatbin.h
${CMAKE_BINARY_DIR}/r2c_c2r_padding_store_callback_fatbin.h)

target_sources(r2c_c2r_legacy_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_legacy_callback_example.cu
target_sources(r2c_c2r_padding_legacy_callback_example
PRIVATE ${PROJECT_SOURCE_DIR}/src/r2c_c2r_padding_legacy_callback_example.cu
${PROJECT_SOURCE_DIR}/src/common.cpp
${PROJECT_SOURCE_DIR}/src/r2c_c2r_reference.cu)
${PROJECT_SOURCE_DIR}/src/r2c_c2r_padding_reference.cu)

if(CB_USE_CONSTANT_MEMORY)
target_compile_definitions(r2c_c2r_legacy_callback_example PRIVATE CB_USE_CONSTANT_MEMORY)
target_compile_definitions(r2c_c2r_lto_callback_example PRIVATE CB_USE_CONSTANT_MEMORY)
target_compile_definitions(r2c_c2r_lto_nvrtc_callback_example PRIVATE CB_USE_CONSTANT_MEMORY)
target_compile_definitions(r2c_c2r_windowing_legacy_callback_example PRIVATE CB_USE_CONSTANT_MEMORY)
target_compile_definitions(r2c_c2r_windowing_lto_callback_example PRIVATE CB_USE_CONSTANT_MEMORY)
target_compile_definitions(r2c_c2r_windowing_lto_nvrtc_callback_example PRIVATE CB_USE_CONSTANT_MEMORY)
endif()

target_compile_definitions(r2c_c2r_lto_nvrtc_callback_example PRIVATE CUDA_ARCH=${CUDA_LTO_ARCHITECTURE} CUDA_PATH=${CUDAToolkit_BIN_DIR}/.. -DSOURCE_PATH=${CMAKE_SOURCE_DIR}/src)
target_compile_definitions(r2c_c2r_windowing_lto_nvrtc_callback_example PRIVATE CUDA_ARCH=${CUDA_LTO_ARCHITECTURE} CUDA_PATH=${CUDAToolkit_BIN_DIR}/.. -DSOURCE_PATH=${CMAKE_SOURCE_DIR}/src)

set_target_properties(r2c_c2r_legacy_callback_example PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON
set_target_properties(r2c_c2r_windowing_legacy_callback_example PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON
CUDA_SEPARABLE_COMPILATION ON)

set_target_properties(r2c_c2r_padding_legacy_callback_example PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON
CUDA_SEPARABLE_COMPILATION ON)

message(STATUS "CMAKE_BINARY_DIR: ${CMAKE_BINARY_DIR}")
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/r2c_c2r_lto_callback_device.fatbin ${CMAKE_BINARY_DIR}/r2c_c2r_lto_callback_device_fatbin.h
DEPENDS ${CMAKE_SOURCE_DIR}/src/r2c_c2r_lto_callback_device.cu
COMMAND ${CMAKE_CUDA_COMPILER} --std=c++11 ${GENCODE_FLAGS_LTO} -dc -fatbin ${CMAKE_SOURCE_DIR}/src/r2c_c2r_lto_callback_device.cu -o ${CMAKE_BINARY_DIR}/r2c_c2r_lto_callback_device.fatbin
COMMAND ${CUDAToolkit_BIN_DIR}/bin2c --name window_callback --type longlong ${CMAKE_BINARY_DIR}/r2c_c2r_lto_callback_device.fatbin > ${CMAKE_BINARY_DIR}/r2c_c2r_lto_callback_device_fatbin.h
OUTPUT ${CMAKE_BINARY_DIR}/r2c_c2r_windowing_lto_callback_device.fatbin ${CMAKE_BINARY_DIR}/r2c_c2r_windowing_lto_callback_device_fatbin.h
DEPENDS ${CMAKE_SOURCE_DIR}/src/r2c_c2r_windowing_lto_callback_device.cu
COMMAND ${CMAKE_CUDA_COMPILER} --std=c++11 ${GENCODE_FLAGS_LTO} -dc -fatbin ${CMAKE_SOURCE_DIR}/src/r2c_c2r_windowing_lto_callback_device.cu -o ${CMAKE_BINARY_DIR}/r2c_c2r_windowing_lto_callback_device.fatbin
COMMAND ${CUDAToolkit_BIN_DIR}/bin2c --name window_callback --type longlong ${CMAKE_BINARY_DIR}/r2c_c2r_windowing_lto_callback_device.fatbin > ${CMAKE_BINARY_DIR}/r2c_c2r_windowing_lto_callback_device_fatbin.h
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)

add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/r2c_c2r_padding_load_callback.fatbin ${CMAKE_BINARY_DIR}/r2c_c2r_padding_load_callback_fatbin.h
DEPENDS ${CMAKE_SOURCE_DIR}/src/r2c_c2r_padding_lto_load_callback_device.cu
COMMAND ${CMAKE_CUDA_COMPILER} --std=c++11 ${GENCODE_FLAGS_LTO} -dc -fatbin ${CMAKE_SOURCE_DIR}/src/r2c_c2r_padding_lto_load_callback_device.cu -o ${CMAKE_BINARY_DIR}/r2c_c2r_padding_load_callback.fatbin
COMMAND ${CUDAToolkit_BIN_DIR}/bin2c --name padding_load_callback --type longlong ${CMAKE_BINARY_DIR}/r2c_c2r_padding_load_callback.fatbin > ${CMAKE_BINARY_DIR}/r2c_c2r_padding_load_callback_fatbin.h
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Compiling padding LTO load callback device code"
)

add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/r2c_c2r_padding_store_callback.fatbin ${CMAKE_BINARY_DIR}/r2c_c2r_padding_store_callback_fatbin.h
DEPENDS ${CMAKE_SOURCE_DIR}/src/r2c_c2r_padding_lto_store_callback_device.cu
COMMAND ${CMAKE_CUDA_COMPILER} --std=c++11 ${GENCODE_FLAGS_LTO} -dc -fatbin ${CMAKE_SOURCE_DIR}/src/r2c_c2r_padding_lto_store_callback_device.cu -o ${CMAKE_BINARY_DIR}/r2c_c2r_padding_store_callback.fatbin
COMMAND ${CUDAToolkit_BIN_DIR}/bin2c --name padding_store_callback --type longlong ${CMAKE_BINARY_DIR}/r2c_c2r_padding_store_callback.fatbin > ${CMAKE_BINARY_DIR}/r2c_c2r_padding_store_callback_fatbin.h
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Compiling padding LTO store callback device code"
)

target_link_libraries(r2c_c2r_lto_callback_example PRIVATE CUDA::cufft CUDA::cudart CUDA::nvJitLink)
target_link_libraries(r2c_c2r_lto_nvrtc_callback_example PRIVATE CUDA::cufft CUDA::cudart CUDA::nvJitLink CUDA::nvrtc)
target_link_libraries(r2c_c2r_legacy_callback_example PRIVATE CUDA::cufft_static CUDA::cudart CUDA::culibos)
target_link_libraries(r2c_c2r_windowing_lto_callback_example PRIVATE CUDA::cufft CUDA::cudart CUDA::nvJitLink)
target_link_libraries(r2c_c2r_windowing_lto_nvrtc_callback_example PRIVATE CUDA::cufft CUDA::cudart CUDA::nvJitLink CUDA::nvrtc)
target_link_libraries(r2c_c2r_windowing_legacy_callback_example PRIVATE CUDA::cufft_static CUDA::cudart CUDA::culibos)
target_link_libraries(r2c_c2r_padding_lto_callback_example PRIVATE CUDA::cufft CUDA::cudart CUDA::nvJitLink)
target_link_libraries(r2c_c2r_padding_legacy_callback_example PRIVATE CUDA::cufft_static CUDA::cudart CUDA::culibos)
60 changes: 47 additions & 13 deletions cuFFT/lto_callback_window_1d/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,33 @@

## Description

In this example, we apply a low-pass filter to a batch of signals in the frequency domain. Specifically, the sample code creates a forward (R2C, Real-To-Complex) plan and an inverse (C2R, Complex-To-Real) plan. The low-pass filter is implemented via a load callback in the inverse plan, where values under a certain threshold (specified by the user as a window dimension) are loaded normally, and every other value is zeroed.
This directory contains examples demonstrating the use of cuFFT callbacks with both legacy and LTO (Link-Time Optimization) approaches. Two different use cases are provided: windowing and zero-padding.

Three versions of the code are provided:
* `r2c_c2r_lto_callback_example.cpp` contains the sample code using a load callback with LTO to compute the window function. The LTO callback is compiled offline using nvcc.
* `r2c_c2r_lto_nvrtc_callback_example.cpp` contains the sample code using a load callback with LTO to compute the window function. The LTO callback is compiled at runtime using NVRTC.
* `r2c_c2r_legacy_callback_example.cu` contains the sample code using a 'legacy' (non-LTO) load callback to compute the window function. The callback does not use LTO and requires separate device linking against the cuFFT static library.
### Windowing Examples

In the windowing examples, we apply a low-pass filter to a batch of signals in the frequency domain. Specifically, the sample code creates a forward (R2C, Real-To-Complex) plan and an inverse (C2R, Complex-To-Real) plan. The low-pass filter is implemented via a **load callback** in the inverse plan, where values under a certain threshold (specified by the user as a window dimension) are loaded normally, and every other value is zeroed.

Three versions of the windowing code are provided:
* `r2c_c2r_windowing_lto_callback_example.cpp` contains the sample code using a load callback with LTO to compute the window function. The LTO callback is compiled offline using nvcc.
* `r2c_c2r_windowing_lto_nvrtc_callback_example.cpp` contains the sample code using a load callback with LTO to compute the window function. The LTO callback is compiled at runtime using NVRTC.
* `r2c_c2r_windowing_legacy_callback_example.cu` contains the sample code using a 'legacy' (non-LTO) load callback to compute the window function. The callback does not use LTO and requires separate device linking against the cuFFT static library.

### Padding Examples

In the padding examples, we demonstrate the use of **both load and store callbacks** to implement zero-padding. The load callback is applied on the forward (R2C) plan to pad the input with zeros, and the store callback is applied on the inverse (C2R) plan to truncate the output to the original size and normalize.

Two versions of the padding code are provided:
* `r2c_c2r_padding_lto_callback_example.cpp` contains the sample code using LTO callbacks to implement zero-padding. The LTO callback is compiled offline using nvcc.
* `r2c_c2r_padding_legacy_callback_example.cu` contains the sample code using legacy callbacks to implement zero-padding. The callback does not use LTO and requires separate device linking against the cuFFT static library.

Other source files included:
* `r2c_c2r_lto_callback_device.cu` contains the callback device function used in the LTO and LTO + NVRTC examples.
* `r2c_c2r_reference.cu` contains the code used as reference for the samples. The reference computes the window function using a separate kernel, rather than callbacks.
* `r2c_c2r_windowing_lto_callback_device.cu` contains the callback device function used in the windowing LTO and LTO + NVRTC examples.
* `r2c_c2r_padding_lto_load_callback_device.cu` contains the load callback device function used in the padding LTO example.
* `r2c_c2r_padding_lto_store_callback_device.cu` contains the store callback device function used in the padding LTO example.
* `r2c_c2r_windowing_reference.cu` contains the code used as reference for the windowing samples. The reference computes the window function using a separate kernel, rather than callbacks.
* `r2c_c2r_padding_reference.cu` contains the code used as reference for the padding samples. The reference computes the zero-padding using separate kernels, rather than callbacks.
* `nvrtc_helper.h` contains the required code to do runtime compilation of the LTO callback using NVRTC.
* `common.cpp` and `common.h` include some helper functions, like methods to perform the initialization of the signal in the time domain..
* `common.cpp` and `common.h` include some helper functions, like methods to perform the initialization of the signal in the time domain.

## Supported SM Architectures

Expand Down Expand Up @@ -62,23 +77,42 @@ make
```

## Running the examples

### Windowing examples
```
./bin/r2c_c2r_windowing_lto_callback_example
./bin/r2c_c2r_windowing_lto_nvrtc_callback_example
./bin/r2c_c2r_windowing_legacy_callback_example
```

### Padding examples
```
./bin/r2c_c2r_lto_callback_example
./bin/r2c_c2r_lto_nvrtc_callback_example
./bin/r2c_c2r_callback_example
./bin/r2c_c2r_padding_lto_callback_example
./bin/r2c_c2r_padding_legacy_callback_example
```

Sample of output
Sample of output (windowing)

```
$ ./bin/r2c_c2r_lto_callback_example
$ ./bin/r2c_c2r_windowing_lto_callback_example
Transforming signal cufftExecR2C
Transforming signal cufftExecC2R
Transforming reference cufftExecR2C
Transforming reference cufftExecC2R
L2 error: 0.000000e+00
```

Sample of output (padding)

```
$ ./bin/r2c_c2r_padding_lto_callback_example
Transforming signal cufftExecR2C with padding load callback
Transforming signal cufftExecC2R with padding store callback
Transforming reference cufftExecR2C
Transforming reference cufftExecC2R
L2 error: 0.000000e+00
```

## Troubleshooting
### I am getting an error with the message: "error: ‘cufftXtSetJITCallback’ was not declared in this scope".
Please, make sure you are using cuFFT from CUDA Toolkit 12.6 Update 2 or newer. LTO callbacks are not available on older versions of cuFFT.
Expand Down
11 changes: 11 additions & 0 deletions cuFFT/lto_callback_window_1d/src/callback_params.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,23 @@ struct cb_params {
unsigned signal_size;
};

// Callback parameters structure for padding
struct PaddingCallbackParams {
unsigned int signal_size;
unsigned int padded_signal_size;
};

// Problem input parameters
constexpr unsigned batches = 830;
constexpr unsigned signal_size = 328;
constexpr unsigned window_size = 32;
constexpr unsigned complex_signal_size = signal_size / 2 + 1;

// Padding example parameters - use single batch for simplicity
constexpr unsigned padding_batches = 1;
constexpr unsigned padded_signal_size = 2 * signal_size;
constexpr unsigned padding_complex_signal_size = padded_signal_size / 2 + 1;

// Precision threshold
constexpr float threshold = 1e-6;

Expand Down
2 changes: 2 additions & 0 deletions cuFFT/lto_callback_window_1d/src/common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ constexpr float sampling_dt = 1e-3;

// Initialize the input signal as a composite of sine waves
// with random amplitudes and frequencies
// Uses complex buffer layout (stride = 2 * complex_signal_size)
void init_input_signals(unsigned batches, unsigned signal_size, float* signals) {
std::mt19937 e2(0);

Expand Down Expand Up @@ -60,4 +61,5 @@ void init_input_signals(unsigned batches, unsigned signal_size, float* signals)
}
}


#endif // _COMMON__CPP_
19 changes: 18 additions & 1 deletion cuFFT/lto_callback_window_1d/src/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ inline int checkErrors(cufftResult error, int line_number) {
#define CHECK_ERROR(error) checkErrors(error, __LINE__)

template<typename T>
double compute_error(T* ref, T* out, unsigned batches, unsigned signal_size){
double compute_error_windowing(T* ref, T* out, unsigned batches, unsigned signal_size){
double squared_diff = 0;
double squared_norm = 0;
const unsigned batch_offset = 2 * (signal_size / 2 + 1);
Expand All @@ -63,6 +63,23 @@ double compute_error(T* ref, T* out, unsigned batches, unsigned signal_size){
return std::sqrt(squared_diff / squared_norm);
}

// Compute error for real buffers (padding examples)
template<typename T>
double compute_error_padding(T* ref, T* out, unsigned batches, unsigned signal_size){
double squared_diff = 0;
double squared_norm = 0;
const unsigned batch_offset = signal_size;

for (unsigned b = 0; b < batches; b++) {
for (unsigned i = 0; i < signal_size; i++) {
unsigned ref_idx = b * batch_offset + i;
squared_diff += std::norm(ref[ref_idx] - out[ref_idx]);
squared_norm += std::norm(ref[ref_idx]);
}
}
return std::sqrt(squared_diff / squared_norm);
}

// Initialize input signals
void init_input_signals(unsigned batches, unsigned signal_size, float* signals);
#endif // _COMMON__H_
Loading