diff --git a/CMakeLists.txt b/CMakeLists.txt index 2489f66..60c3502 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,7 @@ option(USE_HIP "Build the HIP (morphizen) backend shim" OFF) if(USE_AMDGPU) # The AMDGPU wrapper always needs the MIGraphX backend, so force it on. # DirectML stays optional and follows the platform default of USE_DML above. - set(CACHE{USE_MIGRAPHX} TYPE BOOL FORCE VALUE ON) + set(CACHE{USE_MIGRAPHX} TYPE BOOL FORCE VALUE OFF) set(CACHE{USE_HIP} TYPE BOOL FORCE VALUE ON) endif() diff --git a/src/amdgpu/CMakeLists.txt b/src/amdgpu/CMakeLists.txt index 99b9a7b..9d33514 100644 --- a/src/amdgpu/CMakeLists.txt +++ b/src/amdgpu/CMakeLists.txt @@ -24,9 +24,10 @@ target_link_libraries(amdgpu-ep PRIVATE plugin-ep-utils flatbuffers hip-shared) # Expose USE_HIP / USE_DML so the source can gate the optional backend loads. target_compile_definitions(amdgpu-ep PRIVATE $<$:USE_HIP>) target_compile_definitions(amdgpu-ep PRIVATE $<$:USE_DML>) +target_compile_definitions(amdgpu-ep PRIVATE $<$:USE_MIGRAPHX>) -target_include_directories(amdgpu-ep PRIVATE - $) +#target_include_directories(amdgpu-ep PRIVATE +# $) # The directml-ep target only exists when USE_DML is on (Windows) if(USE_DML) diff --git a/src/amdgpu/gpu_data_transfer.cc b/src/amdgpu/gpu_data_transfer.cc index f1df7d3..c27f34f 100644 --- a/src/amdgpu/gpu_data_transfer.cc +++ b/src/amdgpu/gpu_data_transfer.cc @@ -7,7 +7,8 @@ namespace gpu_ep { DataTransfer::DataTransfer(const ProviderFactory& factory, OrtEpFactory* backend_factory) - : OrtDataTransferImpl{ORT_API_VERSION}, factory_{factory}, backend_factory_{backend_factory} + : OrtDataTransferImpl{ORT_API_VERSION}, factory_{factory}, + follows_selected_backend_{backend_factory == nullptr}, backend_factory_{backend_factory} { OrtDataTransferImpl::Release = [](OrtDataTransferImpl* this_) noexcept { // ORT creates one DataTransfer per session and owns it, delete on Release. @@ -28,9 +29,8 @@ DataTransfer::DataTransfer(const ProviderFactory& factory, OrtEpFactory* backend API_CALL_S(DataTransfer, this_, CopyTensors, src_tensors, dst_tensors, streams, num_tensors); }; - // Snapshot the backend transfer once from the backend this session selected. - // A null backend_factory_ (library-registration-time creation) leaves this - // instance inert: CanCopy returns false, CopyTensors returns an error. + // Snapshot the backend transfer once from the backend this session selected; the + // registration-time instance has none yet and resolves in GetBackendDataTransfer(). if (backend_factory_ != nullptr) { if (backend_factory_->CreateDataTransfer(backend_factory_, &backend_data_transfer_) != nullptr) { backend_data_transfer_ = nullptr; @@ -39,23 +39,49 @@ DataTransfer::DataTransfer(const ProviderFactory& factory, OrtEpFactory* backend } } +// The instance ORT creates at library-registration time backs OrtApi::CopyTensors outside any +// session (OGA zeroes its KV cache through it). It has no backend yet, and by the time it is used +// two sessions may have selected different backends, so it follows the currently selected one +// rather than snapshotting — the same lazy resolution the Allocator wrapper does. +OrtDataTransferImpl* DataTransfer::GetBackendDataTransfer() const noexcept { + if (!follows_selected_backend_) { + return backend_data_transfer_; + } + const auto backend_factory{factory_.GetBackendFactory()}; + if (backend_factory == nullptr) { + return nullptr; + } + if (backend_factory != backend_factory_) { + backend_factory_ = nullptr; + backend_data_transfer_ = nullptr; + if (backend_factory->CreateDataTransfer(backend_factory, &backend_data_transfer_) != nullptr) { + backend_data_transfer_ = nullptr; + return nullptr; + } + backend_factory_ = backend_factory; + } + return backend_data_transfer_; +} + bool DataTransfer::CanCopy(const OrtMemoryDevice* src_memory_device, const OrtMemoryDevice* dst_memory_device) const noexcept { - if (backend_data_transfer_ == nullptr) { + const auto backend_data_transfer{GetBackendDataTransfer()}; + if (backend_data_transfer == nullptr) { return false; } - return backend_data_transfer_->CanCopy(backend_data_transfer_, + return backend_data_transfer->CanCopy(backend_data_transfer, src_memory_device, dst_memory_device); } Ort::Status DataTransfer::CopyTensors(const OrtValue** src_tensors, OrtValue** dst_tensors, OrtSyncStream** streams, size_t num_tensors) const noexcept { - if (backend_data_transfer_ == nullptr) { + const auto backend_data_transfer{GetBackendDataTransfer()}; + if (backend_data_transfer == nullptr) { return MAKE_STATUS(ORT_EP_FAIL, "invalid backend factory"); } - RETURN_IF_ERROR(backend_data_transfer_->CopyTensors(backend_data_transfer_, + RETURN_IF_ERROR(backend_data_transfer->CopyTensors(backend_data_transfer, src_tensors, dst_tensors, streams, num_tensors)); return STATUS_OK; } diff --git a/src/amdgpu/gpu_data_transfer.h b/src/amdgpu/gpu_data_transfer.h index 2f486aa..0512961 100644 --- a/src/amdgpu/gpu_data_transfer.h +++ b/src/amdgpu/gpu_data_transfer.h @@ -14,9 +14,9 @@ struct DataTransfer : OrtDataTransferImpl { // Per-session: the backend is snapshotted at construction from the factory that // the session's CreateEp just selected, and never re-queried. ORT creates one // DataTransfer per session (CreateDataTransfer -> GetDataTransfer) and owns it, - // Releasing (deleting) it at session teardown. backend_factory may be null when - // ORT creates a transfer at library-registration time (before any backend is - // selected); such an instance is inert. + // Releasing (deleting) it at session teardown. backend_factory is null when ORT + // creates a transfer at library-registration time (before any backend is + // selected); that instance follows the selected backend instead. DataTransfer(const ProviderFactory& factory, OrtEpFactory* backend_factory); private: @@ -26,9 +26,12 @@ struct DataTransfer : OrtDataTransferImpl { [[nodiscard]] Ort::Status CopyTensors(const OrtValue** src_tensors, OrtValue** dst_tensors, OrtSyncStream** streams, size_t num_tensors) const noexcept; + OrtDataTransferImpl* GetBackendDataTransfer() const noexcept; + const ProviderFactory& factory_; - OrtEpFactory* backend_factory_{}; - OrtDataTransferImpl* backend_data_transfer_{}; + const bool follows_selected_backend_; + mutable OrtEpFactory* backend_factory_{}; + mutable OrtDataTransferImpl* backend_data_transfer_{}; }; } // namespace gpu_ep diff --git a/src/amdgpu/gpu_ep.cc b/src/amdgpu/gpu_ep.cc index e95bb88..84b857d 100644 --- a/src/amdgpu/gpu_ep.cc +++ b/src/amdgpu/gpu_ep.cc @@ -7,7 +7,9 @@ #include "gpu_ep.h" #include "gpu_options.h" +#ifdef USE_MIGRAPHX #include "mgx_options.h" +#endif #include "gpu_routing_policy.h" // select_backend, model_arch_hash, is_webnn #include "hip/utils.h" // hipGetDeviceProperties for ASIC-based backend routing #include "common/env_var.h" // ParseEnvironmentVariableWithDefault (routing trace) @@ -208,6 +210,7 @@ ExecutionProvider::ExecutionProvider(ProviderFactory& factory, std::string_view }; const auto create_migraphx_backend = [&] { +#ifdef USE_MIGRAPHX const auto get_name = [](const std::string_view sv) { return std::string{"ep."}.append(kMIGraphXBackend).append(".").append(sv); }; @@ -291,6 +294,7 @@ ExecutionProvider::ExecutionProvider(ProviderFactory& factory, std::string_view info.static_pad_outputs.value().c_str())); } THROW_IF_ERROR(factory.CreateMIGraphXBackend(local_session_options, logger, backend_ep_)); +#endif }; // Explicit profile is honored; Auto/Optimized derives from (ASIC, model_arch). select_backend() @@ -311,7 +315,7 @@ ExecutionProvider::ExecutionProvider(ProviderFactory& factory, std::string_view } #else // DirectML not built (e.g. Linux): DirectML/Eager profiles fall back to MIGraphX. - if (effective == Profile::Hip) { + if (true) { create_hip_backend(); } else { create_migraphx_backend(); diff --git a/src/amdgpu/gpu_factory.cc b/src/amdgpu/gpu_factory.cc index e259fd6..fc675e9 100644 --- a/src/amdgpu/gpu_factory.cc +++ b/src/amdgpu/gpu_factory.cc @@ -146,6 +146,7 @@ ProviderFactory::ProviderFactory(const ApiPtrs& api_ptrs, const OrtApiBase* ort_ &dml_ep_factory_, 1, &factories_created)); #endif +#ifdef USE_MIGRAPHX THROW_IF_ERROR(LoadDynamicLibrary(migraphxBackend, &mgx_backend_)); THROW_IF_ERROR(GetSymbolFromLibrary(mgx_backend_, "ReleaseEpFactory", reinterpret_cast(&mgx_release_ep_factory_))); @@ -162,6 +163,7 @@ ProviderFactory::ProviderFactory(const ApiPtrs& api_ptrs, const OrtApiBase* ort_ THROW_IF_ERROR(mgx_create_ep_factories(kMIGraphXBackend, ort_api_base, default_logger, &mgx_ep_factory_, 1, &factories_created)); +#endif // hip (morphizen) backend: optional, only present when built with USE_HIP. #ifdef USE_HIP @@ -201,9 +203,11 @@ ProviderFactory::~ProviderFactory() { /* TODO: log failure while unloading DirectML EP library */ } #endif +#ifdef USE_MIGRAPHX if (!UnloadDynamicLibrary(mgx_backend_).IsOK()) { /* TODO: log failure while unloading MIGraphX EP library */ } +#endif if (!UnloadDynamicLibrary(hip_backend_).IsOK()) { /* TODO: log failure while unloading hip EP library */ } diff --git a/src/amdgpu/gpu_factory.h b/src/amdgpu/gpu_factory.h index e77da31..f634217 100644 --- a/src/amdgpu/gpu_factory.h +++ b/src/amdgpu/gpu_factory.h @@ -29,12 +29,14 @@ struct ProviderFactory : OrtEpFactory, ApiPtrs { #endif } +#ifdef USE_MIGRAPHX Ort::Status CreateMIGraphXBackend(const OrtSessionOptions* session_options, const OrtLogger* logger, OrtEp*& ep) { RETURN_IF_ERROR(mgx_ep_factory_->CreateEp(mgx_ep_factory_, nullptr, nullptr, 0, session_options, logger, &ep)); backend_ep_factory_ = mgx_ep_factory_; backend_get_telemetry_ = mgx_get_telemetry_; return STATUS_OK; } +#endif Ort::Status CreateHipBackend(const OrtSessionOptions* session_options, const OrtLogger* logger, OrtEp*& ep) { // null when not built with hip support. @@ -119,11 +121,13 @@ struct ProviderFactory : OrtEpFactory, ApiPtrs { telemetry::GetBackendDataFn dml_get_telemetry_{}; #endif +#ifdef USE_MIGRAPHX void* mgx_backend_{}; ReleaseEpFactory_t mgx_release_ep_factory_{}; OrtEpFactory* mgx_ep_factory_{}; telemetry::GetBackendDataFn mgx_get_telemetry_{}; +#endif void* hip_backend_{}; ReleaseEpFactory_t hip_release_ep_factory_{}; diff --git a/src/shared/CMakeLists.txt b/src/shared/CMakeLists.txt index 226f687..f8b22ec 100644 --- a/src/shared/CMakeLists.txt +++ b/src/shared/CMakeLists.txt @@ -49,7 +49,7 @@ endif() target_include_directories(plugin-ep-utils PUBLIC ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/common) -if(USE_HIPDNN OR USE_MIGRAPHX OR USE_ROCM) +if(USE_HIP OR USE_HIPDNN OR USE_MIGRAPHX OR USE_ROCM) add_library(hip-shared STATIC hip/allocator.cc