From 283f7bd879860b395f2c2a3719b3d146b4d754f7 Mon Sep 17 00:00:00 2001 From: Apurba Bose Date: Thu, 23 Jul 2026 00:48:25 +0000 Subject: [PATCH] fix(distributed): auto-resolve NCCL group + unbounded SymInt in extract_var_range_info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TRTEngine.cpp — bind_nccl_comm auto-resolve: PyTorch >= 2.14 throws c10::Error when resolve_process_group() is called with a name that doesn't exist, instead of returning nullptr. The sequential probe loop (0..19) would crash at i=1 on a fresh load-only process that only has WORLD registered as "0". Replace the bare call with a try_resolve lambda that normalises the throw to nullptr so the scan loop stays exception-free. dynamo/utils.py — extract_var_range_info: bound_sympy() returns sympy.oo for unbounded composite exprs (e.g. -s56), which is distinct from int_oo. int(sympy.oo) raises AttributeError ('Infinity' object has no attribute '_mpf_'). Add _bound_to_int_or_none helper that guards both int_oo and sympy.oo cases and catches TypeError/OverflowError/AttributeError so unbounded dims fall back to 1. --- core/runtime/TRTEngine.cpp | 19 +++++++++++++++---- py/torch_tensorrt/dynamo/utils.py | 22 ++++++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/core/runtime/TRTEngine.cpp b/core/runtime/TRTEngine.cpp index 76523012e0..4f44c36402 100644 --- a/core/runtime/TRTEngine.cpp +++ b/core/runtime/TRTEngine.cpp @@ -579,12 +579,23 @@ bool TRTEngine::bind_nccl_comm() { // auto-resolve when there is exactly one — if there are several (TP+DP, // Megatron 4-D parallelism, etc.) we cannot know which group this engine // belongs to and the caller must pin it explicitly. + // PyTorch >= 2.13 throws c10::Error (instead of returning nullptr) when + // the requested group name doesn't exist. try_resolve normalises this to a + // nullptr return so the scan loop stays exception-free. + auto try_resolve = [](const std::string& name) -> c10::intrusive_ptr { + try { + return c10d::resolve_process_group(name); + } catch (const c10::Error&) { + return nullptr; + } + }; std::vector nccl_groups; for (int i = 0; i < 20; ++i) { - auto candidate = std::to_string(i); - auto probe = c10d::resolve_process_group(candidate); - if (probe != nullptr && probe->getBackendType() == c10d::ProcessGroup::BackendType::NCCL) { - nccl_groups.push_back(candidate); + auto pg = try_resolve(std::to_string(i)); + if (!pg) + break; + if (pg->getBackendType() == c10d::ProcessGroup::BackendType::NCCL) { + nccl_groups.push_back(std::to_string(i)); } } diff --git a/py/torch_tensorrt/dynamo/utils.py b/py/torch_tensorrt/dynamo/utils.py index 33595f4709..27d9f51ed7 100644 --- a/py/torch_tensorrt/dynamo/utils.py +++ b/py/torch_tensorrt/dynamo/utils.py @@ -435,10 +435,24 @@ def extract_var_range_info(symbolic_integer: torch.SymInt) -> Dict[str, Optional or expr.xreplace(var_to_val_map) ) assert var_range, var_val - min_val, max_val = ( - int(var_range.lower), - int(var_range.upper) if var_range.upper != int_oo else None, - ) + + # ``var_to_range`` returns ``int_oo`` for unbounded; ``bound_sympy`` (used + # for composite exprs like ``s0+s1``) returns ``sympy.oo`` instead. They + # are distinct objects -- check both, else ``int(sympy.oo)`` raises. + def _bound_to_int_or_none(value: Any) -> Optional[int]: + if value is int_oo or value is -int_oo: + return None + if value == sympy.oo or value == -sympy.oo: + return None + try: + return int(value) + except (TypeError, OverflowError, AttributeError): + return None + + min_val_opt = _bound_to_int_or_none(var_range.lower) + max_val = _bound_to_int_or_none(var_range.upper) + # Unbounded lower shouldn't happen for tensor dims; fall back to 1. + min_val = min_val_opt if min_val_opt is not None else 1 # Torchdynamo 0/1 specialization outlier min_val = 1 if min_val == 2 else min_val