Fix ICE in v0 symbol mangling for deeply nested generic types

compiler/rustc_symbol_mangling/src/lib.rs

@@ -307,7 +307,21 @@ fn compute_symbol_name<'tcx>(
                     mangled_name
                 }
             }
-            SymbolManglingVersion::V0 => v0::mangle(tcx, instance, instantiating_crate, false),
+            SymbolManglingVersion::V0 => {
+                let mangled = v0::mangle(tcx, instance, instantiating_crate, false);
+                // v0 mangling can produce symbols that are too deeply nested for
+                // `rustc_demangle` to handle. This happens with pathologically
+                // nested generic types: the v0 format uses backreferences to encode
+                // repeated path components, and following those backreferences
+                // during demangling recurses, exceeding `rustc_demangle`'s internal
+                // recursion limit. Fall back to hashed mangling in that case so we
+                // always emit a demanglable symbol.
+                if rustc_demangle::try_demangle(&mangled).is_ok() {
+                    mangled
+                } else {
+                    hashed::mangle(tcx, instance, instantiating_crate, || mangled)
+                }
+            }
             SymbolManglingVersion::Hashed => {
                 hashed::mangle(tcx, instance, instantiating_crate, || {
                     v0::mangle(tcx, instance, instantiating_crate, false)

tests/ui/symbol-names/deeply-nested-generic-aliases.rs

@@ -0,0 +1,24 @@
+//@ build-pass
+//@ compile-flags: -C symbol-mangling-version=v0
+
+struct D;
+
+type O0<T> = Option<T>;
+type O1<T> = O0<T>;
+type O2<T> = O1<O1<T>>;
+type O3<T> = O2<O2<T>>;
+type O4<T> = O3<O3<T>>;
+type O5<T> = O4<O4<T>>;
+type O6<T> = O5<O5<T>>;
+type O7<T> = O6<O6<T>>;
+type O8<T> = O7<O7<T>>;
+type Q510<T> = O8<O7<O6<O5<O4<O3<O2<T>>>>>>>;
+
+fn f<T>() {}
+fn describe<T>() {
+    f::<Q510<T>>()
+}
+
+fn main() {
+    describe::<D>();
+}