C++ PATCH for c++/44175 (segv with recursive decltype in template)

commit 0d7f1941a19ce5133c30f8a917379afc6bb31b82
Author: Jason Merrill <jason@redhat.com>
Date:   Tue May 31 17:11:31 2011 -0400

    	PR c++/44175
    	* pt.c (template_args_equal): Handle one arg being NULL_TREE.
    	(deduction_tsubst_fntype): Handle excessive non-infinite recursion.

diff --git a/gcc/cp/pt.c b/gcc/cp/pt.c
index ae3d83d..c1bee3e 100644
--- a/gcc/cp/pt.c
+++ b/gcc/cp/pt.c
@@ -6476,6 +6476,8 @@  template_args_equal (tree ot, tree nt)
 {
   if (nt == ot)
     return 1;
+  if (nt == NULL_TREE || ot == NULL_TREE)
+    return false;
 
   if (TREE_CODE (nt) == TREE_VEC)
     /* For member templates */
@@ -13598,7 +13600,14 @@  static GTY((param_is (spec_entry))) htab_t current_deduction_htab;
 /* In C++0x, it's possible to have a function template whose type depends
    on itself recursively.  This is most obvious with decltype, but can also
    occur with enumeration scope (c++/48969).  So we need to catch infinite
-   recursion and reject the substitution at deduction time.
+   recursion and reject the substitution at deduction time; this function
+   will return error_mark_node for any repeated substitution.
+
+   This also catches excessive recursion such as when f<N> depends on
+   f<N-1> across all integers, and returns error_mark_node for all the
+   substitutions back up to the initial one.
+
+   This is, of course, not reentrant.
 
    Use of a VEC here is O(n^2) in the depth of function template argument
    deduction substitution, but using a hash table creates a lot of constant
@@ -13611,6 +13620,8 @@  static GTY((param_is (spec_entry))) htab_t current_deduction_htab;
 static tree
 deduction_tsubst_fntype (tree fn, tree targs)
 {
+  static bool excessive_deduction_depth;
+
   unsigned i;
   spec_entry **slot;
   spec_entry *p;
@@ -13656,6 +13667,14 @@  deduction_tsubst_fntype (tree fn, tree targs)
   /* If we've created a hash table, look there.  */
   if (current_deduction_htab)
     {
+      if (htab_elements (current_deduction_htab)
+	  > (unsigned) max_tinst_depth)
+	{
+	  /* Trying to recurse across all integers or some such.  */
+	  excessive_deduction_depth = true;
+	  return error_mark_node;
+	}
+
       hash = hash_specialization (&elt);
       slot = (spec_entry **)
 	htab_find_slot_with_hash (current_deduction_htab, &elt, hash, INSERT);
@@ -13701,6 +13720,13 @@  deduction_tsubst_fntype (tree fn, tree targs)
 	r = error_mark_node;
       VEC_pop (spec_entry, current_deduction_vec);
     }
+  if (excessive_deduction_depth)
+    {
+      r = error_mark_node;
+      if (htab_elements (current_deduction_htab) == 0)
+	/* Reset once we're all the way out.  */
+	excessive_deduction_depth = false;
+    }
   return r;
 }
 
diff --git a/gcc/testsuite/g++.dg/cpp0x/decltype28.C b/gcc/testsuite/g++.dg/cpp0x/decltype28.C
new file mode 100644
index 0000000..0ab8932
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/decltype28.C
@@ -0,0 +1,16 @@ 
+// PR c++/44175
+// { dg-options -std=c++0x }
+
+template <bool, class T> struct enable_if { };
+template <class T> struct enable_if <true, T> { typedef T type; };
+
+template <class F, int N>
+void ft (F f, typename enable_if<N!=0, int>::type) {}
+
+template< class F, int N >
+decltype(ft<F, N-1> (F(), 0))
+ft (F f, typename enable_if<N==0, int>::type) {}
+
+int main() {
+  ft<struct a*, 2> (0, 0);
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/decltype29.C b/gcc/testsuite/g++.dg/cpp0x/decltype29.C
new file mode 100644
index 0000000..1dd5a5f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/decltype29.C
@@ -0,0 +1,19 @@ 
+// PR c++/44175
+// { dg-options -std=c++0x }
+
+template <bool, class T> struct enable_if { };
+template <class T> struct enable_if <true, T> { typedef T type; };
+
+template <int x>
+typename enable_if<x==0,int>::type
+ft() {}
+
+template<class F, int N>
+decltype (ft<F> (F()))
+ft() {}
+
+int main() {
+    ft<struct a*, 0>();		// { dg-error "no match" }
+}
+
+// { dg-prune-output "note" }