Report heap memory use into -Q output

Hi,
this patch adds heap memory use report which with -Q is now output
during WPA stream in and after IPA passes.  This is useful to catch
inordinary large memory use of a given pass.

Bootstrapped/regtested x86_64-linux, OK?
	* config.in: Regenerate.
	* configure: Regenerate.
	* configure.ac: Check for mallinfo.
	* ggc-common.c: Include malloc.h if available;
	include options.h
	(report_heap_memory_use): New functoin.
	* ggc-page.c (ggc_grow): Do not print "start".
	* ggc.h (report_heap_memory_use): Declare.
	* pases.c (execute_one_pass): Report memory after IPA passes.
	(ipa_read_summaries_1): Likewise.
	(ipa_read_optimization_summaries_1): Likewise.

	* lto/lto-common.c (read_cgraph_and_symbols): Improve -Q reporting.
	* lto.c (lto_wpa_write_files): Likewise.

On Thu, 24 Oct 2019, Jan Hubicka wrote:

> Hi,
> this patch adds heap memory use report which with -Q is now output
> during WPA stream in and after IPA passes.  This is useful to catch
> inordinary large memory use of a given pass.

Looks reasonable but please let others some time to comment on
portability issues (you only check whether the symbol is defined,
not whether the API you use is provided).

Richard.

> Bootstrapped/regtested x86_64-linux, OK?
> 	* config.in: Regenerate.
> 	* configure: Regenerate.
> 	* configure.ac: Check for mallinfo.
> 	* ggc-common.c: Include malloc.h if available;
> 	include options.h
> 	(report_heap_memory_use): New functoin.
> 	* ggc-page.c (ggc_grow): Do not print "start".
> 	* ggc.h (report_heap_memory_use): Declare.
> 	* pases.c (execute_one_pass): Report memory after IPA passes.
> 	(ipa_read_summaries_1): Likewise.
> 	(ipa_read_optimization_summaries_1): Likewise.
> 
> 	* lto/lto-common.c (read_cgraph_and_symbols): Improve -Q reporting.
> 	* lto.c (lto_wpa_write_files): Likewise.
> 	
> Index: configure.ac
> ===================================================================
> --- configure.ac	(revision 277366)
> +++ configure.ac	(working copy)
> @@ -1359,7 +1359,7 @@ define(gcc_UNLOCKED_FUNCS, clearerr_unlo
>  AC_CHECK_FUNCS(times clock kill getrlimit setrlimit atoq \
>  	popen sysconf strsignal getrusage nl_langinfo \
>  	gettimeofday mbstowcs wcswidth mmap setlocale \
> -	gcc_UNLOCKED_FUNCS madvise)
> +	gcc_UNLOCKED_FUNCS madvise mallinfo)
>  
>  if test x$ac_cv_func_mbstowcs = xyes; then
>    AC_CACHE_CHECK(whether mbstowcs works, gcc_cv_func_mbstowcs_works,
> @@ -1439,6 +1439,14 @@ gcc_AC_CHECK_DECLS(getrlimit setrlimit g
>  #endif
>  ])
>  
> +gcc_AC_CHECK_DECLS(mallinfo, , ,[
> +#include "ansidecl.h"
> +#include "system.h"
> +#ifdef HAVE_MALLOC_H
> +#include <malloc.h>
> +#endif
> +])
> +
>  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
>  #include "ansidecl.h"
>  #include "system.h"
> Index: ggc-common.c
> ===================================================================
> --- ggc-common.c	(revision 277366)
> +++ ggc-common.c	(working copy)
> @@ -21,6 +21,9 @@ along with GCC; see the file COPYING3.
>     any particular GC implementation.  */
>  
>  #include "config.h"
> +#ifdef HAVE_MALLINFO
> +#include <malloc.h>
> +#endif
>  #include "system.h"
>  #include "coretypes.h"
>  #include "timevar.h"
> @@ -29,6 +32,7 @@ along with GCC; see the file COPYING3.
>  #include "params.h"
>  #include "hosthooks.h"
>  #include "plugin.h"
> +#include "options.h"
>  
>  /* When set, ggc_collect will do collection.  */
>  bool ggc_force_collect;
> @@ -1017,3 +1021,14 @@ ggc_prune_overhead_list (void)
>    delete ggc_mem_desc.m_reverse_object_map;
>    ggc_mem_desc.m_reverse_object_map = new map_t (13, false, false, false);
>  }
> +
> +/* Return memory used by heap in kb, 0 if this info is not available.  */
> +
> +void
> +report_heap_memory_use ()
> +{
> +#ifdef HAVE_MALLINFO
> +  if (!quiet_flag)
> +    fprintf (stderr," {heap %luk}", (unsigned long)(mallinfo().arena / 1024));
> +#endif
> +}
> Index: ggc-page.c
> ===================================================================
> --- ggc-page.c	(revision 277366)
> +++ ggc-page.c	(working copy)
> @@ -2267,7 +2267,7 @@ ggc_grow (void)
>    else
>      ggc_collect ();
>    if (!quiet_flag)
> -    fprintf (stderr, " {GC start %luk} ", (unsigned long) G.allocated / 1024);
> +    fprintf (stderr, " {GC %luk} ", (unsigned long) G.allocated / 1024);
>  }
>  
>  void
> Index: ggc.h
> ===================================================================
> --- ggc.h	(revision 277366)
> +++ ggc.h	(working copy)
> @@ -266,6 +266,9 @@ extern void stringpool_statistics (void)
>  /* Heuristics.  */
>  extern void init_ggc_heuristics (void);
>  
> +/* Report current heap memory use to stderr.  */
> +extern void report_heap_memory_use (void);
> +
>  #define ggc_alloc_rtvec_sized(NELT)				\
>    (rtvec_def *) ggc_internal_alloc (sizeof (struct rtvec_def)		\
>  		       + ((NELT) - 1) * sizeof (rtx))		\
> Index: lto/lto-common.c
> ===================================================================
> --- lto/lto-common.c	(revision 277366)
> +++ lto/lto-common.c	(working copy)
> @@ -2784,6 +2784,7 @@ read_cgraph_and_symbols (unsigned nfiles
>    /* At this stage we know that majority of GGC memory is reachable.
>       Growing the limits prevents unnecesary invocation of GGC.  */
>    ggc_grow ();
> +  report_heap_memory_use ();
>  
>    /* Set the hooks so that all of the ipa passes can read in their data.  */
>    lto_set_in_hooks (all_file_decl_data, get_section_data, free_section_data);
> @@ -2791,7 +2792,7 @@ read_cgraph_and_symbols (unsigned nfiles
>    timevar_pop (TV_IPA_LTO_DECL_IN);
>  
>    if (!quiet_flag)
> -    fprintf (stderr, "\nReading the callgraph\n");
> +    fprintf (stderr, "\nReading the symbol table:");
>  
>    timevar_push (TV_IPA_LTO_CGRAPH_IO);
>    /* Read the symtab.  */
> @@ -2831,7 +2832,7 @@ read_cgraph_and_symbols (unsigned nfiles
>    timevar_pop (TV_IPA_LTO_CGRAPH_IO);
>  
>    if (!quiet_flag)
> -    fprintf (stderr, "Merging declarations\n");
> +    fprintf (stderr, "\nMerging declarations:");
>  
>    timevar_push (TV_IPA_LTO_DECL_MERGE);
>    /* Merge global decls.  In ltrans mode we read merged cgraph, we do not
> @@ -2859,12 +2860,13 @@ read_cgraph_and_symbols (unsigned nfiles
>       is explcitly managed by ggc_free and ggc collect is not useful.
>       Exception are the merged declarations.  */
>    ggc_grow ();
> +  report_heap_memory_use ();
>  
>    timevar_pop (TV_IPA_LTO_DECL_MERGE);
>    /* Each pass will set the appropriate timer.  */
>  
>    if (!quiet_flag)
> -    fprintf (stderr, "Reading summaries\n");
> +    fprintf (stderr, "\nReading summaries:");
>  
>    /* Read the IPA summary data.  */
>    if (flag_ltrans)
> @@ -2891,6 +2893,9 @@ read_cgraph_and_symbols (unsigned nfiles
>        /* Finally merge the cgraph according to the decl merging decisions.  */
>        timevar_push (TV_IPA_LTO_CGRAPH_MERGE);
>  
> +      if (!quiet_flag)
> +	fprintf (stderr, "\nMerging symbols:");
> +
>        gcc_assert (!dump_file);
>        dump_file = dump_begin (lto_link_dump_id, NULL);
>  
> @@ -2905,6 +2910,7 @@ read_cgraph_and_symbols (unsigned nfiles
>  	 We could also just remove them while merging.  */
>        symtab->remove_unreachable_nodes (dump_file);
>        ggc_collect ();
> +      report_heap_memory_use ();
>  
>        if (dump_file)
>  	dump_end (lto_link_dump_id, dump_file);
> Index: lto/lto.c
> ===================================================================
> --- lto/lto.c	(revision 277366)
> +++ lto/lto.c	(working copy)
> @@ -312,6 +312,8 @@ lto_wpa_write_files (void)
>        lto_prepare_function_for_streaming (node);
>  
>    ggc_trim ();
> +  report_heap_memory_use ();
> +
>    /* Generate a prefix for the LTRANS unit files.  */
>    blen = strlen (ltrans_output_list);
>    temp_filename = (char *) xmalloc (blen + sizeof ("2147483648.o"));
> Index: passes.c
> ===================================================================
> --- passes.c	(revision 277366)
> +++ passes.c	(working copy)
> @@ -2564,6 +2564,8 @@ execute_one_pass (opt_pass *pass)
>    if (!((todo_after | pass->todo_flags_finish) & TODO_do_not_ggc_collect))
>      ggc_collect ();
>  
> +  if (pass->type == SIMPLE_IPA_PASS || pass->type == IPA_PASS)
> +    report_heap_memory_use ();
>    return true;
>  }
>  
> @@ -2818,6 +2820,8 @@ ipa_read_summaries_1 (opt_pass *pass)
>  	      /* If a timevar is present, start it.  */
>  	      if (pass->tv_id)
>  		timevar_push (pass->tv_id);
> +	      if (!quiet_flag)
> +		fprintf (stderr, " <%s>", pass->name ? pass->name : "");
>  
>  	      pass_init_dump_file (pass);
>  
> @@ -2829,6 +2833,8 @@ ipa_read_summaries_1 (opt_pass *pass)
>  	      /* Stop timevar.  */
>  	      if (pass->tv_id)
>  		timevar_pop (pass->tv_id);
> +	      ggc_grow ();
> +	      report_heap_memory_use ();
>  	    }
>  
>  	  if (pass->sub && pass->sub->type != GIMPLE_PASS)
> @@ -2869,6 +2875,8 @@ ipa_read_optimization_summaries_1 (opt_p
>  	      /* If a timevar is present, start it.  */
>  	      if (pass->tv_id)
>  		timevar_push (pass->tv_id);
> +	      if (!quiet_flag)
> +		fprintf (stderr, " <%s>", pass->name ? pass->name : "");
>  
>  	      pass_init_dump_file (pass);
>  
> @@ -2884,6 +2892,8 @@ ipa_read_optimization_summaries_1 (opt_p
>  
>  	  if (pass->sub && pass->sub->type != GIMPLE_PASS)
>  	    ipa_read_optimization_summaries_1 (pass->sub);
> +	  ggc_grow ();
> +	  report_heap_memory_use ();
>  	}
>        pass = pass->next;
>      }
> Index: config.in
> ===================================================================
> --- config.in	(revision 277366)
> +++ config.in	(working copy)
> @@ -958,6 +958,13 @@
>  #endif
>  
>  
> +/* Define to 1 if we found a declaration for 'mallinfo', otherwise define to
> +   0. */
> +#ifndef USED_FOR_TARGET
> +#undef HAVE_DECL_MALLINFO
> +#endif
> +
> +
>  /* Define to 1 if we found a declaration for 'malloc', otherwise define to 0.
>     */
>  #ifndef USED_FOR_TARGET
> @@ -1627,6 +1634,12 @@
>  #endif
>  
>  
> +/* Define to 1 if you have the `mallinfo' function. */
> +#ifndef USED_FOR_TARGET
> +#undef HAVE_MALLINFO
> +#endif
> +
> +
>  /* Define to 1 if you have the <malloc.h> header file. */
>  #ifndef USED_FOR_TARGET
>  #undef HAVE_MALLOC_H
> Index: configure
> ===================================================================
> --- configure	(revision 277366)
> +++ configure	(working copy)
> @@ -901,6 +901,7 @@ infodir
>  docdir
>  oldincludedir
>  includedir
> +runstatedir
>  localstatedir
>  sharedstatedir
>  sysconfdir
> @@ -1065,6 +1066,7 @@ datadir='${datarootdir}'
>  sysconfdir='${prefix}/etc'
>  sharedstatedir='${prefix}/com'
>  localstatedir='${prefix}/var'
> +runstatedir='${localstatedir}/run'
>  includedir='${prefix}/include'
>  oldincludedir='/usr/include'
>  docdir='${datarootdir}/doc/${PACKAGE}'
> @@ -1317,6 +1319,15 @@ do
>    | -silent | --silent | --silen | --sile | --sil)
>      silent=yes ;;
>  
> +  -runstatedir | --runstatedir | --runstatedi | --runstated \
> +  | --runstate | --runstat | --runsta | --runst | --runs \
> +  | --run | --ru | --r)
> +    ac_prev=runstatedir ;;
> +  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
> +  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
> +  | --run=* | --ru=* | --r=*)
> +    runstatedir=$ac_optarg ;;
> +
>    -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
>      ac_prev=sbindir ;;
>    -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
> @@ -1454,7 +1465,7 @@ fi
>  for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
>  		datadir sysconfdir sharedstatedir localstatedir includedir \
>  		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
> -		libdir localedir mandir
> +		libdir localedir mandir runstatedir
>  do
>    eval ac_val=\$$ac_var
>    # Remove trailing slashes.
> @@ -1607,6 +1618,7 @@ Fine tuning of the installation director
>    --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
>    --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
>    --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
> +  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
>    --libdir=DIR            object code libraries [EPREFIX/lib]
>    --includedir=DIR        C header files [PREFIX/include]
>    --oldincludedir=DIR     C header files for non-gcc [/usr/include]
> @@ -5891,7 +5903,7 @@ else
>      We can't simply define LARGE_OFF_T to be 9223372036854775807,
>      since some C++ compilers masquerading as C compilers
>      incorrectly reject 9223372036854775807.  */
> -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
> +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
>    int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
>  		       && LARGE_OFF_T % 2147483647 == 1)
>  		      ? 1 : -1];
> @@ -5937,7 +5949,7 @@ else
>      We can't simply define LARGE_OFF_T to be 9223372036854775807,
>      since some C++ compilers masquerading as C compilers
>      incorrectly reject 9223372036854775807.  */
> -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
> +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
>    int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
>  		       && LARGE_OFF_T % 2147483647 == 1)
>  		      ? 1 : -1];
> @@ -5961,7 +5973,7 @@ rm -f core conftest.err conftest.$ac_obj
>      We can't simply define LARGE_OFF_T to be 9223372036854775807,
>      since some C++ compilers masquerading as C compilers
>      incorrectly reject 9223372036854775807.  */
> -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
> +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
>    int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
>  		       && LARGE_OFF_T % 2147483647 == 1)
>  		      ? 1 : -1];
> @@ -6006,7 +6018,7 @@ else
>      We can't simply define LARGE_OFF_T to be 9223372036854775807,
>      since some C++ compilers masquerading as C compilers
>      incorrectly reject 9223372036854775807.  */
> -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
> +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
>    int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
>  		       && LARGE_OFF_T % 2147483647 == 1)
>  		      ? 1 : -1];
> @@ -6030,7 +6042,7 @@ rm -f core conftest.err conftest.$ac_obj
>      We can't simply define LARGE_OFF_T to be 9223372036854775807,
>      since some C++ compilers masquerading as C compilers
>      incorrectly reject 9223372036854775807.  */
> -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
> +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
>    int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
>  		       && LARGE_OFF_T % 2147483647 == 1)
>  		      ? 1 : -1];
> @@ -10034,7 +10046,7 @@ LIBS="$save_LIBS"
>  for ac_func in times clock kill getrlimit setrlimit atoq \
>  	popen sysconf strsignal getrusage nl_langinfo \
>  	gettimeofday mbstowcs wcswidth mmap setlocale \
> -	clearerr_unlocked feof_unlocked   ferror_unlocked fflush_unlocked fgetc_unlocked fgets_unlocked   fileno_unlocked fprintf_unlocked fputc_unlocked fputs_unlocked   fread_unlocked fwrite_unlocked getchar_unlocked getc_unlocked   putchar_unlocked putc_unlocked madvise
> +	clearerr_unlocked feof_unlocked   ferror_unlocked fflush_unlocked fgetc_unlocked fgets_unlocked   fileno_unlocked fprintf_unlocked fputc_unlocked fputs_unlocked   fread_unlocked fwrite_unlocked getchar_unlocked getc_unlocked   putchar_unlocked putc_unlocked madvise mallinfo
>  do :
>    as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
>  ac_fn_cxx_check_func "$LINENO" "$ac_func" "$as_ac_var"
> @@ -11448,6 +11460,61 @@ fi
>  done
>  
>  
> +for ac_func in mallinfo
> +do
> +  ac_tr_decl=`$as_echo "HAVE_DECL_$ac_func" | $as_tr_cpp`
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $ac_func is declared" >&5
> +$as_echo_n "checking whether $ac_func is declared... " >&6; }
> +if eval \${gcc_cv_have_decl_$ac_func+:} false; then :
> +  $as_echo_n "(cached) " >&6
> +else
> +  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
> +/* end confdefs.h.  */
> +#undef $ac_tr_decl
> +#define $ac_tr_decl 1
> +
> +#include "ansidecl.h"
> +#include "system.h"
> +#ifdef HAVE_MALLOC_H
> +#include <malloc.h>
> +#endif
> +
> +
> +int
> +main ()
> +{
> +#ifndef $ac_func
> +char *(*pfn) = (char *(*)) $ac_func ;
> +#endif
> +  ;
> +  return 0;
> +}
> +_ACEOF
> +if ac_fn_cxx_try_compile "$LINENO"; then :
> +  eval "gcc_cv_have_decl_$ac_func=yes"
> +else
> +  eval "gcc_cv_have_decl_$ac_func=no"
> +fi
> +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
> +fi
> +
> +if eval "test \"`echo '$gcc_cv_have_decl_'$ac_func`\" = yes"; then
> +  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
> +$as_echo "yes" >&6; } ; cat >>confdefs.h <<_ACEOF
> +#define $ac_tr_decl 1
> +_ACEOF
> +
> +else
> +  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
> +$as_echo "no" >&6; } ; cat >>confdefs.h <<_ACEOF
> +#define $ac_tr_decl 0
> +_ACEOF
> +
> +fi
> +
> +done
> +
> +
>  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
>  /* end confdefs.h.  */
>  
> @@ -18851,7 +18918,7 @@ else
>    lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
>    lt_status=$lt_dlunknown
>    cat > conftest.$ac_ext <<_LT_EOF
> -#line 18854 "configure"
> +#line 18921 "configure"
>  #include "confdefs.h"
>  
>  #if HAVE_DLFCN_H
> @@ -18957,7 +19024,7 @@ else
>    lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
>    lt_status=$lt_dlunknown
>    cat > conftest.$ac_ext <<_LT_EOF
> -#line 18960 "configure"
> +#line 19027 "configure"
>  #include "confdefs.h"
>  
>  #if HAVE_DLFCN_H
>