2019-10-28  Nathan Sidwell  <nathan@acm.org>

	* parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism.
	Add tree_check_p flag, use as nested union discriminator.
	(struct cp_lexer): Add saved_type & saved_keyword fields.
	* parser.c (eof_token): Delete.
	(cp_lexer_new_main): Always init last_token to last token of
	buffer.
	(cp_lexer_new_from_tokens): Overlay EOF token at end of range.
	(cp_lexer_destroy): Restore token under the EOF.
	(cp_lexer_previous_token_position): No check for eof_token here.
	(cp_lexer_get_preprocessor_token): Clear tree_check_p.
	(cp_lexer_peek_nth_token): Check CPP_EOF not eof_token.
	(cp_lexer_consume_token): Assert not CPP_EOF, no check for
	eof_token.
	(cp_lexer_purge_token): Likewise.
	(cp_lexer_purge_tokens_after): No check for EOF token.
	(cp_parser_nested_name_specifier, cp_parser_decltype)
	(cp_parser_template_id): Set tree_check_p.

Index: gcc/cp/parser.c
===================================================================
--- gcc/cp/parser.c	(revision 277460)
+++ gcc/cp/parser.c	(working copy)
@@ -53,9 +53,4 @@ along with GCC; see the file COPYING3.
    and c-lex.c) and the C++ parser.  */
 
-static cp_token eof_token =
-{
-  CPP_EOF, RID_MAX, 0, false, false, false, 0, { NULL }
-};
-
 /* The various kinds of non integral constant we encounter. */
 enum non_integral_constant {
@@ -661,10 +656,8 @@ cp_lexer_new_main (void)
     }
 
-  lexer->last_token = lexer->buffer->address ()
+  lexer->next_token = lexer->buffer->address ();
+  lexer->last_token = lexer->next_token
                       + lexer->buffer->length ()
 		      - 1;
-  lexer->next_token = lexer->buffer->length ()
-		      ? lexer->buffer->address ()
-		      : &eof_token;
 
   /* Subsequent preprocessor diagnostics should use compiler
@@ -688,5 +681,12 @@ cp_lexer_new_from_tokens (cp_token_cache
   /* We do not own the buffer.  */
   lexer->buffer = NULL;
-  lexer->next_token = first == last ? &eof_token : first;
+
+  /* Insert an EOF token.  */
+  lexer->saved_type = last->type;
+  lexer->saved_keyword = last->keyword;
+  last->type = CPP_EOF;
+  last->keyword = RID_MAX;
+
+  lexer->next_token = first;
   lexer->last_token = last;
 
@@ -705,5 +705,12 @@ static void
 cp_lexer_destroy (cp_lexer *lexer)
 {
-  vec_free (lexer->buffer);
+  if (lexer->buffer)
+    vec_free (lexer->buffer);
+  else
+    {
+      /* Restore the token we overwrite with EOF.  */
+      lexer->last_token->type = lexer->saved_type;
+      lexer->last_token->keyword = lexer->saved_keyword;
+    }
   lexer->saved_tokens.release ();
   ggc_free (lexer);
@@ -732,6 +739,4 @@ static inline cp_token_position
 cp_lexer_token_position (cp_lexer *lexer, bool previous_p)
 {
-  gcc_assert (!previous_p || lexer->next_token != &eof_token);
-
   return lexer->next_token - previous_p;
 }
@@ -752,8 +757,5 @@ static inline cp_token_position
 cp_lexer_previous_token_position (cp_lexer *lexer)
 {
-  if (lexer->next_token == &eof_token)
-    return lexer->last_token - 1;
-  else
-    return cp_lexer_token_position (lexer, true);
+  return cp_lexer_token_position (lexer, true);
 }
 
@@ -808,4 +810,5 @@ cp_lexer_get_preprocessor_token (cp_lexe
   token->purged_p = false;
   token->error_reported = false;
+  token->tree_check_p = false;
 
   /* On some systems, some header files are surrounded by an
@@ -1083,14 +1086,7 @@ cp_lexer_peek_nth_token (cp_lexer* lexer
   --n;
   token = lexer->next_token;
-  gcc_assert (!n || token != &eof_token);
-  while (n != 0)
+  while (n && token->type != CPP_EOF)
     {
       ++token;
-      if (token == lexer->last_token)
-	{
-	  token = &eof_token;
-	  break;
-	}
-
       if (!token->purged_p)
 	--n;
@@ -1114,16 +1110,10 @@ cp_lexer_consume_token (cp_lexer* lexer)
   cp_token *token = lexer->next_token;
 
-  gcc_assert (token != &eof_token);
   gcc_assert (!lexer->in_pragma || token->type != CPP_PRAGMA_EOL);
 
   do
     {
+      gcc_assert (token->type != CPP_EOF);
       lexer->next_token++;
-      if (lexer->next_token == lexer->last_token)
-	{
-	  lexer->next_token = &eof_token;
-	  break;
-	}
-
     }
   while (lexer->next_token->purged_p);
@@ -1151,5 +1141,5 @@ cp_lexer_purge_token (cp_lexer *lexer)
   cp_token *tok = lexer->next_token;
 
-  gcc_assert (tok != &eof_token);
+  gcc_assert (tok->type != CPP_EOF);
   tok->purged_p = true;
   tok->location = UNKNOWN_LOCATION;
@@ -1158,12 +1148,5 @@ cp_lexer_purge_token (cp_lexer *lexer)
 
   do
-    {
-      tok++;
-      if (tok == lexer->last_token)
-	{
-	  tok = &eof_token;
-	  break;
-	}
-    }
+    tok++;
   while (tok->purged_p);
   lexer->next_token = tok;
@@ -1179,10 +1162,7 @@ cp_lexer_purge_tokens_after (cp_lexer *l
   cp_token *peek = lexer->next_token;
 
-  if (peek == &eof_token)
-    peek = lexer->last_token;
-
   gcc_assert (tok < peek);
 
-  for ( tok += 1; tok != peek; tok += 1)
+  for (tok++; tok != peek; tok++)
     {
       tok->purged_p = true;
@@ -6617,4 +6597,5 @@ cp_parser_nested_name_specifier_opt (cp_
 	 so the memory will not be reclaimed during token replacing below.  */
       token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
+      token->tree_check_p = true;
       token->u.tree_check_value->value = parser->scope;
       token->u.tree_check_value->checks = get_deferred_access_checks ();
@@ -14802,4 +14783,5 @@ cp_parser_decltype (cp_parser *parser)
   start_token->type = CPP_DECLTYPE;
   start_token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
+  start_token->tree_check_p = true;
   start_token->u.tree_check_value->value = expr;
   start_token->u.tree_check_value->checks = get_deferred_access_checks ();
@@ -16589,4 +16571,5 @@ cp_parser_template_id (cp_parser *parser
 	 so the memory will not be reclaimed during token replacing below.  */
       token->u.tree_check_value = ggc_cleared_alloc<struct tree_check> ();
+      token->tree_check_p = true;
       token->u.tree_check_value->value = template_id;
       token->u.tree_check_value->checks = get_deferred_access_checks ();
Index: gcc/cp/parser.h
===================================================================
--- gcc/cp/parser.h	(revision 277460)
+++ gcc/cp/parser.h	(working copy)
@@ -42,21 +42,23 @@ struct GTY(()) tree_check {
 struct GTY (()) cp_token {
   /* The kind of token.  */
-  ENUM_BITFIELD (cpp_ttype) type : 8;
+  enum cpp_ttype type : 8;
   /* If this token is a keyword, this value indicates which keyword.
      Otherwise, this value is RID_MAX.  */
-  ENUM_BITFIELD (rid) keyword : 8;
+  enum rid keyword : 8;
   /* Token flags.  */
   unsigned char flags;
   /* True if this token is from a context where it is implicitly extern "C" */
-  BOOL_BITFIELD implicit_extern_c : 1;
+  bool implicit_extern_c : 1;
   /* True if an error has already been reported for this token, such as a
      CPP_NAME token that is not a keyword (i.e., for which KEYWORD is
      RID_MAX) iff this name was looked up and found to be ambiguous.  */
-  BOOL_BITFIELD error_reported : 1;
+  bool error_reported : 1;
   /* True for a token that has been purged.  If a token is purged,
      it is no longer a valid token and it should be considered
      deleted.  */
-  BOOL_BITFIELD purged_p : 1;
-  /* 5 unused bits.  */
+  bool purged_p : 1;
+  bool tree_check_p : 1;
+  /* 4 unused bits.  */
+
   /* The location at which this token was found.  */
   location_t location;
@@ -64,10 +66,8 @@ struct GTY (()) cp_token {
   union cp_token_value {
     /* Used for compound tokens such as CPP_NESTED_NAME_SPECIFIER.  */
-    struct tree_check* GTY((tag ("1"))) tree_check_value;
+    struct tree_check* GTY((tag ("true"))) tree_check_value;
     /* Use for all other tokens.  */
-    tree GTY((tag ("0"))) value;
-  } GTY((desc ("(%1.type == CPP_TEMPLATE_ID)"
-	       "|| (%1.type == CPP_NESTED_NAME_SPECIFIER)"
-	       "|| (%1.type == CPP_DECLTYPE)"))) u;
+    tree GTY((tag ("false"))) value;
+  } GTY((desc ("%1.tree_check_p"))) u;
 };
 
@@ -100,4 +100,8 @@ struct GTY (()) cp_lexer {
   vec<cp_token_position> GTY ((skip)) saved_tokens;
 
+  /* Saved pieces of end token we replaced with the eof token.  */
+  enum cpp_ttype saved_type : 8;
+  enum rid saved_keyword : 8;
+
   /* The next lexer in a linked list of lexers.  */
   struct cp_lexer *next;
