cpphash.h: ISvspace, is_vspace, is_nvspace: New.

* cpphash.h: ISvspace, is_vspace, is_nvspace: New.
	IShspace, ISspace: Update.

	* cppinit.c: ISTABLE: Update.
	V: New.

	* cpplex.c (IS_HSPACE, S_NEWLINE): Remove.
	(IS_DIRECTIVE): Rename KNOWN_DIRECTIVE.
	(skip_block_comment, skip_line_comment,	parse_string,
	lex_line): Use is_vspace rather than IS_NEWLINE.
	(skip_whitespace, lex_line): Clean up to use is_nvspace.
	(lex_line): Use KNOWN_DIRECTIVE.  Any kind of directive
	gets a BOL flag.
	(lex_next): Unconditionally stop if within a directive.
	Treat directives within macro invocations as directives
	(after parse_args emits error), not as the argument.

	* testsuite/gcc.dg/cpp/directiv.c: New tests.
	* testsuite/gcc.dg/cpp/undef1.c: Update.

From-SVN: r34933

gcc/ChangeLog

@@ -1,3 +1,22 @@
+2000-07-09  Neil Booth  <NeilB@earthling.net>
+
+	* cpphash.h: ISvspace, is_vspace, is_nvspace: New.
+	IShspace, ISspace: Update.
+
+	* cppinit.c: ISTABLE: Update.
+	V: New.
+
+	* cpplex.c (IS_HSPACE, S_NEWLINE): Remove.
+	(IS_DIRECTIVE): Rename KNOWN_DIRECTIVE.
+	(skip_block_comment, skip_line_comment,	parse_string,
+	lex_line): Use is_vspace rather than IS_NEWLINE.
+	(skip_whitespace, lex_line): Clean up to use is_nvspace.
+	(lex_line): Use KNOWN_DIRECTIVE.  Any kind of directive
+	gets a BOL flag.
+	(lex_next): Unconditionally stop if within a directive.
+	Treat directives within macro invocations as directives
+	(after parse_args emits error), not as the argument.
+
 2000-07-09  Gabriel Dos Reis  <gdr@codesourcery.com>
 
 	* diagnostic.c (diagnostic_args): New macro.

gcc/cpphash.h

@@ -141,12 +141,17 @@ struct spec_nodes
 
 /* Character classes.
    If the definition of `numchar' looks odd to you, please look up the
-   definition of a pp-number in the C standard [section 6.4.8 of C99] */
+   definition of a pp-number in the C standard [section 6.4.8 of C99].
+
+   In the unlikely event that characters other than \r and \n enter
+   the set is_vspace, the macro handle_newline() in cpplex.c must be
+   updated.  */
 #define ISidnum		0x01	/* a-zA-Z0-9_ */
 #define ISidstart	0x02	/* _a-zA-Z */
 #define ISnumstart	0x04	/* 0-9 */
-#define IShspace	0x08	/* ' ' \t \f \v */
+#define IShspace	0x08	/* ' ' \t */
-#define ISspace		0x10	/* ' ' \t \f \v \n */
+#define ISvspace	0x10	/* \r \n */
+#define ISspace		0x20	/* ' ' \t \r \n \f \v \0 */
 
 #define _dollar_ok(x)	((x) == '$' && CPP_OPTION (pfile, dollars_in_ident))
 
@@ -155,6 +160,8 @@ struct spec_nodes
 #define is_numchar(x)	(_cpp_IStable[x] & ISidnum)
 #define is_numstart(x)	(_cpp_IStable[x] & ISnumstart)
 #define is_hspace(x)	(_cpp_IStable[x] & IShspace)
+#define is_vspace(x)	(_cpp_IStable[x] & ISvspace)
+#define is_nvspace(x)	((_cpp_IStable[x] & (ISspace | ISvspace)) == ISspace)
 #define is_space(x)	(_cpp_IStable[x] & ISspace)
 
 /* This table is constant if it can be initialized at compile time,

gcc/cppinit.c

@@ -138,6 +138,7 @@ enum { QUOTE = 0, BRACKET, SYSTEM, AFTER };
 #define A(x) s(x, ISidnum|ISidstart)
 #define N(x) s(x, ISidnum|ISnumstart)
 #define H(x) s(x, IShspace|ISspace)
+#define V(x) s(x, ISvspace|ISspace)
 #define S(x) s(x, ISspace)
 
 ISTABLE
@@ -153,14 +154,17 @@ ISTABLE
 
   N('1') N('2') N('3') N('4') N('5') N('6') N('7') N('8') N('9') N('0')
 
-  H('\0') H(' ') H('\t') H('\v') H('\f')
+  H(' ') H('\t')
 
-  S('\n')
+  V('\n') V('\r')
+
+  S('\0') S('\v') S('\f')
 END
 
 #undef A
 #undef N
 #undef H
+#undef V
 #undef S
 #undef s
 #undef ISTABLE

gcc/cpplex.c

@@ -24,22 +24,17 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
 
 Cleanups to do:-
 
-o Fix ISTABLE to flag the parts we want for IS_HSPACE and IS_NEWLINE.
 o Get use of digraphs in sync with the standard reqd on the command line.
 o -dM and with _cpp_dump_list: too many \n output.
 o Put a printer object in cpp_reader?
 o Check line numbers assigned to all errors.
 o Replace strncmp with memcmp almost everywhere.
 o lex_line's use of cur_token, flags and list->token_used is a bit opaque.
-o Get rid of cpp_get_directive_token.
 o Convert do_ functions to return void.  Kaveh thinks its OK; and said he'll
   give it a run when we've got some code.
-o _cpp_parse_expr updated to new lexer.
 o Distinguish integers, floats, and 'other' pp-numbers.
 o Store ints and char constants as binary values.
 o New command-line assertion syntax.
-o Merge hash table text pointer and token list text pointer for identifiers.
-o Have _cpp_parse_expr use all the information the new lexer provides.
 o Work towards functions in cpperror.c taking a message level parameter.
   If we do this, merge the common code of do_warning and do_error.
 o Comment all functions, and describe macro expansion algorithm.
@@ -122,10 +117,6 @@ static void process_directive PARAMS ((cpp_reader *, const cpp_token *));
    ((prevc) == 'e' || (prevc) == 'E' \
     || (((prevc) == 'p' || (prevc) == 'P') && !CPP_OPTION (pfile, c89))))
 
-/* Maybe put these in the ISTABLE eventually.  */
-#define IS_HSPACE(c) ((c) == ' ' || (c) == '\t')
-#define IS_NEWLINE(c) ((c) == '\n' || (c) == '\r')
-
 /* Handle LF, CR, CR-LF and LF-CR style newlines.  Assumes next
    character, if any, is in buffer.  */
 
@@ -863,7 +854,7 @@ skip_block_comment (pfile)
 	      cpp_warning (pfile, "'/*' within comment");
 	    }
 	}
-      else if (IS_NEWLINE(c))
+      else if (is_vspace (c))
 	{
 	  const unsigned char* bslash = backslash_start (pfile, cur - 2);
 
@@ -897,7 +888,7 @@ skip_line_comment (pfile)
     {
       unsigned char c = *cur++;
 
-      if (IS_NEWLINE (c))
+      if (is_vspace (c))
 	{
 	  /* Check for a (trigaph?) backslash escaping the newline.  */
 	  if (!backslash_start (pfile, cur - 2))
@@ -922,37 +913,38 @@ skip_whitespace (pfile, in_directive)
      int in_directive;
 {
   cpp_buffer *buffer = pfile->buffer;
-  register const unsigned char *cur = buffer->cur;
+  unsigned short warned = 0;
-  unsigned short null_count = 0;
 
-  for (; cur < buffer->rlimit; )
+  /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
+  while (buffer->cur < buffer->rlimit)
     {
-      unsigned char c = *cur++;
+      unsigned char c = *buffer->cur;
 
-      if (c == '\t')
+      if (!is_nvspace (c))
-	{
+	break;
-	  unsigned int col = CPP_BUF_COLUMN (buffer, cur - 1);
+
-	  pfile->col_adjust += (CPP_OPTION (pfile, tabstop) - 1
+      buffer->cur++;
-				- col % CPP_OPTION(pfile, tabstop));
+      /* Horizontal space always OK.  */
-	}
+      if (c == ' ')
-      if (IS_HSPACE(c))		/* FIXME: Fix ISTABLE.  */
 	continue;
-      if (!is_space(c) || IS_NEWLINE (c)) /* Main loop handles newlines.  */
+      else if (c == '\t')
-	goto out;
+	pfile->col_adjust += CPP_OPTION (pfile, tabstop) - 1
-      if (c == '\0')
+	  - (CPP_BUF_COL (buffer) - 1) % CPP_OPTION(pfile, tabstop);
-	null_count++;
+      /* Must be \f \v or \0.  */
-      /* Mut be '\f' or '\v' */
+      else if (c == '\0')
+	{
+	  if (!warned)
+	    cpp_warning_with_line (pfile, CPP_BUF_LINE (buffer),
+				   CPP_BUF_COL (buffer),
+				   "embedded null character ignored");
+	  warned = 1;
+	}
       else if (in_directive && CPP_PEDANTIC (pfile))
-	cpp_pedwarn (pfile, "%s in preprocessing directive",
+	cpp_pedwarn_with_line (pfile, CPP_BUF_LINE (buffer),
-		     c == '\f' ? "formfeed" : "vertical tab");
+			       CPP_BUF_COL (buffer),
+			       "%s in preprocessing directive",
+			       c == '\f' ? "form feed" : "vertical tab");
     }
-  cur++;
-
- out:
-  buffer->cur = cur - 1;
-  if (null_count)
-    cpp_warning (pfile, null_count > 1 ? "embedded null characters ignored"
-		 : "embedded null character ignored");
 }
 
 /* Parse (append) an identifier.  */
@@ -1073,7 +1065,7 @@ parse_string (pfile, list, token, terminator)
 
       if (c == '\0')
 	null_count++;
-      else if (c == terminator || IS_NEWLINE (c))
+      else if (c == terminator || is_vspace (c))
 	{
 	  /* Needed for trigraph_replace and multiline string warning.  */
 	  buffer->cur = cur;
@@ -1090,7 +1082,7 @@ parse_string (pfile, list, token, terminator)
 	    }
 
 	  namebuf--;     /* Drop the newline / terminator from the name.  */
-	  if (IS_NEWLINE (c))
+	  if (is_vspace (c))
 	    {
 	      /* Drop a backslash newline, and continue. */
 	      if (namebuf[-1] == '\\')
@@ -1159,7 +1151,7 @@ parse_string (pfile, list, token, terminator)
   /* We may not have trigraph-replaced the input for this code path,
      but as the input is in error by being unterminated we don't
      bother.  Prevent warnings about no newlines at EOF.  */
-  if (IS_NEWLINE(cur[-1]))
+  if (is_vspace (cur[-1]))
     cur--;
 
  unterminated:
@@ -1240,7 +1232,7 @@ save_comment (list, token, from, len, type)
  *  even when enabled.
  */
 
-#define IS_DIRECTIVE() (list->directive != 0)
+#define KNOWN_DIRECTIVE() (list->directive != 0)
 #define MIGHT_BE_DIRECTIVE() \
 (cur_token == &list->tokens[first_token + 1] && cur_token[-1].type == CPP_HASH)
 
@@ -1273,21 +1265,22 @@ lex_line (pfile, list)
     {
       unsigned char c;
 
-      /* Optimize whitespace skipping, as most tokens are probably
+      /* Optimize non-vertical whitespace skipping; most tokens are
-	 separated by whitespace. (' ' '\t' '\v' '\f' '\0').  */
+	 probably separated by whitespace. (' ' '\t' '\v' '\f' '\0').  */
-      c = *cur++;
+      c = *cur;
-      if (is_hspace (c))
+      if (is_nvspace (c))
 	{
-	  /* Step back to get the null warning and tab correction.  */
+	  buffer->cur = cur;
-	  buffer->cur = cur - 1;
+	  skip_whitespace (pfile, (list->tokens[first_token].type == CPP_HASH
-	  skip_whitespace (pfile, IS_DIRECTIVE ());
+				   && cur_token > &list->tokens[first_token]));
 	  cur = buffer->cur;
 
 	  flags = PREV_WHITE;
 	  if (cur == buffer->rlimit)
 	    break;
-	  c = *cur++;
+	  c = *cur;
 	}
+      cur++;
 
       /* Initialize current token.  CPP_EOF will not be fixed up by
 	 expand_name_space.  */
@@ -1428,7 +1421,7 @@ lex_line (pfile, list)
 		      /* Back-up to first '-' or '/'.  */
 		      cur_token--;
 		      if (!CPP_OPTION (pfile, discard_comments)
-			  && (!IS_DIRECTIVE()
+			  && (!KNOWN_DIRECTIVE()
 			      || (list->directive->flags & COMMENTS)))
 			save_comment (list, cur_token++, cur,
 				      buffer->cur - cur, c);
@@ -1466,7 +1459,7 @@ lex_line (pfile, list)
 		  /* Back up to opening '/'.  */
 		  cur_token--;
 		  if (!CPP_OPTION (pfile, discard_comments)
-		      && (!IS_DIRECTIVE()
+		      && (!KNOWN_DIRECTIVE()
 			  || (list->directive->flags & COMMENTS)))
 		    save_comment (list, cur_token++, cur,
 				  buffer->cur - cur, c);
@@ -1647,7 +1640,7 @@ lex_line (pfile, list)
 	      break;
 	    }
 	  /* Is this the beginning of a header name?  */
-	  if (IS_DIRECTIVE () && (list->directive->flags & INCL))
+	  if (KNOWN_DIRECTIVE () && (list->directive->flags & INCL))
 	    {
 	      c = '>';	/* Terminator.  */
 	      cur_token->type = CPP_HEADER_NAME;
@@ -1755,7 +1748,7 @@ lex_line (pfile, list)
   cur_token->flags = flags;
   if (cur_token == &list->tokens[first_token] && pfile->done_initializing)
     {
-      if (cur > buffer->buf && !IS_NEWLINE (cur[-1]))
+      if (cur > buffer->buf && !is_vspace (cur[-1]))
 	cpp_pedwarn_with_line (pfile, CPP_BUF_LINE (buffer),
 			       CPP_BUF_COLUMN (buffer, cur),
 			       "no newline at end of file");
@@ -1780,16 +1773,16 @@ lex_line (pfile, list)
 	cpp_error (pfile, "invalid preprocessing directive");
     }
 
-  /* Put EOF at end of directives.  This covers "directives do not
+  /* Put EOF at end of known directives.  This covers "directives do
-     extend beyond the end of the line (description 6.10 part 2)".  */
+     not extend beyond the end of the line (description 6.10 part 2)".  */
-  if (IS_DIRECTIVE () || !pfile->done_initializing)
+  if (KNOWN_DIRECTIVE () || !pfile->done_initializing)
     {
       pfile->first_directive_token = first;
       cur_token++->type = CPP_EOF;
     }
 
-  if (first_token == 0 || IS_DIRECTIVE ())
+  /* Directives, known or not, always start a new line.  */
-    /* Set beginning of line flag.  */
+  if (first_token == 0 || list->tokens[first_token].type == CPP_HASH)
     first->flags |= BOL;
   else
     /* 6.10.3.10: Within the sequence of preprocessing tokens making
@@ -2968,6 +2961,16 @@ lex_next (pfile, clear)
   const cpp_token *old_list = list->tokens;
   unsigned int old_used = list->tokens_used;
 
+  /* If we are currently processing a directive, do not advance.  6.10
+     paragraph 2: A new-line character ends the directive even if it
+     occurs within what would otherwise be an invocation of a
+     function-like macro.
+
+     It is possible that clear == 1 too; e.g. "#if funlike_macro ("
+     since parse_args swallowed the directive's EOF.  */
+  if (list->directive)
+    return 1;
+
   if (clear)
     {
       /* Release all temporary tokens.  */
@@ -2976,15 +2979,6 @@ lex_next (pfile, clear)
       if (pfile->temp_used)
 	release_temp_tokens (pfile);
     }
-  else
-    {
-      /* If we are currently processing a directive, do not advance.
-	 (6.10 paragraph 2: A new-line character ends the directive
-	 even if it occurs within what would otherwise be an
-	 invocation of a function-like macro.)  */
-      if (list->directive)
-	return 1;
-    }
      
   lex_line (pfile, list);
   pfile->contexts[0].count = list->tokens_used;
@@ -3021,11 +3015,7 @@ lex_next (pfile, clear)
 			       list->tokens[old_used].col,
 			       "#%s may not be used inside a macro argument",
 			       list->directive->name);
-	  /* Don't treat as a directive: clear list->directive,
+	  return 1;
-	     prune the final EOF from the list.  */
-	  list->directive = 0;
-	  list->tokens_used--;
-	  pfile->contexts[0].count--;
 	}
     }
 

gcc/testsuite/ChangeLog

@@ -1,3 +1,8 @@
+2000-07-09  Neil Booth  <NeilB@earthling.net>
+
+	* testsuite/gcc.dg/cpp/directiv.c: New tests.
+	* testsuite/gcc.dg/cpp/undef1.c: Update.
+
 2000-07-08  Angela Marie Thomas <angela@cygnus.com>
 
 	* lib/c-torture.exp: Make compiler_conditional_xfail_data global.

gcc/testsuite/gcc.dg/cpp/directiv.c

@@ -0,0 +1,47 @@
+/* Copyright (C) 2000 Free Software Foundation, Inc.  */
+
+/* { dg-do preprocess } */
+/* { dg-options -pedantic } */
+
+/* Tests general directive syntax, and directive error recovery.  */
+
+
+/* Test directive name is not expanded.  */
+#define foo define
+#foo EMPTY			/* { dg-error "invalid" } */
+
+/* Test # must be first on line.  */
+EMPTY #define bar
+#ifdef bar
+#error bar is defined
+#endif
+
+/* Test form feed and vertical tab warn pedantically, see 6.10
+   paragraph 5.  Tab is OK.  */
+#define something		/* { dg-warning "form feed" } */
+#define something_else	/* { dg-warning "vertical tab" } */
+#define some	thing		/* Tab OK, as is form feed before #.  */
+
+/* Our friend the null directive OK?  */
+#
+
+/* Check that directives always start a line, even if in middle of
+   macro expansion.  */
+#define func(x) x
+func (2				/* { dg-error "unterminated invocation" }  */
+#define foobar			/* { dg-error "may not be used inside" } */
+
+/* For tidiness, I think the directive should still be processed
+   above.  Certainly, continuing to try to find the closing ')' can
+   lead to some really confusing error messages.  Hence this test.  */
+#ifndef foobar
+#error It is nice if the directive is processed!
+#endif
+
+/* Check newlines end directives, even in function-like macro
+   invocations.  6.10 paragraph 1.
+
+   Note that the #if is still treated as a conditional, so there
+   should be no errors about #endif without #if.  */
+#if func (			/* { dg-error "unterminated invocation" } */
+#endif

gcc/testsuite/gcc.dg/cpp/undef1.c

@@ -9,6 +9,6 @@
 
 #define foo(bar) bar
 
-foo( blah
+foo( blah			/* { dg-error "unterminated invocation" }  */
 #undef foo	/* { dg-error "may not be used inside" "foo(#undef foo)" } */
      blah )