Add --unicode option to control how unicode characters are handled by display tools.

* nm.c: Add --unicode option to control how unicode characters are
	handled.
	* objdump.c: Likewise.
	* readelf.c: Likewise.
	* strings.c: Likewise.
	* binutils.texi: Document the new feature.
	* NEWS: Document the new feature.
	* testsuite/binutils-all/unicode.exp: New file.
	* testsuite/binutils-all/nm.hex.unicode
	* testsuite/binutils-all/strings.escape.unicode
	* testsuite/binutils-all/objdump.highlight.unicode
	* testsuite/binutils-all/readelf.invalid.unicode
This commit is contained in:
Nick Clifton 2021-11-09 13:25:42 +00:00
parent 024120b6ee
commit b3aa80b45c
7 changed files with 1411 additions and 105 deletions

View file

@ -55,6 +55,19 @@
-T {bfdname}
Specify a non-default object file format.
--unicode={default|locale|invalid|hex|escape|highlight}
-u {d|l|i|x|e|h}
Determine how to handle UTF-8 unicode characters. The default
is no special treatment. All other versions of this option
only apply if the encoding is valid and enabling the option
implies --encoding=S.
The 'locale' option displays the characters according to the
current locale. The 'invalid' option treats them as
non-string characters. The 'hex' option displays them as hex
byte sequences. The 'escape' option displays them as escape
sequences and the 'highlight' option displays them as
coloured escape sequences.
--output-separator=sep_string
-s sep_string String used to separate parsed strings in output.
Default is newline.
@ -76,6 +89,22 @@
#include "safe-ctype.h"
#include "bucomm.h"
#ifndef streq
#define streq(a,b) (strcmp ((a),(b)) == 0)
#endif
typedef enum unicode_display_type
{
unicode_default = 0,
unicode_locale,
unicode_escape,
unicode_hex,
unicode_highlight,
unicode_invalid
} unicode_display_type;
static unicode_display_type unicode_display = unicode_default;
#define STRING_ISGRAPHIC(c) \
( (c) >= 0 \
&& (c) <= 255 \
@ -94,7 +123,7 @@ extern int errno;
static int address_radix;
/* Minimum length of sequence of graphic chars to trigger output. */
static int string_min;
static uint string_min;
/* Whether or not we include all whitespace as a graphic char. */
static bool include_all_whitespace;
@ -121,21 +150,22 @@ static char *output_separator;
static struct option long_options[] =
{
{"all", no_argument, NULL, 'a'},
{"data", no_argument, NULL, 'd'},
{"print-file-name", no_argument, NULL, 'f'},
{"bytes", required_argument, NULL, 'n'},
{"radix", required_argument, NULL, 't'},
{"include-all-whitespace", no_argument, NULL, 'w'},
{"data", no_argument, NULL, 'd'},
{"encoding", required_argument, NULL, 'e'},
{"target", required_argument, NULL, 'T'},
{"output-separator", required_argument, NULL, 's'},
{"help", no_argument, NULL, 'h'},
{"include-all-whitespace", no_argument, NULL, 'w'},
{"output-separator", required_argument, NULL, 's'},
{"print-file-name", no_argument, NULL, 'f'},
{"radix", required_argument, NULL, 't'},
{"target", required_argument, NULL, 'T'},
{"unicode", required_argument, NULL, 'U'},
{"version", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
};
static bool strings_file (char *);
static void print_strings (const char *, FILE *, file_ptr, int, int, char *);
static void print_strings (const char *, FILE *, file_ptr, int, char *);
static void usage (FILE *, int) ATTRIBUTE_NORETURN;
int main (int, char **);
@ -171,7 +201,7 @@ main (int argc, char **argv)
encoding = 's';
output_separator = NULL;
while ((optc = getopt_long (argc, argv, "adfhHn:wot:e:T:s:Vv0123456789",
while ((optc = getopt_long (argc, argv, "adfhHn:wot:e:T:s:U:Vv0123456789",
long_options, (int *) 0)) != EOF)
{
switch (optc)
@ -244,6 +274,23 @@ main (int argc, char **argv)
output_separator = optarg;
break;
case 'U':
if (streq (optarg, "default") || streq (optarg, "d"))
unicode_display = unicode_default;
else if (streq (optarg, "locale") || streq (optarg, "l"))
unicode_display = unicode_locale;
else if (streq (optarg, "escape") || streq (optarg, "e"))
unicode_display = unicode_escape;
else if (streq (optarg, "invalid") || streq (optarg, "i"))
unicode_display = unicode_invalid;
else if (streq (optarg, "hex") || streq (optarg, "x"))
unicode_display = unicode_hex;
else if (streq (optarg, "highlight") || streq (optarg, "h"))
unicode_display = unicode_highlight;
else
fatal (_("invalid argument to -U/--unicode: %s"), optarg);
break;
case 'V':
case 'v':
print_version ("strings");
@ -258,6 +305,9 @@ main (int argc, char **argv)
}
}
if (unicode_display != unicode_default)
encoding = 'S';
if (numeric_opt != 0)
{
string_min = (int) strtoul (argv[numeric_opt - 1] + 1, &s, 0);
@ -293,14 +343,14 @@ main (int argc, char **argv)
{
datasection_only = false;
SET_BINARY (fileno (stdin));
print_strings ("{standard input}", stdin, 0, 0, 0, (char *) NULL);
print_strings ("{standard input}", stdin, 0, 0, (char *) NULL);
files_given = true;
}
else
{
for (; optind < argc; ++optind)
{
if (strcmp (argv[optind], "-") == 0)
if (streq (argv[optind], "-"))
datasection_only = false;
else
{
@ -342,7 +392,7 @@ strings_a_section (bfd *abfd, asection *sect, const char *filename,
}
*got_a_section = true;
print_strings (filename, NULL, sect->filepos, 0, sectsize, (char *) mem);
print_strings (filename, NULL, sect->filepos, sectsize, (char *) mem);
free (mem);
}
@ -427,7 +477,7 @@ strings_file (char *file)
return false;
}
print_strings (file, stream, (file_ptr) 0, 0, 0, (char *) 0);
print_strings (file, stream, (file_ptr) 0, 0, (char *) NULL);
if (fclose (stream) == EOF)
{
@ -551,11 +601,627 @@ unget_part_char (long c, file_ptr *address, int *magiccount, char **magic)
}
}
}
static void
print_filename_and_address (const char * filename, file_ptr address)
{
if (print_filenames)
printf ("%s: ", filename);
if (! print_addresses)
return;
switch (address_radix)
{
case 8:
if (sizeof (address) > sizeof (long))
{
#ifndef __MSVCRT__
printf ("%7llo ", (unsigned long long) address);
#else
printf ("%7I64o ", (unsigned long long) address);
#endif
}
else
printf ("%7lo ", (unsigned long) address);
break;
case 10:
if (sizeof (address) > sizeof (long))
{
#ifndef __MSVCRT__
printf ("%7llu ", (unsigned long long) address);
#else
printf ("%7I64d ", (unsigned long long) address);
#endif
}
else
printf ("%7ld ", (long) address);
break;
case 16:
if (sizeof (address) > sizeof (long))
{
#ifndef __MSVCRT__
printf ("%7llx ", (unsigned long long) address);
#else
printf ("%7I64x ", (unsigned long long) address);
#endif
}
else
printf ("%7lx ", (unsigned long) address);
break;
}
}
/* Return non-zero if the bytes starting at BUFFER form a valid UTF-8 encoding.
If the encoding is valid then returns the number of bytes it uses. */
static unsigned int
is_valid_utf8 (const unsigned char * buffer, unsigned long buflen)
{
if (buffer[0] < 0xc0)
return 0;
if (buflen < 2)
return 0;
if ((buffer[1] & 0xc0) != 0x80)
return 0;
if ((buffer[0] & 0x20) == 0)
return 2;
if (buflen < 3)
return 0;
if ((buffer[2] & 0xc0) != 0x80)
return 0;
if ((buffer[0] & 0x10) == 0)
return 3;
if (buflen < 4)
return 0;
if ((buffer[3] & 0xc0) != 0x80)
return 0;
return 4;
}
/* Display a UTF-8 encoded character in BUFFER according to the setting
of unicode_display. The character is known to be valid.
Returns the number of bytes consumed. */
static uint
display_utf8_char (const unsigned char * buffer)
{
uint j;
uint utf8_len;
switch (buffer[0] & 0x30)
{
case 0x00:
case 0x10:
utf8_len = 2;
break;
case 0x20:
utf8_len = 3;
break;
default:
utf8_len = 4;
}
switch (unicode_display)
{
default:
fprintf (stderr, "ICE: unexpected unicode display type\n");
break;
case unicode_escape:
case unicode_highlight:
if (unicode_display == unicode_highlight && isatty (1))
printf ("\x1B[31;47m"); /* Red. */
switch (utf8_len)
{
case 2:
printf ("\\u%02x%02x",
((buffer[0] & 0x1c) >> 2),
((buffer[0] & 0x03) << 6) | (buffer[1] & 0x3f));
break;
case 3:
printf ("\\u%02x%02x",
((buffer[0] & 0x0f) << 4) | ((buffer[1] & 0x3c) >> 2),
((buffer[1] & 0x03) << 6) | ((buffer[2] & 0x3f)));
break;
case 4:
printf ("\\u%02x%02x%02x",
((buffer[0] & 0x07) << 6) | ((buffer[1] & 0x3c) >> 2),
((buffer[1] & 0x03) << 6) | ((buffer[2] & 0x3c) >> 2),
((buffer[2] & 0x03) << 6) | ((buffer[3] & 0x3f)));
break;
default:
/* URG. */
break;
}
if (unicode_display == unicode_highlight && isatty (1))
printf ("\033[0m"); /* Default colour. */
break;
case unicode_hex:
putchar ('<');
printf ("0x");
for (j = 0; j < utf8_len; j++)
printf ("%02x", buffer [j]);
putchar ('>');
break;
case unicode_locale:
printf ("%.1s", buffer);
break;
}
return utf8_len;
}
/* Display strings in BUFFER. Treat any UTF-8 encoded characters encountered
according to the setting of the unicode_display variable. The buffer
contains BUFLEN bytes.
Display the characters as if they started at ADDRESS and are contained in
FILENAME. */
static void
print_unicode_buffer (const char * filename,
file_ptr address,
const unsigned char * buffer,
unsigned long buflen)
{
/* Paranoia checks... */
if (filename == NULL
|| buffer == NULL
|| unicode_display == unicode_default
|| encoding != 'S'
|| encoding_bytes != 1)
{
fprintf (stderr, "ICE: bad arguments to print_unicode_buffer\n");
return;
}
if (buflen == 0)
return;
/* We must only display strings that are at least string_min *characters*
long. So we scan the buffer in two stages. First we locate the start
of a potential string. Then we walk along it until we have found
string_min characters. Then we go back to the start point and start
displaying characters according to the unicode_display setting. */
unsigned long start_point = 0;
unsigned long i = 0;
unsigned int char_len = 1;
unsigned int num_found = 0;
for (i = 0; i < buflen; i += char_len)
{
int c = buffer[i];
char_len = 1;
/* Find the first potential character of a string. */
if (! STRING_ISGRAPHIC (c))
{
num_found = 0;
continue;
}
if (c > 126)
{
if (c < 0xc0)
{
num_found = 0;
continue;
}
if ((char_len = is_valid_utf8 (buffer + i, buflen - i)) == 0)
{
char_len = 1;
num_found = 0;
continue;
}
if (unicode_display == unicode_invalid)
{
/* We have found a valid UTF-8 character, but we treat it as non-graphic. */
num_found = 0;
continue;
}
}
if (num_found == 0)
/* We have found a potential starting point for a string. */
start_point = i;
++ num_found;
if (num_found >= string_min)
break;
}
if (num_found < string_min)
return;
print_filename_and_address (filename, address + start_point);
/* We have found string_min characters. Display them and any
more that follow. */
for (i = start_point; i < buflen; i += char_len)
{
int c = buffer[i];
char_len = 1;
if (! STRING_ISGRAPHIC (c))
break;
else if (c < 127)
putchar (c);
else if (! is_valid_utf8 (buffer + i, buflen - i))
break;
else if (unicode_display == unicode_invalid)
break;
else
char_len = display_utf8_char (buffer + i);
}
if (output_separator)
fputs (output_separator, stdout);
else
putchar ('\n');
/* FIXME: Using tail recursion here is lazy programming... */
print_unicode_buffer (filename, address + i, buffer + i, buflen - i);
}
static int
get_unicode_byte (FILE * stream, unsigned char * putback, uint * num_putback, uint * num_read)
{
if (* num_putback > 0)
{
* num_putback = * num_putback - 1;
return putback [* num_putback];
}
* num_read = * num_read + 1;
#if defined(HAVE_GETC_UNLOCKED) && HAVE_DECL_GETC_UNLOCKED
return getc_unlocked (stream);
#else
return getc (stream);
#endif
}
/* Helper function for print_unicode_stream. */
static void
print_unicode_stream_body (const char * filename,
file_ptr address,
FILE * stream,
unsigned char * putback_buf,
uint num_putback,
unsigned char * print_buf)
{
/* It would be nice if we could just read the stream into a buffer
and then process if with print_unicode_buffer. But the input
might be huge or it might time-locked (eg stdin). So instead
we go one byte at a time... */
file_ptr start_point = 0;
uint num_read = 0;
uint num_chars = 0;
uint num_print = 0;
int c;
/* Find a series of string_min characters. Put them into print_buf. */
do
{
if (num_chars >= string_min)
break;
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
if (! STRING_ISGRAPHIC (c))
{
num_chars = num_print = 0;
continue;
}
if (num_chars == 0)
start_point = num_read - 1;
if (c < 127)
{
print_buf[num_print] = c;
num_chars ++;
num_print ++;
continue;
}
if (c < 0xc0)
{
num_chars = num_print = 0;
continue;
}
/* We *might* have a UTF-8 sequence. Time to start peeking. */
char utf8[4];
utf8[0] = c;
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
utf8[1] = c;
if ((utf8[1] & 0xc0) != 0x80)
{
/* Invalid UTF-8. */
putback_buf[num_putback++] = utf8[1];
num_chars = num_print = 0;
continue;
}
else if ((utf8[0] & 0x20) == 0)
{
/* A valid 2-byte UTF-8 encoding. */
if (unicode_display == unicode_invalid)
{
putback_buf[num_putback++] = utf8[1];
num_chars = num_print = 0;
}
else
{
print_buf[num_print ++] = utf8[0];
print_buf[num_print ++] = utf8[1];
num_chars ++;
}
continue;
}
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
utf8[2] = c;
if ((utf8[2] & 0xc0) != 0x80)
{
/* Invalid UTF-8. */
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
num_chars = num_print = 0;
continue;
}
else if ((utf8[0] & 0x10) == 0)
{
/* A valid 3-byte UTF-8 encoding. */
if (unicode_display == unicode_invalid)
{
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
num_chars = num_print = 0;
}
else
{
print_buf[num_print ++] = utf8[0];
print_buf[num_print ++] = utf8[1];
print_buf[num_print ++] = utf8[2];
num_chars ++;
}
continue;
}
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
utf8[3] = c;
if ((utf8[3] & 0xc0) != 0x80)
{
/* Invalid UTF-8. */
putback_buf[num_putback++] = utf8[3];
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
num_chars = num_print = 0;
}
/* We have a valid 4-byte UTF-8 encoding. */
else if (unicode_display == unicode_invalid)
{
putback_buf[num_putback++] = utf8[3];
putback_buf[num_putback++] = utf8[1];
putback_buf[num_putback++] = utf8[2];
num_chars = num_print = 0;
}
else
{
print_buf[num_print ++] = utf8[0];
print_buf[num_print ++] = utf8[1];
print_buf[num_print ++] = utf8[2];
print_buf[num_print ++] = utf8[3];
num_chars ++;
}
}
while (1);
if (num_chars >= string_min)
{
/* We know that we have string_min valid characters in print_buf,
and there may be more to come in the stream. Start displaying
them. */
print_filename_and_address (filename, address + start_point);
uint i;
for (i = 0; i < num_print;)
{
if (print_buf[i] < 127)
putchar (print_buf[i++]);
else
i += display_utf8_char (print_buf + i);
}
/* OK so now we have to start read unchecked bytes. */
/* Find a series of string_min characters. Put them into print_buf. */
do
{
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
if (! STRING_ISGRAPHIC (c))
break;
if (c < 127)
{
putchar (c);
continue;
}
if (c < 0xc0)
break;
/* We *might* have a UTF-8 sequence. Time to start peeking. */
unsigned char utf8[4];
utf8[0] = c;
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
utf8[1] = c;
if ((utf8[1] & 0xc0) != 0x80)
{
/* Invalid UTF-8. */
putback_buf[num_putback++] = utf8[1];
break;
}
else if ((utf8[0] & 0x20) == 0)
{
/* Valid 2-byte UTF-8. */
if (unicode_display == unicode_invalid)
{
putback_buf[num_putback++] = utf8[1];
break;
}
else
{
(void) display_utf8_char (utf8);
continue;
}
}
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
utf8[2] = c;
if ((utf8[2] & 0xc0) != 0x80)
{
/* Invalid UTF-8. */
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
break;
}
else if ((utf8[0] & 0x10) == 0)
{
/* Valid 3-byte UTF-8. */
if (unicode_display == unicode_invalid)
{
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
break;
}
else
{
(void) display_utf8_char (utf8);
continue;
}
}
c = get_unicode_byte (stream, putback_buf, & num_putback, & num_read);
if (c == EOF)
break;
utf8[3] = c;
if ((utf8[3] & 0xc0) != 0x80)
{
/* Invalid UTF-8. */
putback_buf[num_putback++] = utf8[3];
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
break;
}
else if (unicode_display == unicode_invalid)
{
putback_buf[num_putback++] = utf8[3];
putback_buf[num_putback++] = utf8[2];
putback_buf[num_putback++] = utf8[1];
break;
}
else
/* A valid 4-byte UTF-8 encoding. */
(void) display_utf8_char (utf8);
}
while (1);
if (output_separator)
fputs (output_separator, stdout);
else
putchar ('\n');
}
if (c != EOF)
/* FIXME: Using tail recursion here is lazy, but it works. */
print_unicode_stream_body (filename, address + num_read, stream, putback_buf, num_putback, print_buf);
}
/* Display strings read in from STREAM. Treat any UTF-8 encoded characters
encountered according to the setting of the unicode_display variable.
The stream is positioned at ADDRESS and is attached to FILENAME. */
static void
print_unicode_stream (const char * filename,
file_ptr address,
FILE * stream)
{
/* Paranoia checks... */
if (filename == NULL
|| stream == NULL
|| unicode_display == unicode_default
|| encoding != 'S'
|| encoding_bytes != 1)
{
fprintf (stderr, "ICE: bad arguments to print_unicode_stream\n");
return;
}
/* Allocate space for string_min 4-byte utf-8 characters. */
unsigned char * print_buf = xmalloc ((4 * string_min) + 1);
/* We should never have to put back more than 4 bytes. */
unsigned char putback_buf[5];
uint num_putback = 0;
print_unicode_stream_body (filename, address, stream, putback_buf, num_putback, print_buf);
free (print_buf);
}
/* Find the strings in file FILENAME, read from STREAM.
Assume that STREAM is positioned so that the next byte read
is at address ADDRESS in the file.
Stop reading at address STOP_POINT in the file, if nonzero.
If STREAM is NULL, do not read from it.
The caller can supply a buffer of characters
@ -566,20 +1232,29 @@ unget_part_char (long c, file_ptr *address, int *magiccount, char **magic)
static void
print_strings (const char *filename, FILE *stream, file_ptr address,
int stop_point, int magiccount, char *magic)
int magiccount, char *magic)
{
if (unicode_display != unicode_default)
{
if (magic != NULL)
print_unicode_buffer (filename, address,
(const unsigned char *) magic, magiccount);
if (stream != NULL)
print_unicode_stream (filename, address, stream);
return;
}
char *buf = (char *) xmalloc (sizeof (char) * (string_min + 1));
while (1)
{
file_ptr start;
int i;
uint i;
long c;
/* See if the next `string_min' chars are all graphic chars. */
tryline:
if (stop_point && address >= stop_point)
break;
start = address;
for (i = 0; i < string_min; i++)
{
@ -601,51 +1276,7 @@ print_strings (const char *filename, FILE *stream, file_ptr address,
/* We found a run of `string_min' graphic characters. Print up
to the next non-graphic character. */
if (print_filenames)
printf ("%s: ", filename);
if (print_addresses)
switch (address_radix)
{
case 8:
if (sizeof (start) > sizeof (long))
{
#ifndef __MSVCRT__
printf ("%7llo ", (unsigned long long) start);
#else
printf ("%7I64o ", (unsigned long long) start);
#endif
}
else
printf ("%7lo ", (unsigned long) start);
break;
case 10:
if (sizeof (start) > sizeof (long))
{
#ifndef __MSVCRT__
printf ("%7llu ", (unsigned long long) start);
#else
printf ("%7I64d ", (unsigned long long) start);
#endif
}
else
printf ("%7ld ", (long) start);
break;
case 16:
if (sizeof (start) > sizeof (long))
{
#ifndef __MSVCRT__
printf ("%7llx ", (unsigned long long) start);
#else
printf ("%7I64x ", (unsigned long long) start);
#endif
}
else
printf ("%7lx ", (unsigned long) start);
break;
}
print_filename_and_address (filename, start);
buf[i] = '\0';
fputs (buf, stdout);
@ -697,6 +1328,8 @@ usage (FILE *stream, int status)
-T --target=<BFDNAME> Specify the binary file format\n\
-e --encoding={s,S,b,l,B,L} Select character size and endianness:\n\
s = 7-bit, S = 8-bit, {b,l} = 16-bit, {B,L} = 32-bit\n\
--unicode={default|show|invalid|hex|escape|highlight}\n\
-u {d|s|i|x|e|h} Specify how to treat UTF-8 encoded unicode characters\n\
-s --output-separator=<string> String used to separate strings in output.\n\
@<file> Read options from <file>\n\
-h --help Display this information\n\