mirror of
https://github.com/PCRE2Project/pcre2.git
synced 2025-10-16 22:35:45 +08:00
6915395e90
JIT has several good uses of unsigned integer wraparounds, that clang's UBSAN doesn't like (which is controversial, because it is clearly not undefined behaviour), but since it is usually good to know when they happen by accident it makes sense to make sure the rest of PCRE2 codebase benefits from checking it. While at it, upgrade the version of the base image to use a newer OS as a canary from when the rest of the jobs upgrade themselves and be a little more strict to catch other constructs that are not welcomed in our codebase.
348 lines
8.4 KiB
C
348 lines
8.4 KiB
C
/****************************************************
|
|
* PCRE maintainers' helper program: UTF-8 converter *
|
|
****************************************************/
|
|
|
|
/* This is a test program for converting character code points to UTF-8 and
|
|
vice versa. Note that this program conforms to the original definition of
|
|
UTF-8, which allows codepoints up to 7fffffff. The more recent definition
|
|
limits the validity of Unicode UTF-8 codepoints to a maximum of 10ffffff, and
|
|
forbids the "surrogate" code points. This program now gives warnings for these
|
|
invalid code points.
|
|
|
|
The arguments are either single code point values written as U+hh.. or 0xhh..
|
|
for conversion to UTF-8, or sequences of hex values, written without 0x and
|
|
optionally including spaces (but such arguments must be quoted), for conversion
|
|
from UTF-8 to codepoints. For example:
|
|
|
|
./utf8 0x1234
|
|
U+00001234 => e1 88 b4
|
|
|
|
./utf8 "e1 88 b4"
|
|
U+00001234 <= e1 88 b4
|
|
|
|
In the second case, a number of UTF-8 characters can be present in one
|
|
argument. In other words, each such argument is interpreted (after ignoring
|
|
spaces) as a string of UTF-8 bytes representing a string of characters:
|
|
|
|
./utf8 "65 e188b4 77"
|
|
0x00000065 <= 65
|
|
0x00001234 <= e1 88 b4
|
|
0x00000077 <= 77
|
|
|
|
If the option -s is given, the sequence of UTF-bytes is written out between
|
|
angle brackets at the end of the line. On a UTF-8 terminal, this will show the
|
|
appropriate graphic for the code point.
|
|
|
|
Errors provoke error messages, but the program carries on with the next
|
|
argument. The return code is always zero.
|
|
|
|
Philip Hazel
|
|
Original creation data: unknown
|
|
Code extended and tidied to avoid compiler warnings: 26 March 2020
|
|
*/
|
|
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <ctype.h>
|
|
#include <string.h>
|
|
|
|
/* The valid ranges for UTF-8 characters are:
|
|
|
|
0000 0000 to 0000 007f 1 byte (ascii)
|
|
0000 0080 to 0000 07ff 2 bytes
|
|
0000 0800 to 0000 ffff 3 bytes
|
|
0001 0000 to 001f ffff 4 bytes
|
|
0020 0000 to 03ff ffff 5 bytes
|
|
0400 0000 to 7fff ffff 6 bytes
|
|
*/
|
|
|
|
|
|
static const unsigned int utf8_table1[] = {
|
|
0x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff};
|
|
|
|
static const int utf8_table2[] = {
|
|
0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc};
|
|
|
|
static const int utf8_table3[] = {
|
|
0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
|
|
|
|
|
|
/*************************************************
|
|
* Convert character value to UTF-8 *
|
|
*************************************************/
|
|
|
|
/* This function takes an unsigned long integer value in the range 0 -
|
|
0x7fffffff and encodes it as a UTF-8 character in 1 to 6 bytes.
|
|
|
|
Arguments:
|
|
cvalue the character value
|
|
buffer pointer to buffer for result - at least 6 bytes long
|
|
|
|
Returns: number of bytes placed in the buffer
|
|
0 if input code point is too big
|
|
*/
|
|
|
|
static size_t
|
|
ord2utf8(unsigned long int cvalue, unsigned char *buffer)
|
|
{
|
|
size_t i, j;
|
|
for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++)
|
|
if (cvalue <= utf8_table1[i]) break;
|
|
if (i >= sizeof(utf8_table1)/sizeof(int)) return 0;
|
|
buffer += i;
|
|
for (j = i; j > 0; j--)
|
|
{
|
|
*buffer-- = 0x80 | (cvalue & 0x3f);
|
|
cvalue >>= 6;
|
|
}
|
|
*buffer = utf8_table2[i] | cvalue;
|
|
return i + 1;
|
|
}
|
|
|
|
|
|
|
|
/*************************************************
|
|
* Convert UTF-8 string to value *
|
|
*************************************************/
|
|
|
|
/* This function takes one or more bytes that represent a UTF-8 character from
|
|
the start of a string of bytes. It returns the value of the character, or the
|
|
offset of a malformation. For an overlong encoding that works but is not the
|
|
correct (shortest) one, the error offset is just after the last byte.
|
|
|
|
Argument:
|
|
buffer a pointer to the byte vector
|
|
buffend a pointer to the end of the buffer
|
|
vptr a pointer to a variable to receive the value
|
|
lenptr a pointer to a variable to receive the offset when error detected
|
|
|
|
Returns: > 0 => the number of bytes consumed
|
|
0 => invalid UTF-8: first byte missing 0x40 bit
|
|
-1 => invalid UTF-8: first byte has too many high-order 1-bits
|
|
-2 => incomplete sequence at end of string
|
|
-3 => incomplete sequence within string
|
|
-4 => overlong code sequence
|
|
*/
|
|
|
|
static int
|
|
utf82ord(unsigned char *buffer, unsigned char *buffend,
|
|
long unsigned int *vptr, int *lenptr)
|
|
{
|
|
unsigned int c = *buffer++;
|
|
unsigned int d = c;
|
|
int i, j, s;
|
|
|
|
/* Check for an ASCII character, or find the number of additional bytes in a
|
|
multibyte character. */
|
|
|
|
for (i = -1; i < 6; i++)
|
|
{
|
|
if ((d & 0x80) == 0) break;
|
|
d <<= 1;
|
|
}
|
|
|
|
switch (i)
|
|
{
|
|
case -1: /* ASCII character; first byte does not have 0x80 bit */
|
|
*vptr = c;
|
|
return 1;
|
|
|
|
case 0: /* First byte has 0x80 but is missing 0x40 bit */
|
|
*lenptr = 0;
|
|
return 0;
|
|
|
|
case 6:
|
|
*lenptr = 0; /* Too many high bits */
|
|
return -1;
|
|
|
|
default:
|
|
break;
|
|
}
|
|
|
|
/* i now has a value in the range 1-5 */
|
|
|
|
s = 6*i;
|
|
d = (c & utf8_table3[i]) << s;
|
|
|
|
for (j = 0; j < i; j++)
|
|
{
|
|
if (buffer >= buffend)
|
|
{
|
|
*lenptr = j + 1;
|
|
return -2;
|
|
}
|
|
c = *buffer++;
|
|
if ((c & 0xc0) != 0x80)
|
|
{
|
|
*lenptr = j + 1;
|
|
return -3;
|
|
}
|
|
s -= 6;
|
|
d |= (c & 0x3f) << s;
|
|
}
|
|
|
|
/* Valid UTF-8 syntax */
|
|
|
|
*vptr = d;
|
|
|
|
/* Check that encoding was the correct one, not overlong */
|
|
|
|
for (j = 0; j < (int)(sizeof(utf8_table1)/sizeof(int)); j++)
|
|
if (d <= utf8_table1[j]) break;
|
|
if (j != i)
|
|
{
|
|
*lenptr = i + 1;
|
|
return -4;
|
|
}
|
|
|
|
/* Valid value */
|
|
|
|
return i + 1;
|
|
}
|
|
|
|
|
|
|
|
/*************************************************
|
|
* Main Program *
|
|
*************************************************/
|
|
|
|
int
|
|
main(int argc, char **argv)
|
|
{
|
|
int i = 1;
|
|
int show = 0;
|
|
unsigned char buffer[64];
|
|
|
|
if (argc > 1 && strcmp(argv[1], "-s") == 0)
|
|
{
|
|
show = 1;
|
|
i = 2;
|
|
}
|
|
|
|
for (; i < argc; i++)
|
|
{
|
|
char *x = argv[i];
|
|
char *endptr;
|
|
if (strncmp(x, "0x", 2) == 0 || strncmp(x, "U+", 2) == 0)
|
|
{
|
|
size_t rc, j;
|
|
unsigned long int d = strtoul(x+2, &endptr, 16);
|
|
if (*endptr != 0)
|
|
{
|
|
printf("** Invalid hex number %s\n", x);
|
|
continue; /* With next argument */
|
|
}
|
|
rc = ord2utf8(d, buffer);
|
|
printf("U+%08lx => ", d);
|
|
if (rc == 0)
|
|
printf("** Code point greater than 0x7fffffff cannot be encoded");
|
|
else
|
|
{
|
|
for (j = 0; j < rc; j++) printf("%02x ", buffer[j]);
|
|
if (show)
|
|
{
|
|
printf(">");
|
|
for (j = 0; j < rc; j++) printf("%c", buffer[j]);
|
|
printf("< ");
|
|
}
|
|
if (d >= 0xd800 && d <= 0xdfff)
|
|
printf("** Invalid Unicode (surrogate)");
|
|
else if (d > 0x10ffff)
|
|
printf("** Invalid Unicode (greater than U+10ffff)");
|
|
}
|
|
printf("\n");
|
|
}
|
|
else
|
|
{
|
|
unsigned char *bptr;
|
|
unsigned char *buffend;
|
|
int len = 0;
|
|
int y = 0;
|
|
int z = 0;
|
|
|
|
for (;;)
|
|
{
|
|
while (*x == ' ') x++;
|
|
if (*x == 0 && !z) break;
|
|
if (!isxdigit(*x))
|
|
{
|
|
printf("** Malformed hex string: %s\n", argv[i]);
|
|
len = -1;
|
|
break;
|
|
}
|
|
y = y * 16 + (tolower(*x) - ((isdigit(*x))? '0' : 'W'));
|
|
x++;
|
|
if (z)
|
|
{
|
|
buffer[len++] = y;
|
|
y = 0;
|
|
}
|
|
z ^= 1;
|
|
}
|
|
|
|
if (len < 0) continue; /* With next argument after malformation */
|
|
|
|
bptr = buffer;
|
|
buffend = buffer + len;
|
|
|
|
while (bptr < buffend)
|
|
{
|
|
unsigned long int d;
|
|
int j;
|
|
int offset;
|
|
int rc = utf82ord(bptr, buffend, &d, &offset);
|
|
|
|
if (rc > 0)
|
|
{
|
|
printf("U+%08lx <= ", d);
|
|
for (j = 0; j < rc; j++) printf("%02x ", bptr[j]);
|
|
if (show)
|
|
{
|
|
printf(">");
|
|
for (j = 0; j < rc; j++) printf("%c", bptr[j]);
|
|
printf("<");
|
|
}
|
|
printf("\n");
|
|
bptr += rc;
|
|
}
|
|
else if (rc == -4)
|
|
{
|
|
printf("U+%08lx <= ", d);
|
|
for (j = 0; j < offset; j++) printf("%02x ", bptr[j]);
|
|
printf("** Overlong UTF-8 sequence\n");
|
|
bptr += offset;
|
|
}
|
|
else
|
|
{
|
|
switch (rc)
|
|
{
|
|
case 0: printf("** First byte missing 0x40 bit");
|
|
break;
|
|
|
|
case -1: printf("** First byte has too many high-order bits");
|
|
break;
|
|
|
|
case -2: printf("** Incomplete UTF-8 sequence at end of string");
|
|
break;
|
|
|
|
case -3: printf("** Incomplete UTF-8 sequence");
|
|
break;
|
|
|
|
default: printf("** Unexpected return %d from utf82ord()", rc);
|
|
break;
|
|
}
|
|
printf(" at offset %d in string ", offset);
|
|
while (bptr < buffend) printf("%02x ", *bptr++);
|
|
printf("\n");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* End */
|