Merge pull request #35 from ScottPJones/spj/valid

Fix #34 handle 66 Unicode non-characters and surrogates correctly

5 files changed, 212 insertions, 8 deletions

diff --git a/test/charwidth.c b/test/charwidth.c
index bdae9fa..28554a4 100644
--- a/test/charwidth.c
+++ b/test/charwidth.c
@@ -10,7 +10,7 @@ int my_isprint(int c) {
 
 int main(int argc, char **argv)
 {
-     int prevc, c, error = 0;
+     int c, error = 0;
 
      (void) argc; /* unused */
      (void) argv; /* unused */
diff --git a/test/graphemetest.c b/test/graphemetest.c
index a05b6e2..eb3645b 100644
--- a/test/graphemetest.c
+++ b/test/graphemetest.c
@@ -6,6 +6,7 @@ int main(int argc, char **argv)
     size_t bufsize = 0;
     FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
     utf8proc_uint8_t src[1024];
+    int len;
     
     check(f != NULL, "error opening GraphemeBreakTest.txt");
     while (getline(&buf, &bufsize, f) > 0) {
@@ -29,9 +30,10 @@ int main(int argc, char **argv)
             else if (buf[bi] == '#') { /* start of comments */
                 break;
             }
-            else { /* hex-encoded codepoint */
-                bi += encode((char*) (src + si), buf + bi) - 1;
+	    else { /* hex-encoded codepoint */
+                len = encode((char*) (src + si), buf + bi) - 1;
                 while (src[si]) ++si; /* advance to NUL termination */
+                bi += len;
             }
         }
         if (si && src[si-1] == '/')
@@ -59,7 +61,7 @@ int main(int argc, char **argv)
                        utf8proc_errmsg(glen));
                  for (i = 0; i <= glen; ++i)
                       if (g[i] == 0xff)
-                           g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
+                          g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
                  check(!strcmp((char*)g, (char*)src),
                        "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
             }
diff --git a/test/iterate.c b/test/iterate.c
new file mode 100644
index 0000000..cd68a52
--- /dev/null
+++ b/test/iterate.c
@@ -0,0 +1,160 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static  int     tests;
+static  int     error;
+
+#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
+#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
+
+void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
+{
+    utf8proc_int32_t out[16];
+    utf8proc_ssize_t ret;
+
+    tests++;
+    if ((ret = utf8proc_iterate(buf, len, out)) != retval) {
+        fprintf(stderr, "Failed (%d):", line);
+        for (int i = 0; i < len ; i++) {
+            fprintf(stderr, " 0x%02x", buf[i]);
+        }
+        fprintf(stderr, " -> %zd\n", ret);
+        error++;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    uint32_t byt;
+    unsigned char buf[16];
+
+    tests = error = 0;
+
+    // Check valid sequences that were considered valid erroneously before
+    buf[0] = 0xef;
+    buf[1] = 0xb7;
+    for (byt = 0x90; byt < 0xa0; byt++) {
+        CHECKVALID(2, byt, 3);
+    }
+    // Check 0xfffe and 0xffff
+    buf[1] = 0xbf;
+    CHECKVALID(2, 0xbe, 3);
+    CHECKVALID(2, 0xbf, 3);
+    // Check 0x??fffe & 0x??ffff
+    for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
+        buf[0] = 0xf0 | (byt >> 18);
+        buf[1] = 0x80 | ((byt >> 12) & 0x3f);
+        CHECKVALID(3, 0xbe, 4);
+        CHECKVALID(3, 0xbf, 4);
+    }
+    
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Test lead bytes
+    for (byt = 0xc0; byt <= 0xff; byt++) {
+	// Single lead byte at end of string
+        CHECKINVALID(0, byt, 1);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(1, 65, 2);
+	// Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(1, 0xc0, 2);
+    }
+
+    // Test overlong 2-byte
+    buf[0] = 0xc0;
+    for (byt = 0x81; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+    buf[0] = 0xc1;
+    for (byt = 0x80; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+
+    // Test overlong 3-byte
+    buf[0] = 0xe0;
+    buf[2] = 0x80;
+    for (byt = 0x80; byt <= 0x9f; byt++) {
+        CHECKINVALID(1, byt, 3);
+    }
+
+    // Test overlong 4-byte
+    buf[0] = 0xf0;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x80; byt <= 0x8f; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+
+    // Test 4-byte > 0x10ffff
+    buf[0] = 0xf4;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x90; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+    buf[1] = 0x80;
+    for (byt = 0xf5; byt <= 0xf7; byt++) {
+        CHECKINVALID(0, byt, 4);
+    }
+
+    // Test 5-byte
+    buf[4] = 0x80;
+    for (byt = 0xf8; byt <= 0xfb; byt++) {
+        CHECKINVALID(0, byt, 5);
+    }
+
+    // Test 6-byte
+    buf[5] = 0x80;
+    for (byt = 0xfc; byt <= 0xfd; byt++) {
+        CHECKINVALID(0, byt, 6);
+    }
+
+    // Test 7-byte
+    buf[6] = 0x80;
+    CHECKINVALID(0, 0xfe, 7);
+
+    // Three and above byte sequences
+    for (byt = 0xe0; byt < 0xf0; byt++) {
+        // Lead followed by only 1 continuation byte
+        CHECKINVALID(0, byt, 2);
+        // Lead ended by non-continuation character < 0x80
+        CHECKINVALID(2, 65, 3);
+        // Lead ended by non-continuation character > 0xbf
+        CHECKINVALID(2, 0xc0, 3);
+    }
+
+    // 3-byte encoded surrogate character(s)
+    buf[0] = 0xed; buf[2] = 0x80;
+    // Single surrogate
+    CHECKINVALID(1, 0xa0, 3);
+    // Trailing surrogate first
+    CHECKINVALID(1, 0xb0, 3);
+
+    // Four byte sequences
+    buf[1] = 0x80;
+    for (byt = 0xf0; byt < 0xf5; byt++) {
+        // Lead followed by only 1 continuation bytes
+        CHECKINVALID(0, byt, 2);
+        // Lead followed by only 2 continuation bytes
+        CHECKINVALID(0, byt, 3);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(3, 65, 4);
+        // Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(3, 0xc0, 4);
+
+    }
+
+     check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
+     printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
+
+     return 0;
+}
diff --git a/test/tests.h b/test/tests.h
index c27185d..6eb5457 100644
--- a/test/tests.h
+++ b/test/tests.h
@@ -33,10 +33,11 @@ size_t skipspaces(const char *buf, size_t i)
    separated by whitespace, and terminated by any character not in
    [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
    in dest, returning the number of bytes read from buf */
+utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst);
 size_t encode(char *dest, const char *buf)
 {
      size_t i = 0, j, d = 0;
-     do {
+     for (;;) {
           int c;
           i = skipspaces(buf, i);
           for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
@@ -45,9 +46,9 @@ size_t encode(char *dest, const char *buf)
                dest[d] = 0; /* NUL-terminate destination string */
                return i + 1;
           }
-          check(sscanf(buf + i, "%x", &c) == 1, "invalid hex input %s", buf+i);
+          check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
           i = j; /* skip to char after hex input */
-          d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
-     } while (1);
+          d += unsafe_encode_char(c, (utf8proc_uint8_t *) (dest + d));
+     }
 }
 
diff --git a/test/valid.c b/test/valid.c
new file mode 100644
index 0000000..eadfb85
--- /dev/null
+++ b/test/valid.c
@@ -0,0 +1,41 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+int main(int argc, char **argv)
+{
+     int c, error = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of  */
+     for (c = 0; c < 0xd800; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0xe000; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110000; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110010; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
+           error++;
+        }
+     }
+     check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
+     printf("Validity tests SUCCEEDED.\n");
+
+     return 0;
+}