added qse_wcsatombsdup()/qse_mbsatowcsdup()

added macro redefinition for QSE_MBLEN_MAX in some special cases
fixed a bug of defining QSE_TOMUPPER and QSE_TOMLOWER wrongly
changed data types of utf8 functions.
fixed null-terminating bugs in qse_mbstowcs()/qse_wcstombs()

qse/lib/cmn/utf8.c

@@ -38,6 +38,7 @@ struct __utf8_t
 	qse_uint32_t  upper;
 	qse_uint8_t   fbyte;  /* mask to the first utf8 byte */
 	qse_uint8_t   mask;
+	qse_uint8_t   fmask;
 	int           length; /* number of bytes */
 };
 
@@ -45,12 +46,12 @@ typedef struct __utf8_t __utf8_t;
 
 static __utf8_t utf8_table[] = 
 {
-	{0x00000000ul, 0x0000007Ful, 0x00, 0x80, 1},
-	{0x00000080ul, 0x000007FFul, 0xC0, 0xE0, 2},
-	{0x00000800ul, 0x0000FFFFul, 0xE0, 0xF0, 3},
-	{0x00010000ul, 0x001FFFFFul, 0xF0, 0xF8, 4},
-	{0x00200000ul, 0x03FFFFFFul, 0xF8, 0xFC, 5},
-	{0x04000000ul, 0x7FFFFFFFul, 0xFC, 0xFE, 6}
+	{0x00000000ul, 0x0000007Ful, 0x00, 0x80, 0x7F, 1},
+	{0x00000080ul, 0x000007FFul, 0xC0, 0xE0, 0x1F, 2},
+	{0x00000800ul, 0x0000FFFFul, 0xE0, 0xF0, 0x0F, 3},
+	{0x00010000ul, 0x001FFFFFul, 0xF0, 0xF8, 0x07, 4},
+	{0x00200000ul, 0x03FFFFFFul, 0xF8, 0xFC, 0x03, 5},
+	{0x04000000ul, 0x7FFFFFFFul, 0xFC, 0xFE, 0x01, 6}
 };
 
 static QSE_INLINE __utf8_t* get_utf8_slot (qse_wchar_t uc)
@@ -72,23 +73,25 @@ static QSE_INLINE __utf8_t* get_utf8_slot (qse_wchar_t uc)
 	return QSE_NULL; /* invalid character */
 }
 
-int qse_uctoutf8len (qse_wchar_t uc)
+qse_size_t qse_uctoutf8len (qse_wchar_t uc)
 {
 	__utf8_t* cur = get_utf8_slot (uc);
-	return (cur == QSE_NULL)? 0: cur->length;
+	return (cur == QSE_NULL)? 0: (qse_size_t)cur->length;
 }
 
-int qse_uctoutf8 (qse_wchar_t uc, qse_mchar_t* utf8, int size)
+/* wctomb for utf8/unicode */
+qse_size_t qse_uctoutf8 (qse_wchar_t uc, qse_mchar_t* utf8, qse_size_t size)
 {
 	__utf8_t* cur = get_utf8_slot (uc);
 	int index;
 
-	if (cur == QSE_NULL) return 0; /* invalid character */
+	if (cur == QSE_NULL) return 0; /* illegal character */
 
 	if (cur->length > size)
 	{
-		/* buffer not big enough. index indicates the buffer size needed */
-		return -index;
+		/* buffer not big enough. index indicates the buffer 
+		 * size needed */
+		return size + 1;
 	}
 
 	index = cur->length;
@@ -103,16 +106,18 @@ int qse_uctoutf8 (qse_wchar_t uc, qse_mchar_t* utf8, int size)
 	}
 
 	utf8[0] = uc | cur->fbyte;
-	return cur->length;
+	return (qse_size_t)cur->length;
 }
 
-int qse_utf8touc (
-	const qse_mchar_t* utf8, int size, qse_wchar_t* uc)
+/* mbtowc for utf8/unicode */
+qse_size_t qse_utf8touc (
+	const qse_mchar_t* utf8, qse_size_t size, qse_wchar_t* uc)
 {
 	__utf8_t* cur, * end;
+#if 0
 	qse_mchar_t c, t;
-	qse_wchar_t w;
 	int count = 0;
+#endif
 
 	QSE_ASSERT (utf8 != QSE_NULL);
 	QSE_ASSERT (QSE_SIZEOF(qse_mchar_t) == 1);
@@ -120,7 +125,32 @@ int qse_utf8touc (
 
 	end = utf8_table + QSE_COUNTOF(utf8_table);
 	cur = utf8_table;
+
+	while (cur < end) 
+	{
+		if ((utf8[0] & cur->mask) == cur->fbyte) 
+		{
+			int i;
+			qse_wchar_t w;
+
+			if (size < cur->length) return size + 1; 
+
+			w = utf8[0] & cur->fmask;
+			for (i = 1; i < cur->length; i++)
+			{
+				if (!(utf8[i] & 0x80)) return 0; 
+				w = (w << 6) | (utf8[i] & 0x3F);
+			}
+
+			*uc = w;
+			return (qse_size_t)cur->length;
+		}
+		cur++;
+	}
+
+	return 0; /* error - invalid sequence */
 	
+#if 0
 	c = *utf8;
 	w = c;
 
@@ -133,7 +163,7 @@ int qse_utf8touc (
 			w &= cur->upper;
 			if (w < cur->lower) break; /* wrong value */
 			*uc = w;
-			return count;
+			return (qse_size_t)count;
 		}
 
 		if (size <= count) break; /* insufficient input */
@@ -147,10 +177,11 @@ int qse_utf8touc (
 	}
 
 	return 0; /* error - invalid sequence */
+#endif
 }
 
-#if 0
-int qse_utf8len (qse_mchar_t first)
+/* mblen for utf8 */
+qse_size_t qse_utf8len (const qse_mchar_t* utf8, qse_size_t len)
 {
 	__utf8_t* cur, * end;
 
@@ -159,10 +190,32 @@ int qse_utf8len (qse_mchar_t first)
 
 	while (cur < end) 
 	{
-		if ((first & cur->mask) == cur->fbyte) return cur->length;
+		if ((utf8[0] & cur->mask) == cur->fbyte) 
+		{
+			int i;
+
+			if (len < cur->length) 
+			{
+				/* the input is not as large as 
+				 * the expected number of bytes */
+				return len + 1; /* incomplete sequence */
+			}
+
+			for (i = 1; i < cur->length; i++)
+			{
+				/* in utf8, trailing bytes are all
+				 * set with 0x80. if not, invalid */
+				if (!(utf8[i] & 0x80)) return 0; 
+			}
+			return (qse_size_t)cur->length;
+		}
 		cur++;
 	}
 
 	return 0; /* error - invalid sequence */
 }
-#endif
+
+qse_size_t qse_utf8lenmax (void)
+{
+	return QSE_UTF8LEN_MAX;
+}