UTFmax increased, Runeerror changed

18 files changed, 68 insertions, 47 deletions

diff --git a/DragonFly/386/include/lib9.h b/DragonFly/386/include/lib9.h
index dc94cf2c..3af6b735 100644
--- a/DragonFly/386/include/lib9.h
+++ b/DragonFly/386/include/lib9.h
@@ -80,10 +80,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/FreeBSD/386/include/lib9.h b/FreeBSD/386/include/lib9.h
index 075978fe..d9c9d3fc 100644
--- a/FreeBSD/386/include/lib9.h
+++ b/FreeBSD/386/include/lib9.h
@@ -81,10 +81,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Irix/mips/include/lib9.h b/Irix/mips/include/lib9.h
index 9881a3a7..8db78d63 100644
--- a/Irix/mips/include/lib9.h
+++ b/Irix/mips/include/lib9.h
@@ -68,10 +68,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Linux/386/include/lib9.h b/Linux/386/include/lib9.h
index 69ff958a..e278d94a 100644
--- a/Linux/386/include/lib9.h
+++ b/Linux/386/include/lib9.h
@@ -41,11 +41,7 @@ typedef struct Proc Proc;
 
 typedef unsigned char	uchar;
 typedef signed char	schar;
-#ifdef RUNE32
 typedef unsigned int Rune;
-#else
-typedef unsigned short	Rune;
-#endif
 typedef long long int	vlong;
 typedef unsigned long long int	uvlong;
 typedef unsigned int u32int;
@@ -79,21 +75,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-#ifdef RUNE32
 	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
 	Runeerror	= 0xFFFD,	/* decoding error in UTF */
 	Runemax		= 0x10FFFF,	/* 21-bit rune */
 	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
-#else
-	UTFmax		= 3,		/* maximum bytes per rune */
-	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
-	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
-	Runemax		= 0xFFFF,	/* 16-bit rune */
-	Runemask	= 0xFFFF,	/* bits used by runes (see grep) */
-#endif
 };
 
 /*
diff --git a/Linux/arm/include/lib9.h b/Linux/arm/include/lib9.h
index 1a6628b1..002e3d15 100644
--- a/Linux/arm/include/lib9.h
+++ b/Linux/arm/include/lib9.h
@@ -75,10 +75,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Linux/power/include/lib9.h b/Linux/power/include/lib9.h
index a9b9dab8..b3d47212 100644
--- a/Linux/power/include/lib9.h
+++ b/Linux/power/include/lib9.h
@@ -76,10 +76,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Linux/spim/include/lib9.h b/Linux/spim/include/lib9.h
index 5a1b36bf..36f121fb 100644
--- a/Linux/spim/include/lib9.h
+++ b/Linux/spim/include/lib9.h
@@ -77,10 +77,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/MacOSX/386/include/lib9.h b/MacOSX/386/include/lib9.h
index 11194063..3bc4d05e 100644
--- a/MacOSX/386/include/lib9.h
+++ b/MacOSX/386/include/lib9.h
@@ -96,10 +96,12 @@ extern	void	qsort(void*, long, long, int (*)(void*, void*));
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/MacOSX/power/include/lib9.h b/MacOSX/power/include/lib9.h
index be609a54..d632ac3a 100644
--- a/MacOSX/power/include/lib9.h
+++ b/MacOSX/power/include/lib9.h
@@ -96,10 +96,12 @@ extern	void	qsort(void*, long, long, int (*)(void*, void*));
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/NetBSD/386/include/lib9.h b/NetBSD/386/include/lib9.h
index 763128fd..df980a40 100644
--- a/NetBSD/386/include/lib9.h
+++ b/NetBSD/386/include/lib9.h
@@ -84,10 +84,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Nt/386/include/lib9.h b/Nt/386/include/lib9.h
index 1485e0c6..85ec1b4c 100755
--- a/Nt/386/include/lib9.h
+++ b/Nt/386/include/lib9.h
@@ -100,10 +100,12 @@ extern	uvlong	strtoull(const char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/OpenBSD/386/include/lib9.h b/OpenBSD/386/include/lib9.h
index db77f784..d1c1ab73 100644
--- a/OpenBSD/386/include/lib9.h
+++ b/OpenBSD/386/include/lib9.h
@@ -81,10 +81,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Plan9/arm/include/lib9.h b/Plan9/arm/include/lib9.h
index e60fba8a..de314ca7 100644
--- a/Plan9/arm/include/lib9.h
+++ b/Plan9/arm/include/lib9.h
@@ -41,10 +41,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80,		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Plan9/mips/include/lib9.h b/Plan9/mips/include/lib9.h
index 91e0f3d4..003188d3 100644
--- a/Plan9/mips/include/lib9.h
+++ b/Plan9/mips/include/lib9.h
@@ -41,10 +41,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80,		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Plan9/power/include/lib9.h b/Plan9/power/include/lib9.h
index 91e0f3d4..003188d3 100644
--- a/Plan9/power/include/lib9.h
+++ b/Plan9/power/include/lib9.h
@@ -41,10 +41,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80,		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Plan9/sparc/include/lib9.h b/Plan9/sparc/include/lib9.h
index 91e0f3d4..003188d3 100644
--- a/Plan9/sparc/include/lib9.h
+++ b/Plan9/sparc/include/lib9.h
@@ -41,10 +41,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80,		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Solaris/386/include/lib9.h b/Solaris/386/include/lib9.h
index 3e12aa8b..d85114fd 100644
--- a/Solaris/386/include/lib9.h
+++ b/Solaris/386/include/lib9.h
@@ -53,10 +53,12 @@ typedef unsigned long long	uint64;
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*
diff --git a/Solaris/sparc/include/lib9.h b/Solaris/sparc/include/lib9.h
index 7f93cdad..581945de 100644
--- a/Solaris/sparc/include/lib9.h
+++ b/Solaris/sparc/include/lib9.h
@@ -72,10 +72,12 @@ extern	int	tokenize(char*, char**, int);
 
 enum
 {
-	UTFmax		= 3,		/* maximum bytes per rune */
+	UTFmax		= 4,		/* maximum bytes per rune */
 	Runesync	= 0x80,		/* cannot represent part of a UTF sequence (<) */
 	Runeself	= 0x80,		/* rune and UTF sequences are the same (<) */
-	Runeerror	= 0x80		/* decoding error in UTF */
+	Runeerror	= 0xFFFD,	/* decoding error in UTF */
+	Runemax		= 0x10FFFF,	/* 21-bit rune */
+	Runemask	= 0x1FFFFF,	/* bits used by runes (see grep) */
 };
 
 /*