From d5c106a95c49508f5e214f2fa174968eee2352fc Mon Sep 17 00:00:00 2001
From: christos <christos>
Date: Sat, 6 Jun 2015 21:19:07 +0000
Subject: [PATCH] PR/437: Fix handling of invalid unicode characters. tcsh uses
the high order bits to encode attributes in the prompt and the high bit in
regular characters. Make the drawing routines take an argument indicating if
we are drawing the prompt or not, so that we can decide how to deal with the
high bits. This solution is the minimum diff and does not allow "large valued"
unicode characters to be in the prompt (because they would conflict with the
attribute bits). A better solution would be to have a struct for each
character so we could encode extra attributes.
---
Fixes | 1 +
ed.chared.c | 2 +-
ed.refresh.c | 54 ++++++++++++++++++++++++++++++++++++++++++------------
ed.xmap.c | 2 +-
sh.file.c | 2 +-
sh.glob.c | 9 +++++++--
sh.h | 15 ++++++++++++---
sh.hist.c | 2 +-
sh.misc.c | 22 ++++++++++++++++++----
tc.func.c | 7 +++++++
tc.nls.c | 40 ++++++++++++++++++++++++++++++----------
tc.nls.h | 3 ++-
tc.printf.c | 2 +-
tc.str.c | 22 ++++++++++++++++++----
tw.parse.c | 7 ++++++-
15 files changed, 148 insertions(+), 42 deletions(-)
diff --git a/Fixes b/Fixes
index 7d0ceac..aa779b1 100644
--- a/Fixes
+++ b/Fixes
@@ -1,3 +1,4 @@
+ 2. PR/437: Fix handling of invalid unicode characters.
1. PR/451: Fix error messages containing %c to be always '%c'
41. V6.19.00 - 20150521
diff --git a/ed.chared.c b/ed.chared.c
index bade211..c0bd41b 100644
--- a/ed.chared.c
+++ b/ed.chared.c
@@ -3387,7 +3387,7 @@ e_stuff_char(Char c)
(void) Cookedmode();
(void) xwrite(SHIN, "\n", 1);
- len = one_wctomb(buf, c & CHAR);
+ len = one_wctomb(buf, c);
for (i = 0; i < len; i++)
(void) ioctl(SHIN, TIOCSTI, (ioctl_t) &buf[i]);
diff --git a/ed.refresh.c b/ed.refresh.c
index 9e6da00..a88c5e5 100644
--- a/ed.refresh.c
+++ b/ed.refresh.c
@@ -46,7 +46,7 @@ static int vcursor_h, vcursor_v;
static int rprompt_h, rprompt_v;
static int MakeLiteral (Char *, int, Char);
-static int Draw (Char *, int);
+static int Draw (Char *, int, int);
static void Vdraw (Char, int);
static void RefreshPromptpart (Char *);
static void update_line (Char *, Char *, int);
@@ -159,15 +159,44 @@ static int MakeLiteral(Char *str, int len, Char addlit)
return i | LITERAL;
}
+/* draw char at cp, expand tabs, ctl chars */
static int
-Draw(Char *cp, int nocomb) /* draw char at cp, expand tabs, ctl chars */
+Draw(Char *cp, int nocomb, int drawPrompt)
{
int w, i, lv, lh;
Char c, attr;
+#ifdef WIDE_STRINGS
+ if (!drawPrompt) { /* draw command-line */
+ attr = 0;
+ c = *cp;
+ } else { /* draw prompt */
+ /* prompt with attributes(UNDER,BOLD,STANDOUT) */
+ if (*cp & (UNDER | BOLD | STANDOUT)) { /* *cp >= STANDOUT */
+
+ /* example)
+ * We can't distinguish whether (*cp=)0x02ffffff is
+ * U+02FFFFFF or U+00FFFFFF|STANDOUT.
+ * We handle as U+00FFFFFF|STANDOUT, only when drawing prompt. */
+ attr = (*cp & ATTRIBUTES);
+ /* ~(UNDER | BOLD | STANDOUT) = 0xf1ffffff */
+ c = *cp & ~(UNDER | BOLD | STANDOUT);
+
+ /* if c is ctrl code, we handle *cp as havnig no attributes */
+ if ((c < 0x20 && c >= 0) || c == 0x7f) {
+ attr = 0;
+ c = *cp;
+ }
+ } else { /* prompt without attributes */
+ attr = 0;
+ c = *cp;
+ }
+ }
+#else
attr = *cp & ~CHAR;
c = *cp & CHAR;
- w = NLSClassify(c, nocomb);
+#endif
+ w = NLSClassify(c, nocomb, drawPrompt);
switch (w) {
case NLSCLASS_NL:
Vdraw('\0', 0); /* assure end of line */
@@ -201,10 +230,11 @@ Draw(Char *cp, int nocomb) /* draw char at cp, expand tabs, ctl chars */
case NLSCLASS_ILLEGAL2:
case NLSCLASS_ILLEGAL3:
case NLSCLASS_ILLEGAL4:
- Vdraw('\\' | attr, 1);
- Vdraw('U' | attr, 1);
- Vdraw('+' | attr, 1);
- for (i = 8 * NLSCLASS_ILLEGAL_SIZE(w) - 4; i >= 0; i -= 4)
+ case NLSCLASS_ILLEGAL5:
+ Vdraw('\\', 1);
+ Vdraw('U', 1);
+ Vdraw('+', 1);
+ for (i = 16 + 4 * (-w-5); i >= 0; i -= 4)
Vdraw("0123456789ABCDEF"[(c >> i) & 15] | attr, 1);
break;
case 0:
@@ -302,7 +332,7 @@ RefreshPromptpart(Char *buf)
}
}
else
- cp += Draw(cp, cp == buf);
+ cp += Draw(cp, cp == buf, 1);
}
}
@@ -354,7 +384,7 @@ Refresh(void)
cur_v = vcursor_v;
Cursor = cp;
}
- cp += Draw(cp, cp == InputBuf);
+ cp += Draw(cp, cp == InputBuf, 0);
}
if (cur_h == -1) { /* if I haven't been set yet, I'm at the end */
@@ -1126,7 +1156,7 @@ RefCursor(void)
cp++;
continue;
}
- w = NLSClassify(*cp & CHAR, cp == Prompt);
+ w = NLSClassify(*cp & CHAR, cp == Prompt, 0);
cp++;
switch(w) {
case NLSCLASS_NL:
@@ -1158,7 +1188,7 @@ RefCursor(void)
}
for (cp = InputBuf; cp < Cursor;) { /* do input buffer to Cursor */
- w = NLSClassify(*cp & CHAR, cp == InputBuf);
+ w = NLSClassify(*cp & CHAR, cp == InputBuf, 0);
cp++;
switch(w) {
case NLSCLASS_NL:
@@ -1251,7 +1281,7 @@ RefPlusOne(int l)
}
cp = Cursor - l;
c = *cp & CHAR;
- w = NLSClassify(c, cp == InputBuf);
+ w = NLSClassify(c, cp == InputBuf, 0);
switch(w) {
case NLSCLASS_CTRL:
PutPlusOne('^', 1);
diff --git a/ed.xmap.c b/ed.xmap.c
index 6e1d56e..36bce1e 100644
--- a/ed.xmap.c
+++ b/ed.xmap.c
@@ -743,7 +743,7 @@ unparsestring(const CStr *str, const Char *sep)
*b++ = (unsigned char) p;
}
else if (p == ' ' || (Isprint(p) && !Isspace(p)))
- b += one_wctomb((char *)b, p & CHAR);
+ b += one_wctomb((char *)b, p);
else {
*b++ = '\\';
*b++ = ((p >> 6) & 7) + '0';
diff --git a/sh.file.c b/sh.file.c
index 343b774..3989d8a 100644
--- a/sh.file.c
+++ b/sh.file.c
@@ -249,7 +249,7 @@ pushback(const Char *string)
char buf[MB_LEN_MAX];
size_t i, len;
- len = one_wctomb(buf, *p & CHAR);
+ len = one_wctomb(buf, *p);
for (i = 0; i < len; i++)
(void) ioctl(SHOUT, TIOCSTI, (ioctl_t) &buf[i]);
}
diff --git a/sh.glob.c b/sh.glob.c
index fc510bf..7d008aa 100644
--- a/sh.glob.c
+++ b/sh.glob.c
@@ -594,8 +594,13 @@ trim(Char **t)
Char *p;
while ((p = *t++) != '\0')
- while (*p)
- *p++ &= TRIM;
+ while (*p) {
+#if INVALID_BYTE != 0
+ if ((*p & INVALID_BYTE) != INVALID_BYTE) /* *p < INVALID_BYTE */
+#endif
+ *p &= TRIM;
+ p++;
+ }
}
int
diff --git a/sh.h b/sh.h
index e71a24e..75de557 100644
--- a/sh.h
+++ b/sh.h
@@ -707,14 +707,21 @@ extern struct sigaction parterm; /* Parents terminate catch */
#define ASCII 0177
#ifdef WIDE_STRINGS /* Implies SHORT_STRINGS */
/* 31st char bit used for 'ing (not 32nd, we want all values nonnegative) */
-# define QUOTE 0x40000000
-# define TRIM 0x3FFFFFFF /* Mask to strip quote bit */
+/*
+ * Notice
+ *
+ * By fix for handling unicode name file, 32nd bit is used.
+ * We need use '&' instead of '> or <' when comparing with INVALID_BYTE etc..
+ * Cast to uChar is not recommended,
+ * becase Char is 4bytes but uChar is 8bytes on I32LP64. */
+# define QUOTE 0x80000000
+# define TRIM 0x7FFFFFFF /* Mask to strip quote bit */
# define UNDER 0x08000000 /* Underline flag */
# define BOLD 0x04000000 /* Bold flag */
# define STANDOUT 0x02000000 /* Standout flag */
# define LITERAL 0x01000000 /* Literal character flag */
# define ATTRIBUTES 0x0F000000 /* The bits used for attributes */
-# define INVALID_BYTE 0x00800000 /* Invalid character on input */
+# define INVALID_BYTE 0xF0000000 /* Invalid character on input */
# ifdef SOLARIS2
# define CHAR 0x30FFFFFF /* Mask to mask out the character */
# else
@@ -743,6 +750,8 @@ extern struct sigaction parterm; /* Parents terminate catch */
#endif
#define CHAR_DBWIDTH (LITERAL|(LITERAL-1))
+# define MAX_UTF32 0x7FFFFFFF /* max UTF32 is U+7FFFFFFF */
+
EXTERN int AsciiOnly; /* If set only 7 bits expected in characters */
/*
diff --git a/sh.hist.c b/sh.hist.c
index b8f71b7..c0eded5 100644
--- a/sh.hist.c
+++ b/sh.hist.c
@@ -1199,7 +1199,7 @@ fmthist(int fmt, ptr_t ptr)
buf = xmalloc(Strlen(istr) * MB_LEN_MAX + 1);
for (p = buf, ip = istr; *ip != '\0'; ip++)
- p += one_wctomb(p, CHAR & *ip);
+ p += one_wctomb(p, *ip);
*p = '\0';
xfree(istr);
diff --git a/sh.misc.c b/sh.misc.c
index 7232b12..233ba5f 100644
--- a/sh.misc.c
+++ b/sh.misc.c
@@ -450,8 +450,13 @@ strip(Char *cp)
if (!cp)
return (cp);
- while ((*dp++ &= TRIM) != '\0')
- continue;
+ while (*dp != '\0') {
+#if INVALID_BYTE != 0
+ if ((*dp & INVALID_BYTE) != INVALID_BYTE) /* *dp < INVALID_BYTE */
+#endif
+ *dp &= TRIM;
+ dp++;
+ }
return (cp);
}
@@ -462,8 +467,17 @@ quote(Char *cp)
if (!cp)
return (cp);
- while (*dp != '\0')
- *dp++ |= QUOTE;
+ while (*dp != '\0') {
+#ifdef WIDE_STRINGS
+ if ((*dp & 0xffffff80) == 0) /* *dp < 0x80 */
+#elif defined SHORT_STRINGS
+ if ((*dp & 0xff80) == 0) /* *dp < 0x80 */
+#else
+ if ((*dp & 0x80) == 0) /* *dp < 0x80 */
+#endif
+ *dp |= QUOTE;
+ dp++;
+ }
return (cp);
}
diff --git a/tc.func.c b/tc.func.c
index 2b28a68..5a909d6 100644
--- a/tc.func.c
+++ b/tc.func.c
@@ -124,7 +124,14 @@ expand_lex(const struct wordent *sp0, int from, int to)
(((*s & TRIM) == '\\') && (prev_c != '\\')))) {
Strbuf_append1(&buf, '\\');
}
+#if INVALID_BYTE != 0
+ if ((*s & INVALID_BYTE) != INVALID_BYTE) /* *s < INVALID_BYTE */
+ Strbuf_append1(&buf, *s & TRIM);
+ else
+ Strbuf_append1(&buf, *s);
+#else
Strbuf_append1(&buf, *s & TRIM);
+#endif
prev_c = *s;
}
Strbuf_append1(&buf, ' ');
diff --git a/tc.nls.c b/tc.nls.c
index 2c38f3f..22ad173 100644
--- a/tc.nls.c
+++ b/tc.nls.c
@@ -64,7 +64,11 @@ NLSWidth(Char c)
{
# ifdef HAVE_WCWIDTH
int l;
+#if INVALID_BYTE != 0
+ if ((c & INVALID_BYTE) == INVALID_BYTE) /* c >= INVALID_BYTE */
+#else
if (c & INVALID_BYTE)
+#endif
return 1;
l = xwcwidth((wchar_t) c);
return l >= 0 ? l : 0;
@@ -116,12 +120,36 @@ NLSChangeCase(const Char *p, int mode)
}
int
-NLSClassify(Char c, int nocomb)
+NLSClassify(Char c, int nocomb, int drawPrompt)
{
int w;
- if (c & INVALID_BYTE)
+#ifndef SHORT_STRINGS
+ if ((c & 0x80) != 0) /* c >= 0x80 */
return NLSCLASS_ILLEGAL;
+#endif
+ if (!drawPrompt) { /* draw command-line */
+#if INVALID_BYTE != 0
+ if ((c & INVALID_BYTE) == INVALID_BYTE) /* c >= INVALID_BYTE */
+ return NLSCLASS_ILLEGAL;
+ if ((c & INVALID_BYTE) == QUOTE && (c & 0x80) == 0) /* c >= QUOTE */
+ return 1;
+ if (c >= 0x10000000) /* U+10000000 = FC 90 80 80 80 80 */
+ return NLSCLASS_ILLEGAL5;
+ if (c >= 0x1000000) /* U+1000000 = F9 80 80 80 80 */
+ return NLSCLASS_ILLEGAL4;
+ if (c >= 0x100000) /* U+100000 = F4 80 80 80 */
+ return NLSCLASS_ILLEGAL3;
+#endif
+ if (c >= 0x10000) /* U+10000 = F0 90 80 80 */
+ return NLSCLASS_ILLEGAL2;
+ }
w = NLSWidth(c);
+ if (drawPrompt) { /* draw prompt */
+ if (w > 0)
+ return w;
+ if (w == 0)
+ return 1;
+ }
if ((w > 0 && !(Iscntrl(c) && (c & CHAR) < 0x100)) || (Isprint(c) && !nocomb))
return w;
if (Iscntrl(c) && (c & CHAR) < 0x100) {
@@ -131,13 +159,5 @@ NLSClassify(Char c, int nocomb)
return NLSCLASS_TAB;
return NLSCLASS_CTRL;
}
-#ifdef WIDE_STRINGS
- if (c >= 0x1000000)
- return NLSCLASS_ILLEGAL4;
- if (c >= 0x10000)
- return NLSCLASS_ILLEGAL3;
-#endif
- if (c >= 0x100)
- return NLSCLASS_ILLEGAL2;
return NLSCLASS_ILLEGAL;
}
diff --git a/tc.nls.h b/tc.nls.h
index 4d27741..6930682 100644
--- a/tc.nls.h
+++ b/tc.nls.h
@@ -43,7 +43,7 @@ extern int NLSStringWidth (const Char *);
#endif
extern Char *NLSChangeCase (const Char *, int);
-extern int NLSClassify (Char, int);
+extern int NLSClassify (Char, int, int);
#define NLSCLASS_CTRL (-1)
#define NLSCLASS_TAB (-2)
@@ -52,6 +52,7 @@ extern int NLSClassify (Char, int);
#define NLSCLASS_ILLEGAL2 (-5)
#define NLSCLASS_ILLEGAL3 (-6)
#define NLSCLASS_ILLEGAL4 (-7)
+#define NLSCLASS_ILLEGAL5 (-8)
#define NLSCLASS_ILLEGAL_SIZE(x) (-(x) - (-(NLSCLASS_ILLEGAL) - 1))
diff --git a/tc.printf.c b/tc.printf.c
index 7f2612d..c6be145 100644
--- a/tc.printf.c
+++ b/tc.printf.c
@@ -289,7 +289,7 @@ doprnt(void (*addchar) (int), const char *sfmt, va_list ap)
(*addchar) ('\\' | attributes);
count++;
}
- len = one_wctomb(cbuf, *Bp & CHAR);
+ len = one_wctomb(cbuf, *Bp);
for (pos = 0; pos < len; pos++) {
(*addchar) ((unsigned char)cbuf[pos] | attributes
| (*Bp & ATTRIBUTES));
diff --git a/tc.str.c b/tc.str.c
index c407cb8..c2b5ac8 100644
--- a/tc.str.c
+++ b/tc.str.c
@@ -66,10 +66,24 @@ one_wctomb(char *s, Char wchar)
{
int len;
- if (wchar & INVALID_BYTE) {
- s[0] = wchar & 0xFF;
+#if INVALID_BYTE != 0
+ if ((wchar & INVALID_BYTE) == INVALID_BYTE) { /* wchar >= INVALID_BYTE */
+ /* invalid char
+ * exmaple)
+ * if wchar = f0000090(=90|INVALID_BYTE), then *s = ffffff90 */
+ *s = (char)wchar;
len = 1;
+#else
+ if (wchar & (CHAR & INVALID_BYTE)) {
+ s[0] = wchar & (CHAR & 0xFF);
+ len = 1;
+#endif
} else {
+#if INVALID_BYTE != 0
+ wchar &= MAX_UTF32;
+#else
+ wchar &= CHAR;
+#endif
#ifdef UTF16_STRINGS
if (wchar >= 0x10000) {
/* UTF-16 systems can't handle these values directly in calls to
@@ -224,7 +238,7 @@ short2str(const Char *src)
dst = sdst;
edst = &dst[dstsize];
while (*src) {
- dst += one_wctomb(dst, *src & CHAR);
+ dst += one_wctomb(dst, *src);
src++;
if (dst >= edst) {
char *wdst = dst;
@@ -544,7 +558,7 @@ short2qstr(const Char *src)
dst = &edst[-MALLOC_INCR];
}
}
- dst += one_wctomb(dst, *src & CHAR);
+ dst += one_wctomb(dst, *src);
src++;
if (dst >= edst) {
ptrdiff_t i = dst - edst;
diff --git a/tw.parse.c b/tw.parse.c
index 8309ed8..94982d6 100644
--- a/tw.parse.c
+++ b/tw.parse.c
@@ -618,7 +618,12 @@ insert_meta(const Char *cp, const Char *cpend, const Char *word,
break;
wq = w & QUOTE;
- w &= ~QUOTE;
+#if INVALID_BYTE != 0
+ /* add checking INVALID_BYTE for FIX UTF32 */
+ if ((w & INVALID_BYTE) != INVALID_BYTE) /* w < INVALID_BYTE */
+#else
+ w &= ~QUOTE;
+#endif
if (cmap(w, _ESC | _QF))
wq = QUOTE; /* quotes are always quoted */
--
2.5.5