Support UTF-8 encoding when processing input.
Index: html2text-1.3.2a/Area.C
===================================================================
--- html2text-1.3.2a.orig/Area.C 2008-09-20 14:01:44.259190763 +0300
+++ html2text-1.3.2a/Area.C 2008-09-20 14:06:15.255782998 +0300
@@ -36,10 +36,13 @@
#include <iostream>
#include "Area.h"
+#include "html.h"
#include "string.h"
#define LATIN1_nbsp 160
+extern int use_encoding;
+
/* ------------------------------------------------------------------------- */
#define malloc_array(type, size)\
@@ -81,6 +84,53 @@
/* ------------------------------------------------------------------------- */
+/* utf_length() and utf_width()
+ *
+ * Very simplified algorithm of calculating length of UTF-8
+ * string. No check for errors. Counting only ASCII bytes and
+ * leading bytes of UTF-8 multibyte sequences. All bytes like
+ * 10xxxxxx are dropped. If USE_UTF8 is false then returns
+ * usual length. --YS
+ */
+
+size_t utf8_aux_count(char ch)
+{
+ if((ch & 0xe0) == 0xc0)
+ {
+ return 1;
+ }
+ else if((ch & 0xf0) == 0xe0)
+ {
+ return 2;
+ }
+ else if ((ch & 0xf8) == 0xf0)
+ {
+ return 3;
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+unsigned int
+Line::utf_length(size_type f, size_type t) const
+{
+ size_type m = (t < length_ ? t : length_);
+ size_type r = m - f;
+ if(USE_UTF8)
+ {
+ for (int i = f; i < m; i++)
+ {
+ char& ch = cells_[i].character;
+ size_type aux_count = utf8_aux_count(ch);
+ r -= aux_count;
+ i += aux_count;
+ }
+ }
+ return r;
+}
+
void
Line::resize(size_type l)
{
@@ -236,6 +286,28 @@
return *this;
}
+unsigned int
+Area::utf_width()
+{
+ size_type r = width_;
+ if(USE_UTF8) { r = 0;
+ for (size_type yy = 0; yy < height_; yy++) {
+ int i = width_ - 1;
+ while((i >= 0) && isspace(cells_[yy][i].character))
+ {
+ --i;
+ }
+ size_type aux_count_sum = 0;
+ for (; i >= 0; i--) {
+ aux_count_sum += utf8_aux_count(cells_[yy][i].character);
+ }
+ size_type r1 = width_ - aux_count_sum;
+ if(r < r1) r = r1;
+ }
+ }
+ return r;
+}
+
void
Area::resize(size_type w, size_type h)
{
@@ -439,7 +511,7 @@
char c = p->character;
char a = p->attribute;
- if (c == (char) LATIN1_nbsp) c = ' ';
+ if (c == (char) LATIN1_nbsp && !USE_UTF8) c = ' ';
if (a == Cell::NONE) {
os << c;
Index: html2text-1.3.2a/Area.h
===================================================================
--- html2text-1.3.2a.orig/Area.h 2008-09-20 14:01:44.295185701 +0300
+++ html2text-1.3.2a/Area.h 2008-09-20 14:06:15.255782998 +0300
@@ -81,6 +81,8 @@
Cell &operator[](size_type x) { return cells_[x]; }
const Cell *cells() const { return cells_; }
+ unsigned int utf_length(size_type f, size_type t) const;
+
void resize(size_type l);
void enlarge(size_type l) { if (l > length_) resize(l); }
@@ -134,6 +136,8 @@
Cell *operator[](size_type y) { return cells_[y]; }
const Area &operator>>=(size_type rs);
+ unsigned int utf_width();
+
void resize(size_type w, size_type h);
void enlarge(size_type w, size_type h);
Index: html2text-1.3.2a/format.C
===================================================================
--- html2text-1.3.2a.orig/format.C 2008-09-20 14:01:44.311190459 +0300
+++ html2text-1.3.2a/format.C 2008-09-20 14:06:15.259781132 +0300
@@ -1210,6 +1210,7 @@
}
Line::size_type to = from + 1;
+ int to_from;
Line::size_type lbp = (Line::size_type) -1; // "Last break position".
@@ -1238,18 +1239,20 @@
to++;
}
- if (to - from > w && lbp != (Area::size_type) -1) { to = lbp; break; }
+ if (line.utf_length(from,to) > w && lbp != (Area::size_type) -1)
+ { to = lbp; break; }
}
+ to_from = line.utf_length(from,to);
/*
* Copy the "from...to" range from the "line" to the bottom of the "res"
* Area.
*/
Area::size_type x = 0;
Area::size_type len = to - from;
- if (halign == Area::LEFT || len >= w) { ; } else
- if (halign == Area::CENTER) { x += (w - len) / 2; } else
- if (halign == Area::RIGHT) { x += w - len; }
+ if (halign == Area::LEFT || to_from >= w) { ; } else
+ if (halign == Area::CENTER) { x += (w - to_from) / 2; } else
+ if (halign == Area::RIGHT) { x += w - to_from; }
res->insert(line.cells() + from, len, x, res->height());
/*
Index: html2text-1.3.2a/html2text.C
===================================================================
--- html2text-1.3.2a.orig/html2text.C 2008-09-20 14:03:39.181744957 +0300
+++ html2text-1.3.2a/html2text.C 2008-09-20 14:06:15.259781132 +0300
@@ -150,9 +150,10 @@
-o <file> Redirect output into <file>\n\
-nobs Do not use backspaces for boldface and underlining\n\
-ascii Use plain ASCII for output instead of ISO-8859-1\n\
+ -utf8 Assume both terminal and input stream are in UTF-8 mode\n\
";
-int use_iso8859 = 1;
+int use_encoding = ISO8859;
int
main(int argc, char **argv)
@@ -201,7 +202,8 @@
if (!strcmp(arg, "-width" )) { width = atoi(argv[++i]); } else
if (!strcmp(arg, "-o" )) { output_file_name = argv[++i]; } else
if (!strcmp(arg, "-nobs" )) { use_backspaces = false; } else
- if (!strcmp(arg, "-ascii" )) { use_iso8859 = false; } else
+ if (!strcmp(arg, "-ascii" )) { use_encoding = ASCII; } else
+ if (!strcmp(arg, "-utf8" )) { use_encoding = UTF8; } else
{
std::cerr
<< "Unrecognized command line option \""
Index: html2text-1.3.2a/html.h
===================================================================
--- html2text-1.3.2a.orig/html.h 2008-09-20 14:01:44.343193129 +0300
+++ html2text-1.3.2a/html.h 2008-09-20 14:06:15.259781132 +0300
@@ -61,6 +61,11 @@
/* ------------------------------------------------------------------------- */
+enum {ASCII, ISO8859, UTF8};
+#define USE_ISO8859 (use_encoding == ISO8859)
+#define USE_ASCII (use_encoding == ASCII)
+#define USE_UTF8 (use_encoding == UTF8)
+
#define LATIN1_nbsp 160
#define LATIN1_iexcl 161
#define LATIN1_cent 162
Index: html2text-1.3.2a/sgml.C
===================================================================
--- html2text-1.3.2a.orig/sgml.C 2008-09-20 14:01:44.391192735 +0300
+++ html2text-1.3.2a/sgml.C 2008-09-20 14:06:15.259781132 +0300
@@ -62,261 +62,280 @@
char name[8];
int iso8859code;
char *asciistr;
+ unsigned long unicode;
} entities[] = {
- { "AElig", LATIN1_AElig, "AE" },
- { "AMP", 0, "&" },
- { "Aacute", LATIN1_Aacute, "A'" },
- { "Acirc", LATIN1_Acirc, "A^" },
- { "Agrave", LATIN1_Agrave, "A`" },
- { "Alpha", 0, "A" },
- { "Aring", LATIN1_Aring, "AA" },
- { "Atilde", LATIN1_Atilde, "A~" },
- { "Auml", LATIN1_Auml, "A\"" },
- { "Beta", 0, "B" },
- { "Ccedil", LATIN1_Ccedil, "C," },
- { "Chi", 0, "H" },
- { "Dagger", 0, "++" },
- { "Delta", 0, "D" },
- { "ETH", LATIN1_ETH, "D-" },
- { "Eacute", LATIN1_Eacute, "E'" },
- { "Ecirc", LATIN1_Ecirc, "E^" },
- { "Egrave", LATIN1_Egrave, "E`" },
- { "Epsilon", 0, "E" },
- { "Eta", 0, "E" },
- { "Euml", LATIN1_Euml, "E\"" },
- { "GT", 0, ">" },
- { "Gamma", 0, "G" },
- { "Iacute", LATIN1_Iacute, "I'" },
- { "Icirc", LATIN1_Icirc, "I^" },
- { "Igrave", LATIN1_Igrave, "I`" },
- { "Iota", 0, "I" },
- { "Iuml", LATIN1_Iuml, "I\"" },
- { "Kappa", 0, "K" },
- { "LT", 0, "<" },
- { "Lambda", 0, "L" },
- { "Mu", 0, "M" },
- { "Ntilde", LATIN1_Ntilde, "N~" },
- { "Nu", 0, "N" },
- { "OElig", 0, "OE" },
- { "Oacute", LATIN1_Oacute, "O'" },
- { "Ocirc", LATIN1_Ocirc, "O^" },
- { "Ograve", LATIN1_Ograve, "O`" },
- { "Omega", 0, "O" },
- { "Omicron", 0, "O" },
- { "Oslash", LATIN1_Oslash, "O/" },
- { "Otilde", LATIN1_Otilde, "O~" },
- { "Ouml", LATIN1_Ouml, "O\"" },
- { "Phi", 0, "F" },
- { "Pi", 0, "P" },
- { "Prime", 0, "''" },
- { "Psi", 0, "PS" },
- { "QUOT", 0, "\"" },
- { "Rho", 0, "R" },
- { "Scaron", 0, "S" },
- { "Sigma", 0, "S" },
- { "THORN", LATIN1_THORN, "TH" },
- { "Tau", 0, "T" },
- { "Theta", 0, "TH" },
- { "Uacute", LATIN1_Uacute, "U'" },
- { "Ucirc", LATIN1_Ucirc, "U^" },
- { "Ugrave", LATIN1_Ugrave, "U`" },
- { "Upsilon", 0, "U" },
- { "Uuml", LATIN1_Uuml, "U\"" },
- { "Xi", 0, "X" },
- { "Yacute", LATIN1_Yacute, "Y'" },
- { "Yuml", 0, "Y\"" },
- { "Zeta", 0, "Z" },
- { "aacute", LATIN1_aacute, "a'" },
- { "acirc", LATIN1_acirc, "a^" },
- { "acute", LATIN1_acute, "'" },
- { "aelig", LATIN1_aelig, "ae" },
- { "agrave", LATIN1_agrave, "a`" },
+ { "AElig", LATIN1_AElig, "AE", 0x00c6},
+ { "AMP", 0, "&", 0x0026},
+ { "Aacute", LATIN1_Aacute, "A'", 0x00c1},
+ { "Acirc", LATIN1_Acirc, "A^", 0x00c2},
+ { "Agrave", LATIN1_Agrave, "A`", 0x00c0},
+ { "Alpha", 0, "A", 0x0391},
+ { "Aring", LATIN1_Aring, "AA", 0x00c5},
+ { "Atilde", LATIN1_Atilde, "A~", 0x00c3},
+ { "Auml", LATIN1_Auml, "A\"", 0x00c4},
+ { "Beta", 0, "B", 0x0392},
+ { "Ccedil", LATIN1_Ccedil, "C,", 0x00c7},
+ { "Chi", 0, "H", 0x03a7},
+ { "Dagger", 0, "++", 0x2020},
+ { "Delta", 0, "D", 0x0394},
+ { "ETH", LATIN1_ETH, "D-", 0x00d0},
+ { "Eacute", LATIN1_Eacute, "E'", 0x00c9},
+ { "Ecirc", LATIN1_Ecirc, "E^", 0x00ca},
+ { "Egrave", LATIN1_Egrave, "E`", 0x00c8},
+ { "Epsilon", 0, "E", 0x0395},
+ { "Eta", 0, "E", 0x0397},
+ { "Euml", LATIN1_Euml, "E\"", 0x00cb},
+ { "GT", 0, ">", 0x003e},
+ { "Gamma", 0, "G", 0x0393},
+ { "Iacute", LATIN1_Iacute, "I'", 0x00cd},
+ { "Icirc", LATIN1_Icirc, "I^", 0x00ce},
+ { "Igrave", LATIN1_Igrave, "I`", 0x00cc},
+ { "Iota", 0, "I", 0x0399},
+ { "Iuml", LATIN1_Iuml, "I\"", 0x00cf},
+ { "Kappa", 0, "K", 0x039a},
+ { "LT", 0, "<", 0x003c},
+ { "Lambda", 0, "L", 0x039b},
+ { "Mu", 0, "M", 0x039c},
+ { "Ntilde", LATIN1_Ntilde, "N~", 0x00d1},
+ { "Nu", 0, "N", 0x039d},
+ { "OElig", 0, "OE", 0x0152},
+ { "Oacute", LATIN1_Oacute, "O'", 0x00d3},
+ { "Ocirc", LATIN1_Ocirc, "O^", 0x00d4},
+ { "Ograve", LATIN1_Ograve, "O`", 0x00d2},
+ { "Omega", 0, "O", 0x03a9},
+ { "Omicron", 0, "O", 0x039f},
+ { "Oslash", LATIN1_Oslash, "O/", 0x00d8},
+ { "Otilde", LATIN1_Otilde, "O~", 0x00d5},
+ { "Ouml", LATIN1_Ouml, "O\"", 0x00d6},
+ { "Phi", 0, "F", 0x03a6},
+ { "Pi", 0, "P", 0x03a0},
+ { "Prime", 0, "''", },
+ { "Psi", 0, "PS", 0x03a8},
+ { "QUOT", 0, "\"", },
+ { "Rho", 0, "R", 0x03a1},
+ { "Scaron", 0, "S", 0x0161},
+ { "Sigma", 0, "S", 0x03a3},
+ { "THORN", LATIN1_THORN, "TH", 0x00de},
+ { "Tau", 0, "T", 0x03a4},
+ { "Theta", 0, "TH", 0x0398},
+ { "Uacute", LATIN1_Uacute, "U'", 0x00da},
+ { "Ucirc", LATIN1_Ucirc, "U^", 0x00db},
+ { "Ugrave", LATIN1_Ugrave, "U`", 0x00d9},
+ { "Upsilon", 0, "U", 0x03a5},
+ { "Uuml", LATIN1_Uuml, "U\"", 0x00dc},
+ { "Xi", 0, "X", 0x039e},
+ { "Yacute", LATIN1_Yacute, "Y'", 0x00dd},
+ { "Yuml", 0, "Y\"", 0x0178},
+ { "Zeta", 0, "Z", 0x0396},
+ { "aacute", LATIN1_aacute, "a'", 0x00e1},
+ { "acirc", LATIN1_acirc, "a^", 0x00e2},
+ { "acute", LATIN1_acute, "'", 0x00b4},
+ { "aelig", LATIN1_aelig, "ae", 0x00e6},
+ { "agrave", LATIN1_agrave, "a`", 0x00e0},
{ "alefsym", 0, "Aleph" },
- { "alpha", 0, "a" },
+ { "alpha", 0, "a", 0x03b1},
{ "amp", 0, "&" },
{ "and", 0, "AND" },
{ "ang", 0, "-V" },
{ "apos", 0, "'" },
- { "aring", LATIN1_aring, "aa" },
- { "asymp", 0, "~=" },
- { "atilde", LATIN1_atilde, "a~" },
- { "auml", LATIN1_auml, "a\"" },
+ { "aring", LATIN1_aring, "aa", 0x00e5},
+ { "asymp", 0, "~=", 0x2248},
+ { "atilde", LATIN1_atilde, "a~", 0x00e3},
+ { "auml", LATIN1_auml, "a\"", 0x00e5},
{ "bdquo", 0, "\"" },
- { "beta", 0, "b" },
- { "brvbar", LATIN1_brvbar, "|" },
- { "bull", 0, " o " },
+ { "beta", 0, "b", 0x03b2},
+ { "brvbar", LATIN1_brvbar, "|", 0x00a6},
+ { "bull", 0, " o ", 0x2022},
{ "cap", 0, "(U" },
- { "ccedil", LATIN1_ccedil, "c," },
- { "cedil", LATIN1_cedil, "," },
- { "cent", LATIN1_cent, "-c-" },
- { "chi", 0, "h" },
- { "circ", 0, "^" },
+ { "ccedil", LATIN1_ccedil, "c,", 0x00e7},
+ { "cedil", LATIN1_cedil, ",", 0x00b8},
+ { "cent", LATIN1_cent, "-c-", 0x00a2},
+ { "chi", 0, "h", 0x03c7},
+ { "circ", 0, "^", 0x005e},
// { "clubs", 0, "[clubs]" },
{ "cong", 0, "?=" },
- { "copy", LATIN1_copy, "(c)" },
+ { "copy", LATIN1_copy, "(c)", 0x00a9},
{ "crarr", 0, "<-'" },
{ "cup", 0, ")U" },
- { "curren", LATIN1_curren, "CUR" },
+ { "curren", LATIN1_curren, "CUR", 0x00a4},
{ "dArr", 0, "vv" },
- { "dagger", 0, "+" },
+ { "dagger", 0, "+", 0x2020},
{ "darr", 0, "v" },
- { "deg", LATIN1_deg, "DEG" },
- { "delta", 0, "d" },
+ { "deg", LATIN1_deg, "DEG", 0x00b0},
+ { "delta", 0, "d", 0x03b4},
// { "diams", 0, "[diamonds]" },
- { "divide", LATIN1_divide, "/" },
- { "eacute", LATIN1_eacute, "e'" },
- { "ecirc", LATIN1_ecirc, "e^" },
- { "egrave", LATIN1_egrave, "e`" },
+ { "divide", LATIN1_divide, "/", 0x00f7},
+ { "eacute", LATIN1_eacute, "e'", 0x00e9},
+ { "ecirc", LATIN1_ecirc, "e^", 0x00ea},
+ { "egrave", LATIN1_egrave, "e`", 0x00e8},
{ "empty", 0, "{}" },
- { "epsilon", 0, "e" },
- { "equiv", 0, "==" },
- { "eta", 0, "e" },
- { "eth", LATIN1_eth, "d-" },
- { "euml", LATIN1_euml, "e\"" },
- { "euro", 0, "EUR" },
+ { "epsilon", 0, "e", 0x03b5},
+ { "equiv", 0, "==", 0x2261},
+ { "eta", 0, "e", 0x03b7},
+ { "eth", LATIN1_eth, "d-", 0x00f0},
+ { "euml", LATIN1_euml, "e\"", 0x00eb},
+ { "euro", 0, "EUR", 0x20ac},
{ "exist", 0, "TE" },
{ "fnof", 0, "f" },
{ "forall", 0, "FA" },
- { "frac12", LATIN1_frac12, " 1/2" },
- { "frac14", LATIN1_frac14, " 1/4" },
- { "frac34", LATIN1_frac34, " 3/4" },
+ { "frac12", LATIN1_frac12, " 1/2",0x00bd},
+ { "frac14", LATIN1_frac14, " 1/4",0x00bc},
+ { "frac34", LATIN1_frac34, " 3/4",0x00be},
{ "frasl", 0, "/" },
- { "gamma", 0, "g" },
- { "ge", 0, ">=" },
- { "gt", 0, ">" },
+ { "gamma", 0, "g", 0x03b3},
+ { "ge", 0, ">=", 0x2265},
+ { "gt", 0, ">", 0x003e},
{ "hArr", 0, "<=>" },
{ "harr", 0, "<->" },
// { "hearts", 0, "[hearts]" },
- { "hellip", 0, "..." },
- { "iacute", LATIN1_iacute, "i'" },
- { "icirc", LATIN1_icirc, "i^" },
- { "iexcl", LATIN1_iexcl, "!" },
- { "igrave", LATIN1_igrave, "i`" },
+ { "hellip", 0, "...", 0x2026},
+ { "iacute", LATIN1_iacute, "i'", 0x00ed},
+ { "icirc", LATIN1_icirc, "i^", 0x00ee},
+ { "iexcl", LATIN1_iexcl, "!", 0x00a1},
+ { "igrave", LATIN1_igrave, "i`", 0x00ec},
{ "image", 0, "Im" },
- { "infin", 0, "oo" },
- { "int", 0, "INT" },
- { "iota", 0, "i" },
- { "iquest", LATIN1_iquest, "?" },
+ { "infin", 0, "oo", 0x221e},
+ { "int", 0, "INT", 0x222b},
+ { "iota", 0, "i", 0x03b9},
+ { "iquest", LATIN1_iquest, "?", 0x00bf},
{ "isin", 0, "(-" },
- { "iuml", LATIN1_iuml, "i\"" },
- { "kappa", 0, "k" },
+ { "iuml", LATIN1_iuml, "i\"", 0x00ef},
+ { "kappa", 0, "k", 0x03ba},
{ "lArr", 0, "<=" },
- { "lambda", 0, "l" },
+ { "lambda", 0, "l", 0x03bb},
{ "lang", 0, "</" },
{ "laquo", LATIN1_laquo, "<<" },
- { "larr", 0, "<-" },
+ { "larr", 0, "<-", 0x2190},
// { "lceil", 0, "<|" },
{ "ldquo", 0, "\"" },
- { "le", 0, "<=" },
+ { "le", 0, "<=", 0x2264},
// { "lfloor", 0, "|<" },
{ "lowast", 0, "*" },
{ "loz", 0, "<>" },
{ "lsaquo", 0, "<" },
{ "lsquo", 0, "`" },
- { "lt", 0, "<" },
- { "macr", LATIN1_macr, "-" },
+ { "lt", 0, "<", 0x003c},
+ { "macr", LATIN1_macr, "-", 0x00af},
{ "mdash", 0, "--" },
- { "micro", LATIN1_micro, "my" },
- { "middot", LATIN1_middot, "." },
- { "minus", 0, "-" },
- { "mu", 0, "m" },
+ { "micro", LATIN1_micro, "my", 0x00b5},
+ { "middot", LATIN1_middot, ".", 0x00b7},
+ { "minus", 0, "-", 0x2212},
+ { "mu", 0, "m", 0x03bc},
{ "nabla", 0, "Nabla" },
- { "nbsp", LATIN1_nbsp, " " },
+ { "nbsp", LATIN1_nbsp, " ", 0x00a0},
{ "ndash", 0, "-" },
- { "ne", 0, "!=" },
+ { "ne", 0, "!=", 0x2260},
{ "ni", 0, "-)" },
{ "not", LATIN1_not, "NOT" },
{ "notin", 0, "!(-" },
{ "nsub", 0, "!(C" },
- { "ntilde", LATIN1_ntilde, "n~" },
- { "nu", 0, "n" },
- { "oacute", LATIN1_oacute, "o'" },
- { "ocirc", LATIN1_ocirc, "o^" },
+ { "ntilde", LATIN1_ntilde, "n~", 0x00f1},
+ { "nu", 0, "n", 0x03bd},
+ { "oacute", LATIN1_oacute, "o'", 0x00f3},
+ { "ocirc", LATIN1_ocirc, "o^", 0x00f4},
{ "oelig", 0, "oe" },
- { "ograve", LATIN1_ograve, "o`" },
+ { "ograve", LATIN1_ograve, "o`", 0x00f2},
{ "oline", LATIN1_macr, "-" },
- { "omega", 0, "o" },
- { "omicron", 0, "o" },
+ { "omega", 0, "o", 0x03c9},
+ { "omicron", 0, "o", 0x03bf},
{ "oplus", 0, "(+)" },
{ "or", 0, "OR" },
- { "ordf", LATIN1_ordf, "-a" },
- { "ordm", LATIN1_ordm, "-o" },
- { "oslash", LATIN1_oslash, "o/" },
- { "otilde", LATIN1_otilde, "o~" },
+ { "ordf", LATIN1_ordf, "-a", 0x00aa},
+ { "ordm", LATIN1_ordm, "-o", 0x00ba},
+ { "oslash", LATIN1_oslash, "o/", 0x00f8},
+ { "otilde", LATIN1_otilde, "o~", 0x00f5},
{ "otimes", 0, "(x)" },
- { "ouml", LATIN1_ouml, "o\"" },
- { "para", LATIN1_para, "P:" },
- { "part", 0, "PART" },
- { "permil", 0, " 0/00" },
+ { "ouml", LATIN1_ouml, "o\"", 0x00f6},
+ { "para", LATIN1_para, "P:", 0x00b6},
+ { "part", 0, "PART",0x2202},
+ { "permil", 0, " 0/00",0x2030},
{ "perp", 0, "-T" },
- { "phi", 0, "f" },
- { "pi", 0, "p" },
+ { "phi", 0, "f", 0x03c6},
+ { "pi", 0, "p", 0x03c0},
{ "piv", 0, "Pi" },
- { "plusmn", LATIN1_plusmn, "+/-" },
- { "pound", LATIN1_pound, "-L-" },
+ { "plusmn", LATIN1_plusmn, "+/-", 0x00b1},
+ { "pound", LATIN1_pound, "-L-", 0x00a3},
{ "prime", 0, "'" },
- { "prod", 0, "PROD" },
+ { "prod", 0, "PROD",0x220f},
{ "prop", 0, "0(" },
- { "psi", 0, "ps" },
+ { "psi", 0, "ps", 0x03c8},
{ "quot", 0, "\"" },
{ "rArr", 0, "=>" },
- { "radic", 0, "SQRT" },
+ { "radic", 0, "SQRT",0x221a},
{ "rang", 0, "/>" },
{ "raquo", LATIN1_raquo, ">>" },
- { "rarr", 0, "->" },
+ { "rarr", 0, "->", 0x2192},
// { "rceil", 0, ">|" },
{ "rdquo", 0, "\"" },
{ "real", 0, "Re" },
- { "reg", LATIN1_reg, "(R)" },
+ { "reg", LATIN1_reg, "(R)", 0x00ae},
// { "rfloor", 0, "|>" },
- { "rho", 0, "r" },
+ { "rho", 0, "r", 0x03c1},
{ "rsaquo", 0, ">" },
{ "rsquo", 0, "'" },
{ "sbquo", 0, "'" },
- { "scaron", 0, "s" },
+ { "scaron", 0, "s", 0x0161},
{ "sdot", 0, "DOT" },
- { "sect", LATIN1_sect, "S:" },
+ { "sect", LATIN1_sect, "S:", 0x00a7},
{ "shy", LATIN1_shy, "" },
- { "sigma", 0, "s" },
- { "sigmaf", 0, "s" },
+ { "sigma", 0, "s", 0x03c3},
+ { "sigmaf", 0, "s", 0x03c2},
{ "sim", 0, "~" },
// { "spades", 0, "[spades]" },
{ "sub", 0, "(C" },
{ "sube", 0, "(_" },
- { "sum", 0, "SUM" },
+ { "sum", 0, "SUM", 0x2211},
{ "sup", 0, ")C" },
- { "sup1", LATIN1_sup1, "^1" },
- { "sup2", LATIN1_sup2, "^2" },
- { "sup3", LATIN1_sup3, "^3" },
+ { "sup1", LATIN1_sup1, "^1", 0x00b9},
+ { "sup2", LATIN1_sup2, "^2", 0x00b2},
+ { "sup3", LATIN1_sup3, "^3", 0x00b3},
{ "supe", 0, ")_" },
- { "szlig", LATIN1_szlig, "ss" },
- { "tau", 0, "t" },
+ { "szlig", LATIN1_szlig, "ss", 0x00df},
+ { "tau", 0, "t", 0x03c4},
{ "there4", 0, ".:" },
- { "theta", 0, "th" },
- { "thorn", LATIN1_thorn, "th" },
- { "tilde", 0, "~" },
- { "times", LATIN1_times, "x" },
- { "trade", 0, "[TM]" },
+ { "theta", 0, "th", 0x03b8},
+ { "thorn", LATIN1_thorn, "th", 0x00fe},
+ { "tilde", 0, "~", 0x02dc},
+ { "times", LATIN1_times, "x", 0x00d7},
+ { "trade", 0, "[TM]",0x2122},
{ "uArr", 0, "^^" },
- { "uacute", LATIN1_uacute, "u'" },
+ { "uacute", LATIN1_uacute, "u'", 0x00fa},
{ "uarr", 0, "^" },
- { "ucirc", LATIN1_ucirc, "u^" },
- { "ugrave", LATIN1_ugrave, "u`" },
- { "uml", LATIN1_uml, "\"" },
- { "upsilon", 0, "u" },
- { "uuml", LATIN1_uuml, "u\"" },
+ { "ucirc", LATIN1_ucirc, "u^", 0x00fb},
+ { "ugrave", LATIN1_ugrave, "u`", 0x00f9},
+ { "uml", LATIN1_uml, "\"", 0x00a8},
+ { "upsilon", 0, "u", 0x03c5},
+ { "uuml", LATIN1_uuml, "u\"", 0x00fc},
{ "weierp", 0, "P" },
- { "xi", 0, "x" },
- { "yacute", LATIN1_yacute, "y'" },
- { "yen", LATIN1_yen, "YEN" },
- { "yuml", LATIN1_yuml, "y\"" },
- { "zeta", 0, "z" },
+ { "xi", 0, "x", 0x03be},
+ { "yacute", LATIN1_yacute, "y'", 0x00fd},
+ { "yen", LATIN1_yen, "YEN", 0x00a5},
+ { "yuml", LATIN1_yuml, "y\"", 0x00ff},
+ { "zeta", 0, "z", 0x03b6},
};
-extern int use_iso8859;
+extern int use_encoding;
/* ------------------------------------------------------------------------- */
+char ubuf[4];
+
+char *mkutf(unsigned long x)
+{
+ memset(ubuf, 0, 4);
+ if(x < 128) ubuf[0] = x;
+ else if(x < 0x800) {
+ ubuf[0] = (0xc0 | ((x >> 6) & 0x1f));
+ ubuf[1] = (0x80 | (x & 0x3f));
+ }
+ else {
+ ubuf[0] = (0xe0 | ((x >> 12) & 0x0f));
+ ubuf[1] = (0x80 | ((x >> 6) & 0x3f));
+ ubuf[2] = (0x80 | (x & 0x3f));
+ }
+ return ubuf;
+}
+
void
replace_sgml_entities(string *s)
{
@@ -330,9 +349,9 @@
*/
while (j < l && s->at(j) != '&') ++j;
/*
- * We could convert high-bit chars to "é" here if use_iso8859
- * is off, then let them be translated or not. Is the purpose of
- * !use_iso8859 to allow SGML entities to be seen, or to strongly
+ * We could convert high-bit chars to "é" here if USE_ASCII
+ * is on, then let them be translated or not. Is the purpose of
+ * USE_ASCII to allow SGML entities to be seen, or to strongly
* filter against high-ASCII chars that might blow up a terminal
* that doesn't speak ISO8859? For the moment, "allow SGML entities
* to be seen" -- no filtering here.
@@ -370,7 +389,11 @@
if (!isdigit(c)) break;
x = 10 * x + c - '0';
}
- if (use_iso8859 || (x < 128)) {
+ if (USE_UTF8) {
+ s->replace(beg, j - beg, mkutf(x));
+ j = beg + 1;
+ }
+ else if (USE_ISO8859 && (x < 256) || USE_ASCII && (x < 128)) {
s->replace(beg, j - beg, 1, (char) x);
j = beg + 1;
} else {
@@ -408,13 +431,17 @@
(int (*)(const void *, const void *)) strcmp
);
if (entity != NULL) {
- if (use_iso8859 && entity->iso8859code) {
+ if (USE_ISO8859 && entity->iso8859code) {
s->replace(beg, j - beg, 1, (char) entity->iso8859code);
j = beg + 1;
- } else if (entity->asciistr) {
+ } else if (USE_ASCII && entity->asciistr) {
s->replace(beg, j - beg, entity->asciistr);
j = beg + 1;
} /* else don't replace it at all, we don't have a translation */
+ else if(USE_UTF8 && entity->unicode) {
+ s->replace(beg, j - beg, mkutf(entity->unicode));
+ j = beg + 1;
+ }
}
} else {
; /* EXTENSION: Allow literal '&' sometimes. */
Index: html2text-1.3.2a/table.C
===================================================================
--- html2text-1.3.2a.orig/table.C 2008-09-20 14:01:44.415186916 +0300
+++ html2text-1.3.2a/table.C 2008-09-20 14:06:15.259781132 +0300
@@ -175,7 +175,7 @@
- (*number_of_columns_return - 1) * (column_spacing + 0),
Area::LEFT // Yields better results than "p->halign"!
));
- p->width = tmp.get() ? tmp->width() : 0;
+ p->width = tmp.get() ? tmp->utf_width() : 0;
}
p->minimized = false;
@@ -308,7 +308,7 @@
left_of_column + old_column_width - 1,
Area::LEFT // Yields better results than "lc.halign"!
));
- w = tmp->width();
+ w = tmp->utf_width();
if (w >= left_of_column + old_column_width) lc.minimized = true;
}
if (w > left_of_column + new_column_width) {