Support for handling multibyte encodings in cpstripped(), which

means that multibyte character encodings should work correctly
on all systems which support the POSIX wide character functions.

diff --git a/Makefile.am b/Makefile.am
index ad6b1e6..2c4dcfe 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,6 +37,7 @@ TESTS_ENVIRONMENT = MH_OBJ_DIR=@abs_builddir@ \
                    MH_TEST_DIR=@abs_builddir@/test/testdir \
                    auxexecdir=$(auxexecdir) bindir=$(bindir) \
                    mandir=$(mandir) sysconfdir=$(sysconfdir) \
+                   MULTIBYTE_ENABLED=$(MULTIBYTE_ENABLED) \
                    $(TESTS_SHELL) ## Keep at end of TESTS_ENVIRONMENT.
 ##
 ## Important note: the "cleanup" test should always be last
@@ -66,7 +67,7 @@ TESTS = test/bad-input/test-header \
        test/refile/test-refile \
        test/repl/test-if-str test/repl/test-trailing-newline \
        test/repl/test-multicomp \
-       test/scan/test-scan \
+       test/scan/test-scan test/scan/test-scan-multibyte \
        test/sequences/test-flist test/sequences/test-mark \
        test/whatnow/test-attach-detach test/whatnow/test-cd \
        test/whatnow/test-ls test/whom/test-whom \

diff --git a/configure.ac b/configure.ac
index ee79a0a..daec29c 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -441,11 +441,15 @@ dnl ---------------
 AC_CHECK_FUNCS([wcwidth mbtowc writev lstat nl_langinfo getutxent])
 
 dnl Check for multibyte character set support
-if test "x$ac_cv_header_wchar_h" = "xyes" -a "x$ac_cv_header_wctype_h" = "xyes" \
-    -a "x$ac_cv_func_wcwidth" = "xyes" -a "x$ac_cv_func_mbtowc" = "xyes"; then
-  AC_DEFINE(MULTIBYTE_SUPPORT, 1,
-    [Define to enable support for multibyte character sets.])
-fi
+AS_IF([test "x$ac_cv_header_wchar_h" = "xyes" -a \
+           "x$ac_cv_header_wctype_h" = "xyes" -a \
+           "x$ac_cv_func_wcwidth" = "xyes" -a \
+           "x$ac_cv_func_mbtowc" = "xyes"],
+      [AC_DEFINE([MULTIBYTE_SUPPORT], [1],
+                [Define to enable support for multibyte character sets.])
+       MULTIBYTE_ENABLED=1],
+      [MULTIBYTE_ENABLED=0])
+AC_SUBST([MULTIBYTE_ENABLED])
 
 dnl -------------------
 dnl CHECK FOR LIBRARIES

diff --git a/sbr/fmt_scan.c b/sbr/fmt_scan.c
index 3f004e5..74cfd92 100644 (file)
--- a/sbr/fmt_scan.c
+++ b/sbr/fmt_scan.c
@@ -208,37 +208,64 @@ cptrimmed(char **dest, char *str, unsigned int wid, char fill, size_t n) {
 }
 
 static void
-cpstripped (char **start, char *end, char *str)
+cpstripped (char **dest, char *end, char *str)
 {
-    int c;
-    char *s = str;
+    int prevCtrl = 1;  /* This is 1 so we strip out leading spaces */
+    int len;
+#ifdef MULTIBYTE_SUPPORT
+    int char_len;
+    wchar_t wide_char;
+#endif /* MULTIBYTE_SUPPORT */
 
-    if (!s)
+    if (!str)
        return;
 
-    /* skip any initial control characters or spaces */
-    while ((c = (unsigned char) *s) &&
-#ifdef LOCALE
-           (iscntrl(c) || isspace(c)))
-#else
-           (c <= 32))
-#endif
-       s++;
-
-    /* compact repeated control characters and spaces into a single space */
-    while((c = (unsigned char) *s++) && *start < end)
-       if (!iscntrl(c) && !isspace(c))
-           *(*start)++ = c;
-       else {
-           while ((c = (unsigned char) *s) &&
-#ifdef LOCALE
-                   (iscntrl(c) || isspace(c)))
-#else
-                   (c <= 32))
-#endif
-               s++;
-           *(*start)++ = ' ';
+    len = strlen(str);
+
+#ifdef MULTIBYTE_SUPPORT
+    mbtowc(NULL, NULL, 0);  /* Reset shift state */
+#endif /* MULTIBYTE_SUPPORT */
+
+    /*
+     * Process each character at a time; if we have multibyte support
+     * then deal with that here.
+     */
+
+    while (*str != '\0' && len > 0 && *dest < end) {
+#ifdef MULTIBYTE_SUPPORT
+       char_len = mbtowc(&wide_char, str, len);
+
+       if (char_len <= 0 || *dest + char_len > end)
+           break;
+
+       len -= char_len;
+
+       if (iswcntrl(wide_char) || iswspace(wide_char)) {
+           str += char_len;
+#else /* MULTIBYTE_SUPPORT */
+       int c = *str;
+       len--;
+       if (iscntrl(c) || isspace(c)) {
+           str++;
+#endif /* MULTIBYTE_SUPPORT */
+           if (! prevCtrl) {
+               *(*dest)++ = ' ';
+           }
+
+           prevCtrl = 1;
+           continue;
        }
+
+       prevCtrl = 0;
+
+#ifdef MULTIBYTE_SUPPORT
+       memcpy(*dest, str, char_len);
+       str += char_len;
+       *dest += char_len;
+#else /* MULTIBYE_SUPPORT */
+       *(*dest)++ = *str++
+#endif /* MULTIBYTE_SUPPORT */
+    }
 }
 
 static char *lmonth[] = { "January",  "February","March",   "April",

diff --git a/test/common.sh.in b/test/common.sh.in
index b1fd484..705e993 100644 (file)
--- a/test/common.sh.in
+++ b/test/common.sh.in
@@ -14,7 +14,9 @@ test -z "$bindir"  &&  bindir="@bindir@"
 test -z "$mandir"  &&  mandir="@mandir@"
 test -z "$sysconfdir"  &&  sysconfdir="@sysconfdir@"
 test -z "$pagerpath"  &&  pagerpath="@pagerpath@"
+test -z "$MULTIBYTE_ENABLED"  &&  MULTIBYTE_ENABLED="@MULTIBYTE_ENABLED@"
 export MH_TEST_DIR auxexecdir bindir mandir sysconfdir pagerpath
+export MULTIBYTE_ENABLED
 
 test -z "$MH_INST_DIR"  &&  MH_INST_DIR=${MH_TEST_DIR}/inst
 export MH_INST_DIR

diff --git a/test/scan/test-scan-multibyte b/test/scan/test-scan-multibyte
new file mode 100755 (executable)
index 0000000..bd0da47
--- /dev/null
+++ b/test/scan/test-scan-multibyte
@@ -0,0 +1,55 @@
+#!/bin/sh
+############################################################
+#
+# Test scan to see if multibyte support (UTF-8 locale) works
+#
+# Other tests will get the normal ASCII case, so all we care
+# about here is UTF-8 encoded headers (RFC 2047).
+#
+# Note that this file should be edited via a UTF-8 aware
+# editor, since UTF-8 characters are in it.
+#
+############################################################
+
+set -e
+
+if test -z "${MH_OBJ_DIR}"; then
+    srcdir=`dirname "$0"`/../..
+    MH_OBJ_DIR=`cd "$srcdir" && pwd`; export MH_OBJ_DIR
+fi
+
+. "$MH_OBJ_DIR/test/common.sh"
+
+setup_test
+
+if test "${MULTIBYTE_ENABLED}" -ne 1; then
+    test_skip "configure did not detect multibyte support"
+fi
+
+export LC_ALL=en_US.UTF-8
+
+#
+# Create a test message with RFC 2047 headers we can scan
+#
+
+cat > "${MH_TEST_DIR}/Mail/inbox/11" <<EOF
+From: David =?utf-8?q?=EF=AC=86?= Hubbins <hubbins@example.com>
+To: Sir Denis =?utf-8?q?Eton=E2=80=93Hogg? <sirdenis@example.com>
+Date: Friday, 2 Mar 1984 00:00:00
+Subject: =?utf-8?q?Spin=CC=88al_Tap_=E2=86=92_Tap_into_America!?=
+
+Things are looking great!
+EOF
+
+expected=$MH_TEST_DIR/$$.expected
+actual=$MH_TEST_DIR/$$.actual
+
+cat > $expected <<EOF
+  11  03/02 David st Hubbins    Spin̈al Tap → Tap into America!<<Things are
+EOF
+
+scan -width 80 +inbox 11 > $actual || exit 1
+
+check "$expected" "$actual"
+
+exit $failed