Added SMTPUTF8 (RFC 6531) support:

    1) Allow 8-bit characters in addresses.
    2) Disable RFC 2047 encoding of (all) header field bodies, via
       mhbuild -headerencoding 8bit.
    3) Have post enable it in the SMTP transaction.
    4) If enabled, add the commands to the SMTP transaction
       (MAIL FROM: <address> BODY=8BITMIME SMTPUTF8).
Some things may changed, such as the 8bit encoding algorithm name.

diff --git a/Makefile.am b/Makefile.am
index 0db9cfb..ca46a4c 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -94,7 +94,8 @@ TESTS = test/ali/test-ali test/anno/test-anno \
        test/post/test-post-dcc test/post/test-post-fcc \
        test/post/test-post-multifrom test/post/test-post-envelope \
        test/post/test-post-group test/post/test-mts test/post/test-messageid \
-       test/post/test-sendfiles test/prompter/test-prompter \
+       test/post/test-rfc6531 test/post/test-sendfiles \
+       test/prompter/test-prompter \
        test/rcv/test-rcvdist test/rcv/test-rcvpack test/rcv/test-rcvstore \
        test/rcv/test-rcvtty test/refile/test-refile test/repl/test-convert \
        test/repl/test-if-str test/repl/test-trailing-newline \

diff --git a/docs/pending-release-notes b/docs/pending-release-notes
index 9428772..70ed4ff 100644 (file)
--- a/docs/pending-release-notes
+++ b/docs/pending-release-notes
@@ -48,6 +48,8 @@ NEW FEATURES
 - post(8) -snoop now attempts to decode base64-encoded SMTP traffic.
 - folder(1) -nocreate now prints a warning message for a non-existant folder.
 - mhfixmsg(1) now allows -decodetext binary, though 8bit is still the default.
+- Support for SMTPUTF8 (RFC 6531) has been added.  mhshow(1) already supported
+  RFC 6532, assuming all 8-bit message header field bodies are UTF-8.
 
 -----------------
 OBSOLETE FEATURES

diff --git a/h/mf.h b/h/mf.h
index 585a83a..1272640 100644 (file)
--- a/h/mf.h
+++ b/h/mf.h
@@ -74,5 +74,5 @@ int isfrom(const char *);
 int lequal (const char *, const char *);
 int mfgets (FILE *, char **);
 char *legal_person (const char *);
-struct adrx *getadrx (const char *);
+struct adrx *getadrx (const char *, int);
 

diff --git a/man/mhbuild.man b/man/mhbuild.man
index 3d2fbba..c1237a7 100644 (file)
--- a/man/mhbuild.man
+++ b/man/mhbuild.man
@@ -1,4 +1,4 @@
-.TH MHBUILD %manext1% "December 14, 2014" "%nmhversion%"
+.TH MHBUILD %manext1% "September 23, 2016" "%nmhversion%"
 .\"
 .\" %nmhwarning%
 .\"
@@ -765,9 +765,13 @@ switch will indicate which algorithm to use when encoding any message headers
 that contain 8\-bit characters.
 The valid arguments are
 .I base64
-for based\-64 encoding and
+for based\-64 encoding,
 .I quoted
-for quoted\-printable encoding.
+for quoted\-printable encoding, and
+.I 8bit
+for raw 8-bit encoding, which requires that all header field bodies be
+encoded as UTF-8 (RFC 6530) and that the SMTP server support SMTPUTF8
+(RFC 6531).
 The
 .B \-autoheaderencoding
 switch will instruct
@@ -1038,25 +1042,31 @@ Template for composing contents.
 .IR mhstore (1)
 .PP
 .I "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies"
-(RFC 2045),
+(RFC 2045)
 .PP
 .I "Multipurpose Internet Mail Extensions (MIME) Part Two: Media Types"
-(RFC 2046),
+(RFC 2046)
 .PP
 .I "Multipurpose Internet Mail Extensions (MIME) Part Three: Message Header Extensions for Non-ASCII Text"
-(RFC 2047),
+(RFC 2047)
 .PP
 .I "MIME Parameter Value and Encoded Word Extensions: Character Sets, Languages, and Continuations"
 (RFC 2231)
 .PP
 .I "Proposed Standard for Message Encapsulation"
-(RFC 934),
+(RFC 934)
 .PP
 .I "The Content-MD5 Header Field"
-(RFC 1864),
+(RFC 1864)
 .PP
 .I "Definition of the URL MIME External-Body Access-Type"
-(RFC 2017),
+(RFC 2017)
+.PP
+.I "Overview and Framework for Internationalized Email"
+(RFC 6530)
+.PP
+.I "SMTP Extension for Internationalized Email"
+(RFC 6531)
 .SH DEFAULTS
 .nf
 \-autoheaderencoding

diff --git a/mts/smtp/smtp.c b/mts/smtp/smtp.c
index 79c1e79..47215a8 100644 (file)
--- a/mts/smtp/smtp.c
+++ b/mts/smtp/smtp.c
@@ -593,9 +593,22 @@ rclient (char *server, char *service)
 }
 
 int
-sm_winit (char *from)
+sm_winit (char *from, int smtputf8)
 {
-    switch (smtalk (SM_MAIL, "MAIL FROM:<%s>", from)) {
+    const char *const mail_parameters = smtputf8
+        ? " BODY=8BITMIME SMTPUTF8"
+        : "";
+
+    /* Just for information, if an attempt is made to send to an 8-bit
+       address without specifying SMTPUTF8, Gmail responds with
+       555 5.5.2 Syntax error.
+       Gmail doesn't require the 8BITMIME, but RFC 6531 Sec. 1.2 does. */
+    if (smtputf8  &&  (! EHLOset("8BITMIME") || ! EHLOset("SMTPUTF8"))) {
+       sm_end (NOTOK);
+       return RP_UCMD;
+    }
+
+    switch (smtalk (SM_MAIL, "MAIL FROM:<%s>%s", from, mail_parameters)) {
        case 250: 
            sm_addrs = 0;
            return RP_OK;

diff --git a/mts/smtp/smtp.h b/mts/smtp/smtp.h
index 2ed5cc2..3063866 100644 (file)
--- a/mts/smtp/smtp.h
+++ b/mts/smtp/smtp.h
@@ -18,7 +18,7 @@ struct smtp {
 /* int client (); */
 int sm_init (char *, char *, char *, int, int, int, int, int, char *, char *,
              const char *, int);
-int sm_winit (char *);
+int sm_winit (char *, int);
 int sm_wadr (char *, char *, char *);
 int sm_waend (void);
 int sm_wtxt (char *, int);

diff --git a/sbr/addrsbr.c b/sbr/addrsbr.c
index 555822e..3b57f20 100644 (file)
--- a/sbr/addrsbr.c
+++ b/sbr/addrsbr.c
@@ -82,12 +82,13 @@ char *
 getname (const char *addrs)
 {
     struct adrx *ap;
-
+    const char *const eai = context_find ("eai");
     pers = mbox = host = route = grp = note = NULL;
     err[0] = '\0';
 
-    if ((ap = getadrx (addrs ? addrs : "")) == NULL)
+    if ((ap = getadrx (addrs ? addrs : "", eai && strncasecmp (eai, "8bit", 4) == 0)) == NULL) {
        return NULL;
+    }
 
     strncpy (adr, ap->text, sizeof(adr));
     pers = ap->pers;

diff --git a/sbr/mf.c b/sbr/mf.c
index c11b0c1..136f94c 100644 (file)
--- a/sbr/mf.c
+++ b/sbr/mf.c
@@ -174,8 +174,9 @@ static char adr[BUFSIZ];
 static struct adrx  adrxs2;
 
 
+/* eai = Email Address Internationalization */
 struct adrx *
-getadrx (const char *addrs)
+getadrx (const char *addrs, int eai)
 {
     register char *bp;
     register struct adrx *adrxp = &adrxs2;
@@ -230,13 +231,15 @@ getadrx (const char *addrs)
            break;
        }
 
-    /*
-     * Reject the address if key fields contain 8bit characters
-     */
+    if (! eai) {
+        /*
+         * Reject the address if key fields contain 8bit characters
+         */
 
-    if (contains8bit(mbox, NULL) || contains8bit(host, NULL) ||
-       contains8bit(path, NULL) || contains8bit(grp, NULL)) {
-       strcpy(err, "Address contains 8-bit characters");
+        if (contains8bit(mbox, NULL) || contains8bit(host, NULL) ||
+            contains8bit(path, NULL) || contains8bit(grp, NULL)) {
+            strcpy(err, "Address contains 8-bit characters");
+        }
     }
 
     if (err[0])

diff --git a/test/fakesmtp.c b/test/fakesmtp.c
index 42d4f18..449fdec 100644 (file)
--- a/test/fakesmtp.c
+++ b/test/fakesmtp.c
@@ -101,7 +101,11 @@ main(int argc, char *argv[])
                        smtp_state = SMTP_DATA;
                        continue;
                }
-               if (xoauth != NULL) {
+               if (xoauth == NULL) {
+                       putcrlf(conn, "250-ready");
+                       putcrlf(conn, "250-8BITMIME");
+                       putcrlf(conn, "250-SMTPUTF8");
+               } else {
                        /* XOAUTH2 support enabled; handle EHLO and AUTH. */
                        if (strncmp(line, "EHLO", 4) == 0) {
                                putcrlf(conn, "250-ready");

diff --git a/test/post/test-rfc6531 b/test/post/test-rfc6531
new file mode 100755 (executable)
index 0000000..1df9a29
--- /dev/null
+++ b/test/post/test-rfc6531
@@ -0,0 +1,73 @@
+#!/bin/sh
+######################################################
+#
+# Test SMTPUTF8 (RFC 6531) support.
+#
+######################################################
+
+set -e
+
+if test -z "${MH_OBJ_DIR}"; then
+    srcdir=`dirname $0`/../..
+    MH_OBJ_DIR=`cd $srcdir && pwd`; export MH_OBJ_DIR
+fi
+
+. "${srcdir}/test/post/test-post-common.sh"
+
+setup_test
+mhl="${MH_LIBEXEC_DIR}/mhl"
+
+#### Make sure that html-to-text conversion is what we expect.
+require_locale en_US.utf-8 en_US.utf8
+LC_ALL=en_US.UTF-8; export LC_ALL
+
+#### Enable EAI in mhbuild, via the profile.
+sed "s%^\(mhbuild: .*\)%\1 -headerencoding 8bit%" "$MH" >"$MH.new"
+mv -f "$MH.new" "$MH"
+
+# check SMTPUTF8 basic
+start_test 'SMTPUTF8 basic'
+
+cat > "${testname}.expected" <<EOF
+EHLO nosuchhost.example.com
+MAIL FROM:<senderø@example.com> BODY=8BITMIME SMTPUTF8
+RCPT TO:<recipientæ@example.com>
+DATA
+From: senderø@example.com
+To: recipientæ@example.com
+Subject: Blåbærsyltetøy
+MIME-Version: 1.0
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: 8bit
+Date:
+
+Blåbærsyltetøy er veldig godt. Jeg liker blåbærsyltetøy.
+Jøran.
+
+Text source:
+Arnt Gulbrandsen and Jiankang Yao, "Email Address Internationalization",
+Computing Edge 1:6, June 2015,
+http://www.computer.org/cms/computing-edge/ce-jun15-final.pdf, p. 50.
+.
+QUIT
+EOF
+
+cat > "${MH_TEST_DIR}/Mail/draft" <<EOF
+From: senderø@example.com
+To: recipientæ@example.com
+Subject: Blåbærsyltetøy
+--------
+Blåbærsyltetøy er veldig godt. Jeg liker blåbærsyltetøy.
+Jøran.
+
+Text source:
+Arnt Gulbrandsen and Jiankang Yao, "Email Address Internationalization",
+Computing Edge 1:6, June 2015,
+http://www.computer.org/cms/computing-edge/ce-jun15-final.pdf, p. 50.
+EOF
+
+test_post "${testname}.actual" "${testname}.expected" -eai
+
+
+finish_test
+exit $failed

diff --git a/test/send/test-sendfrom b/test/send/test-sendfrom
index 68ddb27..2d704f1 100755 (executable)
--- a/test/send/test-sendfrom
+++ b/test/send/test-sendfrom
@@ -53,10 +53,19 @@ EOF
 cat > "${testname}.post-expected" <<EOF
 <= 220 Not really an ESMTP server
 => EHLO nosuchhost.example.com
+<= 250-ready
+<= 250-8BITMIME
+<= 250-SMTPUTF8
 <= 250 I'll buy that for a dollar!
 => MAIL FROM:<nobody@example.com>
+<= 250-ready
+<= 250-8BITMIME
+<= 250-SMTPUTF8
 <= 250 I'll buy that for a dollar!
 => RCPT TO:<somebody@example.com>
+<= 250-ready
+<= 250-8BITMIME
+<= 250-SMTPUTF8
 <= 250 I'll buy that for a dollar!
 => DATA
 <= 354 Go ahead

diff --git a/uip/mhbuild.c b/uip/mhbuild.c
index a0903c5..8684c70 100644 (file)
--- a/uip/mhbuild.c
+++ b/uip/mhbuild.c
@@ -56,9 +56,11 @@ DEFINE_SWITCH_ENUM(MHBUILD);
 DEFINE_SWITCH_ARRAY(MHBUILD, switches);
 #undef X
 
+/* 8bit is Email Address Internationalization. */
 #define MIMEENCODING_SWITCHES \
     X("base64", 0, BASE64SW) \
     X("quoted-printable", 0, QUOTEDPRINTSW) \
+    X("8bit", 0, EIGHTBITSW) \
 
 #define X(sw, minchars, id) id,
 DEFINE_SWITCH_ENUM(MIMEENCODING);
@@ -244,6 +246,9 @@ main (int argc, char **argv)
                case QUOTEDPRINTSW:
                    header_encoding = CE_QUOTED;
                    break;
+               case EIGHTBITSW:
+                   header_encoding = CE_8BIT;
+                   break;
                default:
                    adios (NULL, "Internal error: algorithm %s", cp);
                }

diff --git a/uip/mhbuildsbr.c b/uip/mhbuildsbr.c
index b998993..26d8524 100644 (file)
--- a/uip/mhbuildsbr.c
+++ b/uip/mhbuildsbr.c
@@ -360,15 +360,17 @@ finish_field:
     }
     m_getfld_state_destroy (&gstate);
 
-    /*
-     * Iterate through the list of headers and call the function to MIME-ify
-     * them if required.
-     */
-
-    for (hp = ct->c_first_hf; hp != NULL; hp = hp->next) {
-       if (encode_rfc2047(hp->name, &hp->value, header_encoding, NULL)) {
-           adios(NULL, "Unable to encode header \"%s\"", hp->name);
-       }
+    if (header_encoding != CE_8BIT) {
+        /*
+         * Iterate through the list of headers and call the function to MIME-ify
+         * them if required.
+         */
+
+        for (hp = ct->c_first_hf; hp != NULL; hp = hp->next) {
+            if (encode_rfc2047(hp->name, &hp->value, header_encoding, NULL)) {
+                adios(NULL, "Unable to encode header \"%s\"", hp->name);
+            }
+        }
     }
 
     /*
@@ -1748,8 +1750,11 @@ build_headers (CT ct, int header_encoding)
     if (ct->c_descr) {
        np = add (DESCR_FIELD, NULL);
        vp = concat (" ", ct->c_descr, NULL);
-       if (encode_rfc2047(DESCR_FIELD, &vp, header_encoding, NULL))
-           adios(NULL, "Unable to encode %s header", DESCR_FIELD);
+       if (header_encoding != CE_8BIT) {
+           if (encode_rfc2047(DESCR_FIELD, &vp, header_encoding, NULL)) {
+               adios(NULL, "Unable to encode %s header", DESCR_FIELD);
+           }
+       }
        add_header (ct, np, vp);
     }
 

diff --git a/uip/post.c b/uip/post.c
index 0fee356..6e2367a 100644 (file)
--- a/uip/post.c
+++ b/uip/post.c
@@ -64,6 +64,8 @@
     X("library directory", -7, LIBSW) /* interface from send, whom */ \
     X("mime", 0, MIMESW) \
     X("nomime", 0, NMIMESW) \
+    X("eai", 0, EAISW) \
+    X("noeai", 0, NEAISW) \
     X("msgid", 0, MSGDSW) \
     X("nomsgid", 0, NMSGDSW) \
     X("verbose", 0, VERBSW) \
@@ -223,6 +225,7 @@ static int badmsg = 0;              /* message has bad semantics             */
 static int verbose = 0;                /* spell it out                          */
 static int format = 1;         /* format addresses                      */
 static int mime = 0;           /* use MIME-style encapsulations for Bcc */
+static int eai = 0;            /* use EAI (SMTPUTF8)                    */
 static int msgid = 0;          /* add msgid                             */
 static int debug = 0;          /* debugging post                        */
 static int watch = 0;          /* watch the delivery process            */
@@ -398,6 +401,13 @@ main (int argc, char **argv)
                    mime = 0;
                    continue;
 
+               case EAISW:
+                   eai = 1;
+                   continue;
+               case NEAISW:
+                   eai = 0;
+                   continue;
+
                case MSGDSW: 
                    msgid++;
                    continue;
@@ -585,6 +595,12 @@ main (int argc, char **argv)
     if ((in = fopen (msg, "r")) == NULL)
        adios (msg, "unable to open");
 
+    /* Support EAI. */
+    if (eai) {
+        /* Add eai profile entry, to pass 8bit setting to getname()/getadrx(). */
+        add_profile_entry ("eai", "8bit");
+    }
+
     start_headers ();
     if (debug) {
        verbose++;
@@ -1634,12 +1650,13 @@ post (char *file, int bccque, int talk, char *envelope, int oauth_flag,
                break;
        }
     } else {
-        if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch,
-                                        verbose, snoop, sasl, saslssf,
+       if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch,
+                                       verbose, snoop, sasl, saslssf,
                                        saslmech, user,
                                        oauth_flag ? auth_svc : NULL, tls))
-            || rp_isbad (retval = sm_winit (envelope)))
+            || rp_isbad (retval = sm_winit (envelope, eai))) {
            die (NULL, "problem initializing server; %s", rp_string (retval));
+       }
 
         do_addresses (bccque, talk && verbose);
         if ((fd = open (file, O_RDONLY)) == NOTOK)
@@ -1674,13 +1691,15 @@ verify_all_addresses (int talk, char *envelope, int oauth_flag, char *auth_svc)
 
     sigon ();
 
-    if (!whomsw || checksw)
+    if (!whomsw || checksw) {
        if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch,
                                        verbose, snoop, sasl, saslssf,
                                        saslmech, user,
                                        oauth_flag ? auth_svc : NULL, tls))
-               || rp_isbad (retval = sm_winit (envelope)))
+               || rp_isbad (retval = sm_winit (envelope, eai))) {
            die (NULL, "problem initializing server; %s", rp_string (retval));
+       }
+    }
 
     if (talk && !whomsw)
        printf (" -- Address Verification --\n");

diff --git a/uip/send.c b/uip/send.c
index 5dcdce0..d8a92cb 100644 (file)
--- a/uip/send.c
+++ b/uip/send.c
@@ -42,6 +42,8 @@
     X("noforward", 0, NFORWSW) \
     X("mime", 0, MIMESW) \
     X("nomime", 0, NMIMESW) \
+    X("eai", 0, EAISW) \
+    X("noeai", 0, NEAISW) \
     X("msgid", 0, MSGDSW) \
     X("nomsgid", 0, NMSGDSW) \
     X("push", 0, PUSHSW) \
@@ -236,6 +238,13 @@ main (int argc, char **argv)
                    vec[vecp++] = --cp;
                    continue;
 
+               case EAISW:
+                   vec[vecp++] = --cp;
+                   continue;
+               case NEAISW:
+                   vec[vecp++] = --cp;
+                   continue;
+
                case SNOOPSW:
                     snoop++;
                    vec[vecp++] = --cp;
@@ -478,7 +487,7 @@ go_to_it:
     closefds (3);
 
     for (msgnum = 0; msgnum < msgp; msgnum++) {
-       switch (sendsbr (vec, vecp, program, msgs[msgnum], &st, 1, auth_svc)) {
+        switch (sendsbr (vec, vecp, program, msgs[msgnum], &st, 1, auth_svc)) {
            case DONE: 
                done (++status);
            case NOTOK: 

diff --git a/uip/sendsbr.c b/uip/sendsbr.c
index c4d2502..5da67fc 100644 (file)
--- a/uip/sendsbr.c
+++ b/uip/sendsbr.c
@@ -76,23 +76,44 @@ sendsbr (char **vec, int vecp, char *program, char *draft, struct stat *st,
        by longjmp. */
     volatile int nvecs = vecp;
     int *nvecsp = (int *) &nvecs;
+    int eai = 0;
 
     /*
      * Run the mimebuildproc (which is by default mhbuild) on the message
      * with the addition of the "-auto" flag
      */
 
+    /* Make sure that EAI support is set up before calling buildmimeproc. */
+    for (i = 0; i < vecp; ++i) {
+        if (strcasecmp (vec[i], "-eai") == 0) {
+            eai = 1;
+        } else if (strcasecmp (vec[i], "-noeai") == 0) {
+            eai = 0;
+        }
+    }
+
     switch (child = fork()) {
     case NOTOK:
        adios("fork", "unable to");
        break;
 
     case OK:
-       buildvec = argsplit(buildmimeproc, &buildprogram, &i);
+        buildvec = argsplit(buildmimeproc, &buildprogram, &i);
        buildvec[i++] = "-auto";
        if (distfile)
            buildvec[i++] = "-dist";
        buildvec[i++] = (char *) drft;
+       if (eai) {
+           /* Add eai profile entry, to pass 8bit setting to
+              getname()/getadrx().  This doesn't seem to be necessary
+               now, but it's here just in case the code changes
+               later. */
+           add_profile_entry("eai", "8bit");
+
+           /* Add mhbuild switch to enable 8bit headers. */
+           buildvec[i++] = "-headerencoding";
+           buildvec[i++] = "8bit";
+       }
        buildvec[i] = NULL;
        execvp(buildprogram, buildvec);
        fprintf(stderr, "unable to exec ");

diff --git a/uip/whatnowsbr.c b/uip/whatnowsbr.c
index 80386e9..c2f8bbf 100644 (file)
--- a/uip/whatnowsbr.c
+++ b/uip/whatnowsbr.c
@@ -923,6 +923,8 @@ buildfile (char **argp, char *file)
     X("noforward", 0, NFORWSW) \
     X("mime", 0, MIMESW) \
     X("nomime", 0, NMIMESW) \
+    X("eai", 0, EAISW) \
+    X("noeai", 0, NEAISW) \
     X("msgid", 0, MSGDSW) \
     X("nomsgid", 0, NMSGDSW) \
     X("push", 0, SPSHSW) \
@@ -1129,6 +1131,8 @@ sendit (char *sp, char **arg, char *file, int pushed)
                case NBITSTUFFSW:
                case MIMESW:
                case NMIMESW:
+               case EAISW:
+               case NEAISW:
                case MSGDSW:
                case NMSGDSW:
                case WATCSW:

diff --git a/uip/whom.c b/uip/whom.c
index b1e8b1e..081978a 100644 (file)
--- a/uip/whom.c
+++ b/uip/whom.c
@@ -42,6 +42,8 @@
     X("tls", TLSminc(-3), TLSSW) \
     X("initialtls", TLSminc(-10), INITTLSSW) \
     X("notls", TLSminc(-5), NTLSSW) \
+    X("eai", 0, EAISW) \
+    X("noeai", 0, NEAISW) \
     X("mts smtp|sendmail/smtp|sendmail/pipe", 0, MTSSW) \
 
 #define X(sw, minchars, id) id,
@@ -100,6 +102,8 @@ main (int argc, char **argv)
                case TLSSW:
                case INITTLSSW:
                case NTLSSW:
+                case EAISW:
+                case NEAISW:
                    vec[vecp++] = --cp;
                    continue;