From: David Levine Date: Fri, 23 Sep 2016 15:55:20 +0000 (-0400) Subject: Added SMTPUTF8 (RFC 6531) support: X-Git-Url: https://diplodocus.org/git/nmh/commitdiff_plain/a2c417d3e1641ba4ab151b7945b90f55176705b1?ds=sidebyside Added SMTPUTF8 (RFC 6531) support: 1) Allow 8-bit characters in addresses. 2) Disable RFC 2047 encoding of (all) header field bodies, via mhbuild -headerencoding 8bit. 3) Have post enable it in the SMTP transaction. 4) If enabled, add the commands to the SMTP transaction (MAIL FROM:
BODY=8BITMIME SMTPUTF8). Some things may changed, such as the 8bit encoding algorithm name. --- diff --git a/Makefile.am b/Makefile.am index 0db9cfb5..ca46a4cd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -94,7 +94,8 @@ TESTS = test/ali/test-ali test/anno/test-anno \ test/post/test-post-dcc test/post/test-post-fcc \ test/post/test-post-multifrom test/post/test-post-envelope \ test/post/test-post-group test/post/test-mts test/post/test-messageid \ - test/post/test-sendfiles test/prompter/test-prompter \ + test/post/test-rfc6531 test/post/test-sendfiles \ + test/prompter/test-prompter \ test/rcv/test-rcvdist test/rcv/test-rcvpack test/rcv/test-rcvstore \ test/rcv/test-rcvtty test/refile/test-refile test/repl/test-convert \ test/repl/test-if-str test/repl/test-trailing-newline \ diff --git a/docs/pending-release-notes b/docs/pending-release-notes index 9428772e..70ed4ff6 100644 --- a/docs/pending-release-notes +++ b/docs/pending-release-notes @@ -48,6 +48,8 @@ NEW FEATURES - post(8) -snoop now attempts to decode base64-encoded SMTP traffic. - folder(1) -nocreate now prints a warning message for a non-existant folder. - mhfixmsg(1) now allows -decodetext binary, though 8bit is still the default. +- Support for SMTPUTF8 (RFC 6531) has been added. mhshow(1) already supported + RFC 6532, assuming all 8-bit message header field bodies are UTF-8. ----------------- OBSOLETE FEATURES diff --git a/h/mf.h b/h/mf.h index 585a83a4..1272640b 100644 --- a/h/mf.h +++ b/h/mf.h @@ -74,5 +74,5 @@ int isfrom(const char *); int lequal (const char *, const char *); int mfgets (FILE *, char **); char *legal_person (const char *); -struct adrx *getadrx (const char *); +struct adrx *getadrx (const char *, int); diff --git a/man/mhbuild.man b/man/mhbuild.man index 3d2fbbaf..c1237a75 100644 --- a/man/mhbuild.man +++ b/man/mhbuild.man @@ -1,4 +1,4 @@ -.TH MHBUILD %manext1% "December 14, 2014" "%nmhversion%" +.TH MHBUILD %manext1% "September 23, 2016" "%nmhversion%" .\" .\" %nmhwarning% .\" @@ -765,9 +765,13 @@ switch will indicate which algorithm to use when encoding any message headers that contain 8\-bit characters. The valid arguments are .I base64 -for based\-64 encoding and +for based\-64 encoding, .I quoted -for quoted\-printable encoding. +for quoted\-printable encoding, and +.I 8bit +for raw 8-bit encoding, which requires that all header field bodies be +encoded as UTF-8 (RFC 6530) and that the SMTP server support SMTPUTF8 +(RFC 6531). The .B \-autoheaderencoding switch will instruct @@ -1038,25 +1042,31 @@ Template for composing contents. .IR mhstore (1) .PP .I "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies" -(RFC 2045), +(RFC 2045) .PP .I "Multipurpose Internet Mail Extensions (MIME) Part Two: Media Types" -(RFC 2046), +(RFC 2046) .PP .I "Multipurpose Internet Mail Extensions (MIME) Part Three: Message Header Extensions for Non-ASCII Text" -(RFC 2047), +(RFC 2047) .PP .I "MIME Parameter Value and Encoded Word Extensions: Character Sets, Languages, and Continuations" (RFC 2231) .PP .I "Proposed Standard for Message Encapsulation" -(RFC 934), +(RFC 934) .PP .I "The Content-MD5 Header Field" -(RFC 1864), +(RFC 1864) .PP .I "Definition of the URL MIME External-Body Access-Type" -(RFC 2017), +(RFC 2017) +.PP +.I "Overview and Framework for Internationalized Email" +(RFC 6530) +.PP +.I "SMTP Extension for Internationalized Email" +(RFC 6531) .SH DEFAULTS .nf \-autoheaderencoding diff --git a/mts/smtp/smtp.c b/mts/smtp/smtp.c index 79c1e79b..47215a8b 100644 --- a/mts/smtp/smtp.c +++ b/mts/smtp/smtp.c @@ -593,9 +593,22 @@ rclient (char *server, char *service) } int -sm_winit (char *from) +sm_winit (char *from, int smtputf8) { - switch (smtalk (SM_MAIL, "MAIL FROM:<%s>", from)) { + const char *const mail_parameters = smtputf8 + ? " BODY=8BITMIME SMTPUTF8" + : ""; + + /* Just for information, if an attempt is made to send to an 8-bit + address without specifying SMTPUTF8, Gmail responds with + 555 5.5.2 Syntax error. + Gmail doesn't require the 8BITMIME, but RFC 6531 Sec. 1.2 does. */ + if (smtputf8 && (! EHLOset("8BITMIME") || ! EHLOset("SMTPUTF8"))) { + sm_end (NOTOK); + return RP_UCMD; + } + + switch (smtalk (SM_MAIL, "MAIL FROM:<%s>%s", from, mail_parameters)) { case 250: sm_addrs = 0; return RP_OK; diff --git a/mts/smtp/smtp.h b/mts/smtp/smtp.h index 2ed5cc22..3063866e 100644 --- a/mts/smtp/smtp.h +++ b/mts/smtp/smtp.h @@ -18,7 +18,7 @@ struct smtp { /* int client (); */ int sm_init (char *, char *, char *, int, int, int, int, int, char *, char *, const char *, int); -int sm_winit (char *); +int sm_winit (char *, int); int sm_wadr (char *, char *, char *); int sm_waend (void); int sm_wtxt (char *, int); diff --git a/sbr/addrsbr.c b/sbr/addrsbr.c index 555822e7..3b57f20c 100644 --- a/sbr/addrsbr.c +++ b/sbr/addrsbr.c @@ -82,12 +82,13 @@ char * getname (const char *addrs) { struct adrx *ap; - + const char *const eai = context_find ("eai"); pers = mbox = host = route = grp = note = NULL; err[0] = '\0'; - if ((ap = getadrx (addrs ? addrs : "")) == NULL) + if ((ap = getadrx (addrs ? addrs : "", eai && strncasecmp (eai, "8bit", 4) == 0)) == NULL) { return NULL; + } strncpy (adr, ap->text, sizeof(adr)); pers = ap->pers; diff --git a/sbr/mf.c b/sbr/mf.c index c11b0c10..136f94cd 100644 --- a/sbr/mf.c +++ b/sbr/mf.c @@ -174,8 +174,9 @@ static char adr[BUFSIZ]; static struct adrx adrxs2; +/* eai = Email Address Internationalization */ struct adrx * -getadrx (const char *addrs) +getadrx (const char *addrs, int eai) { register char *bp; register struct adrx *adrxp = &adrxs2; @@ -230,13 +231,15 @@ getadrx (const char *addrs) break; } - /* - * Reject the address if key fields contain 8bit characters - */ + if (! eai) { + /* + * Reject the address if key fields contain 8bit characters + */ - if (contains8bit(mbox, NULL) || contains8bit(host, NULL) || - contains8bit(path, NULL) || contains8bit(grp, NULL)) { - strcpy(err, "Address contains 8-bit characters"); + if (contains8bit(mbox, NULL) || contains8bit(host, NULL) || + contains8bit(path, NULL) || contains8bit(grp, NULL)) { + strcpy(err, "Address contains 8-bit characters"); + } } if (err[0]) diff --git a/test/fakesmtp.c b/test/fakesmtp.c index 42d4f181..449fdec0 100644 --- a/test/fakesmtp.c +++ b/test/fakesmtp.c @@ -101,7 +101,11 @@ main(int argc, char *argv[]) smtp_state = SMTP_DATA; continue; } - if (xoauth != NULL) { + if (xoauth == NULL) { + putcrlf(conn, "250-ready"); + putcrlf(conn, "250-8BITMIME"); + putcrlf(conn, "250-SMTPUTF8"); + } else { /* XOAUTH2 support enabled; handle EHLO and AUTH. */ if (strncmp(line, "EHLO", 4) == 0) { putcrlf(conn, "250-ready"); diff --git a/test/post/test-rfc6531 b/test/post/test-rfc6531 new file mode 100755 index 00000000..1df9a291 --- /dev/null +++ b/test/post/test-rfc6531 @@ -0,0 +1,73 @@ +#!/bin/sh +###################################################### +# +# Test SMTPUTF8 (RFC 6531) support. +# +###################################################### + +set -e + +if test -z "${MH_OBJ_DIR}"; then + srcdir=`dirname $0`/../.. + MH_OBJ_DIR=`cd $srcdir && pwd`; export MH_OBJ_DIR +fi + +. "${srcdir}/test/post/test-post-common.sh" + +setup_test +mhl="${MH_LIBEXEC_DIR}/mhl" + +#### Make sure that html-to-text conversion is what we expect. +require_locale en_US.utf-8 en_US.utf8 +LC_ALL=en_US.UTF-8; export LC_ALL + +#### Enable EAI in mhbuild, via the profile. +sed "s%^\(mhbuild: .*\)%\1 -headerencoding 8bit%" "$MH" >"$MH.new" +mv -f "$MH.new" "$MH" + +# check SMTPUTF8 basic +start_test 'SMTPUTF8 basic' + +cat > "${testname}.expected" < BODY=8BITMIME SMTPUTF8 +RCPT TO: +DATA +From: senderø@example.com +To: recipientæ@example.com +Subject: Blåbærsyltetøy +MIME-Version: 1.0 +Content-Type: text/plain; charset="UTF-8" +Content-Transfer-Encoding: 8bit +Date: + +Blåbærsyltetøy er veldig godt. Jeg liker blåbærsyltetøy. +Jøran. + +Text source: +Arnt Gulbrandsen and Jiankang Yao, "Email Address Internationalization", +Computing Edge 1:6, June 2015, +http://www.computer.org/cms/computing-edge/ce-jun15-final.pdf, p. 50. +. +QUIT +EOF + +cat > "${MH_TEST_DIR}/Mail/draft" < "${testname}.post-expected" < EHLO nosuchhost.example.com +<= 250-ready +<= 250-8BITMIME +<= 250-SMTPUTF8 <= 250 I'll buy that for a dollar! => MAIL FROM: +<= 250-ready +<= 250-8BITMIME +<= 250-SMTPUTF8 <= 250 I'll buy that for a dollar! => RCPT TO: +<= 250-ready +<= 250-8BITMIME +<= 250-SMTPUTF8 <= 250 I'll buy that for a dollar! => DATA <= 354 Go ahead diff --git a/uip/mhbuild.c b/uip/mhbuild.c index a0903c57..8684c70f 100644 --- a/uip/mhbuild.c +++ b/uip/mhbuild.c @@ -56,9 +56,11 @@ DEFINE_SWITCH_ENUM(MHBUILD); DEFINE_SWITCH_ARRAY(MHBUILD, switches); #undef X +/* 8bit is Email Address Internationalization. */ #define MIMEENCODING_SWITCHES \ X("base64", 0, BASE64SW) \ X("quoted-printable", 0, QUOTEDPRINTSW) \ + X("8bit", 0, EIGHTBITSW) \ #define X(sw, minchars, id) id, DEFINE_SWITCH_ENUM(MIMEENCODING); @@ -244,6 +246,9 @@ main (int argc, char **argv) case QUOTEDPRINTSW: header_encoding = CE_QUOTED; break; + case EIGHTBITSW: + header_encoding = CE_8BIT; + break; default: adios (NULL, "Internal error: algorithm %s", cp); } diff --git a/uip/mhbuildsbr.c b/uip/mhbuildsbr.c index b9989934..26d8524d 100644 --- a/uip/mhbuildsbr.c +++ b/uip/mhbuildsbr.c @@ -360,15 +360,17 @@ finish_field: } m_getfld_state_destroy (&gstate); - /* - * Iterate through the list of headers and call the function to MIME-ify - * them if required. - */ - - for (hp = ct->c_first_hf; hp != NULL; hp = hp->next) { - if (encode_rfc2047(hp->name, &hp->value, header_encoding, NULL)) { - adios(NULL, "Unable to encode header \"%s\"", hp->name); - } + if (header_encoding != CE_8BIT) { + /* + * Iterate through the list of headers and call the function to MIME-ify + * them if required. + */ + + for (hp = ct->c_first_hf; hp != NULL; hp = hp->next) { + if (encode_rfc2047(hp->name, &hp->value, header_encoding, NULL)) { + adios(NULL, "Unable to encode header \"%s\"", hp->name); + } + } } /* @@ -1748,8 +1750,11 @@ build_headers (CT ct, int header_encoding) if (ct->c_descr) { np = add (DESCR_FIELD, NULL); vp = concat (" ", ct->c_descr, NULL); - if (encode_rfc2047(DESCR_FIELD, &vp, header_encoding, NULL)) - adios(NULL, "Unable to encode %s header", DESCR_FIELD); + if (header_encoding != CE_8BIT) { + if (encode_rfc2047(DESCR_FIELD, &vp, header_encoding, NULL)) { + adios(NULL, "Unable to encode %s header", DESCR_FIELD); + } + } add_header (ct, np, vp); } diff --git a/uip/post.c b/uip/post.c index 0fee356b..6e2367a4 100644 --- a/uip/post.c +++ b/uip/post.c @@ -64,6 +64,8 @@ X("library directory", -7, LIBSW) /* interface from send, whom */ \ X("mime", 0, MIMESW) \ X("nomime", 0, NMIMESW) \ + X("eai", 0, EAISW) \ + X("noeai", 0, NEAISW) \ X("msgid", 0, MSGDSW) \ X("nomsgid", 0, NMSGDSW) \ X("verbose", 0, VERBSW) \ @@ -223,6 +225,7 @@ static int badmsg = 0; /* message has bad semantics */ static int verbose = 0; /* spell it out */ static int format = 1; /* format addresses */ static int mime = 0; /* use MIME-style encapsulations for Bcc */ +static int eai = 0; /* use EAI (SMTPUTF8) */ static int msgid = 0; /* add msgid */ static int debug = 0; /* debugging post */ static int watch = 0; /* watch the delivery process */ @@ -398,6 +401,13 @@ main (int argc, char **argv) mime = 0; continue; + case EAISW: + eai = 1; + continue; + case NEAISW: + eai = 0; + continue; + case MSGDSW: msgid++; continue; @@ -585,6 +595,12 @@ main (int argc, char **argv) if ((in = fopen (msg, "r")) == NULL) adios (msg, "unable to open"); + /* Support EAI. */ + if (eai) { + /* Add eai profile entry, to pass 8bit setting to getname()/getadrx(). */ + add_profile_entry ("eai", "8bit"); + } + start_headers (); if (debug) { verbose++; @@ -1634,12 +1650,13 @@ post (char *file, int bccque, int talk, char *envelope, int oauth_flag, break; } } else { - if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch, - verbose, snoop, sasl, saslssf, + if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch, + verbose, snoop, sasl, saslssf, saslmech, user, oauth_flag ? auth_svc : NULL, tls)) - || rp_isbad (retval = sm_winit (envelope))) + || rp_isbad (retval = sm_winit (envelope, eai))) { die (NULL, "problem initializing server; %s", rp_string (retval)); + } do_addresses (bccque, talk && verbose); if ((fd = open (file, O_RDONLY)) == NOTOK) @@ -1674,13 +1691,15 @@ verify_all_addresses (int talk, char *envelope, int oauth_flag, char *auth_svc) sigon (); - if (!whomsw || checksw) + if (!whomsw || checksw) { if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch, verbose, snoop, sasl, saslssf, saslmech, user, oauth_flag ? auth_svc : NULL, tls)) - || rp_isbad (retval = sm_winit (envelope))) + || rp_isbad (retval = sm_winit (envelope, eai))) { die (NULL, "problem initializing server; %s", rp_string (retval)); + } + } if (talk && !whomsw) printf (" -- Address Verification --\n"); diff --git a/uip/send.c b/uip/send.c index 5dcdce0e..d8a92cb3 100644 --- a/uip/send.c +++ b/uip/send.c @@ -42,6 +42,8 @@ X("noforward", 0, NFORWSW) \ X("mime", 0, MIMESW) \ X("nomime", 0, NMIMESW) \ + X("eai", 0, EAISW) \ + X("noeai", 0, NEAISW) \ X("msgid", 0, MSGDSW) \ X("nomsgid", 0, NMSGDSW) \ X("push", 0, PUSHSW) \ @@ -236,6 +238,13 @@ main (int argc, char **argv) vec[vecp++] = --cp; continue; + case EAISW: + vec[vecp++] = --cp; + continue; + case NEAISW: + vec[vecp++] = --cp; + continue; + case SNOOPSW: snoop++; vec[vecp++] = --cp; @@ -478,7 +487,7 @@ go_to_it: closefds (3); for (msgnum = 0; msgnum < msgp; msgnum++) { - switch (sendsbr (vec, vecp, program, msgs[msgnum], &st, 1, auth_svc)) { + switch (sendsbr (vec, vecp, program, msgs[msgnum], &st, 1, auth_svc)) { case DONE: done (++status); case NOTOK: diff --git a/uip/sendsbr.c b/uip/sendsbr.c index c4d25029..5da67fc9 100644 --- a/uip/sendsbr.c +++ b/uip/sendsbr.c @@ -76,23 +76,44 @@ sendsbr (char **vec, int vecp, char *program, char *draft, struct stat *st, by longjmp. */ volatile int nvecs = vecp; int *nvecsp = (int *) &nvecs; + int eai = 0; /* * Run the mimebuildproc (which is by default mhbuild) on the message * with the addition of the "-auto" flag */ + /* Make sure that EAI support is set up before calling buildmimeproc. */ + for (i = 0; i < vecp; ++i) { + if (strcasecmp (vec[i], "-eai") == 0) { + eai = 1; + } else if (strcasecmp (vec[i], "-noeai") == 0) { + eai = 0; + } + } + switch (child = fork()) { case NOTOK: adios("fork", "unable to"); break; case OK: - buildvec = argsplit(buildmimeproc, &buildprogram, &i); + buildvec = argsplit(buildmimeproc, &buildprogram, &i); buildvec[i++] = "-auto"; if (distfile) buildvec[i++] = "-dist"; buildvec[i++] = (char *) drft; + if (eai) { + /* Add eai profile entry, to pass 8bit setting to + getname()/getadrx(). This doesn't seem to be necessary + now, but it's here just in case the code changes + later. */ + add_profile_entry("eai", "8bit"); + + /* Add mhbuild switch to enable 8bit headers. */ + buildvec[i++] = "-headerencoding"; + buildvec[i++] = "8bit"; + } buildvec[i] = NULL; execvp(buildprogram, buildvec); fprintf(stderr, "unable to exec "); diff --git a/uip/whatnowsbr.c b/uip/whatnowsbr.c index 80386e95..c2f8bbf1 100644 --- a/uip/whatnowsbr.c +++ b/uip/whatnowsbr.c @@ -923,6 +923,8 @@ buildfile (char **argp, char *file) X("noforward", 0, NFORWSW) \ X("mime", 0, MIMESW) \ X("nomime", 0, NMIMESW) \ + X("eai", 0, EAISW) \ + X("noeai", 0, NEAISW) \ X("msgid", 0, MSGDSW) \ X("nomsgid", 0, NMSGDSW) \ X("push", 0, SPSHSW) \ @@ -1129,6 +1131,8 @@ sendit (char *sp, char **arg, char *file, int pushed) case NBITSTUFFSW: case MIMESW: case NMIMESW: + case EAISW: + case NEAISW: case MSGDSW: case NMSGDSW: case WATCSW: diff --git a/uip/whom.c b/uip/whom.c index b1e8b1ed..081978a5 100644 --- a/uip/whom.c +++ b/uip/whom.c @@ -42,6 +42,8 @@ X("tls", TLSminc(-3), TLSSW) \ X("initialtls", TLSminc(-10), INITTLSSW) \ X("notls", TLSminc(-5), NTLSSW) \ + X("eai", 0, EAISW) \ + X("noeai", 0, NEAISW) \ X("mts smtp|sendmail/smtp|sendmail/pipe", 0, MTSSW) \ #define X(sw, minchars, id) id, @@ -100,6 +102,8 @@ main (int argc, char **argv) case TLSSW: case INITTLSSW: case NTLSSW: + case EAISW: + case NEAISW: vec[vecp++] = --cp; continue;