From: David Levine Date: Sun, 25 Sep 2016 13:28:05 +0000 (-0400) Subject: Merge remote-tracking branch 'origin' into smtputf8 X-Git-Url: https://diplodocus.org/git/nmh/commitdiff_plain/aaf014c77a4fb19bdc33370f5b6af5b8497decf8?hp=38e17e1b8ba306ed6137b174f2f196d09d02816f Merge remote-tracking branch 'origin' into smtputf8 --- diff --git a/Makefile.am b/Makefile.am index 4197dc75..cb1c39fb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -94,7 +94,8 @@ TESTS = test/ali/test-ali test/anno/test-anno \ test/post/test-post-dcc test/post/test-post-fcc \ test/post/test-post-multifrom test/post/test-post-envelope \ test/post/test-post-group test/post/test-mts test/post/test-messageid \ - test/post/test-sendfiles test/prompter/test-prompter \ + test/post/test-rfc6531 test/post/test-sendfiles \ + test/prompter/test-prompter \ test/rcv/test-rcvdist test/rcv/test-rcvpack test/rcv/test-rcvstore \ test/rcv/test-rcvtty test/refile/test-refile test/repl/test-convert \ test/repl/test-if-str test/repl/test-trailing-newline \ diff --git a/docs/pending-release-notes b/docs/pending-release-notes index 4dfc9ec5..f44eb066 100644 --- a/docs/pending-release-notes +++ b/docs/pending-release-notes @@ -49,6 +49,9 @@ NEW FEATURES - folder(1) -nocreate now prints a warning message for a non-existant folder. - mhfixmsg(1) now allows -decodetext binary, though 8bit is still the default. - inc(1) and msgchk(1) now support TLS encryption natively. +- Support for SMTPUTF8 (RFC 6531) has been added. mhshow(1) already supported + RFC 6532, assuming all 8-bit message header field bodies are UTF-8 and use + of a UTF-8 locale. ----------------- OBSOLETE FEATURES diff --git a/h/addrsbr.h b/h/addrsbr.h index de5dc3b4..1b147f68 100644 --- a/h/addrsbr.h +++ b/h/addrsbr.h @@ -40,6 +40,12 @@ struct mailname { void mnfree(struct mailname *); int ismymbox(struct mailname *); +/* + * Enable Email Address Internationalization, which requires + * support of 8-bit addresses. + */ +void enable_eai(); + /* * Parse an address header, and return a sequence of email addresses. * This function is the main entry point into the nmh address parser. diff --git a/h/mf.h b/h/mf.h index 585a83a4..1272640b 100644 --- a/h/mf.h +++ b/h/mf.h @@ -74,5 +74,5 @@ int isfrom(const char *); int lequal (const char *, const char *); int mfgets (FILE *, char **); char *legal_person (const char *); -struct adrx *getadrx (const char *); +struct adrx *getadrx (const char *, int); diff --git a/man/mhbuild.man b/man/mhbuild.man index 3d2fbbaf..18e82abf 100644 --- a/man/mhbuild.man +++ b/man/mhbuild.man @@ -1,4 +1,4 @@ -.TH MHBUILD %manext1% "December 14, 2014" "%nmhversion%" +.TH MHBUILD %manext1% "September 23, 2016" "%nmhversion%" .\" .\" %nmhwarning% .\" @@ -765,14 +765,19 @@ switch will indicate which algorithm to use when encoding any message headers that contain 8\-bit characters. The valid arguments are .I base64 -for based\-64 encoding and +for base\-64 encoding, .I quoted -for quoted\-printable encoding. +for quoted\-printable encoding, and +.I utf\-8 +which requires that all 8\-bit header field bodies be encoded as UTF\-8 +(RFC 6530) and that the message be sent to a SMTP server that supports +SMTPUTF8 (RFC 6531). The .B \-autoheaderencoding -switch will instruct +switch instructs .B mhbuild -to automatically pick the algorithm that results in a shorter encoded string. +to automatically pick the encoding, either base64 or quoted\-printable, +that results in a shorter encoded string. .PP Putting this all together, here is an example of a more complicated message draft. @@ -1038,25 +1043,31 @@ Template for composing contents. .IR mhstore (1) .PP .I "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies" -(RFC 2045), +(RFC 2045) .PP .I "Multipurpose Internet Mail Extensions (MIME) Part Two: Media Types" -(RFC 2046), +(RFC 2046) .PP .I "Multipurpose Internet Mail Extensions (MIME) Part Three: Message Header Extensions for Non-ASCII Text" -(RFC 2047), +(RFC 2047) .PP .I "MIME Parameter Value and Encoded Word Extensions: Character Sets, Languages, and Continuations" (RFC 2231) .PP .I "Proposed Standard for Message Encapsulation" -(RFC 934), +(RFC 934) .PP .I "The Content-MD5 Header Field" -(RFC 1864), +(RFC 1864) .PP .I "Definition of the URL MIME External-Body Access-Type" -(RFC 2017), +(RFC 2017) +.PP +.I "Overview and Framework for Internationalized Email" +(RFC 6530) +.PP +.I "SMTP Extension for Internationalized Email" +(RFC 6531) .SH DEFAULTS .nf \-autoheaderencoding diff --git a/mts/smtp/smtp.c b/mts/smtp/smtp.c index 4fb6b1f4..b5e68fa1 100644 --- a/mts/smtp/smtp.c +++ b/mts/smtp/smtp.c @@ -441,9 +441,22 @@ rclient (char *server, char *service) } int -sm_winit (char *from) +sm_winit (char *from, int smtputf8) { - switch (smtalk (SM_MAIL, "MAIL FROM:<%s>", from)) { + const char *const mail_parameters = smtputf8 + ? " BODY=8BITMIME SMTPUTF8" + : ""; + + /* Just for information, if an attempt is made to send to an 8-bit + address without specifying SMTPUTF8, Gmail responds with + 555 5.5.2 Syntax error. + Gmail doesn't require the 8BITMIME, but RFC 6531 Sec. 1.2 does. */ + if (smtputf8 && (! EHLOset("8BITMIME") || ! EHLOset("SMTPUTF8"))) { + sm_end (NOTOK); + return RP_UCMD; + } + + switch (smtalk (SM_MAIL, "MAIL FROM:<%s>%s", from, mail_parameters)) { case 250: sm_addrs = 0; return RP_OK; diff --git a/mts/smtp/smtp.h b/mts/smtp/smtp.h index 45e8c4c3..fb842fe6 100644 --- a/mts/smtp/smtp.h +++ b/mts/smtp/smtp.h @@ -17,8 +17,8 @@ struct smtp { */ /* int client (); */ int sm_init (char *, char *, char *, int, int, int, int, const char *, - const char *, const char *, int); -int sm_winit (char *); + const char *, const char *, int); +int sm_winit (char *, int); int sm_wadr (char *, char *, char *); int sm_waend (void); int sm_wtxt (char *, int); diff --git a/sbr/addrsbr.c b/sbr/addrsbr.c index 555822e7..6bb94211 100644 --- a/sbr/addrsbr.c +++ b/sbr/addrsbr.c @@ -76,18 +76,24 @@ static char *grp = NULL; static char *note = NULL; static char err[BUFSIZ]; static char adr[BUFSIZ]; +static int eai = 0; + +void +enable_eai() { + eai = 1; +} char * getname (const char *addrs) { struct adrx *ap; - pers = mbox = host = route = grp = note = NULL; err[0] = '\0'; - if ((ap = getadrx (addrs ? addrs : "")) == NULL) + if ((ap = getadrx (addrs ? addrs : "", eai)) == NULL) { return NULL; + } strncpy (adr, ap->text, sizeof(adr)); pers = ap->pers; diff --git a/sbr/mf.c b/sbr/mf.c index c11b0c10..136f94cd 100644 --- a/sbr/mf.c +++ b/sbr/mf.c @@ -174,8 +174,9 @@ static char adr[BUFSIZ]; static struct adrx adrxs2; +/* eai = Email Address Internationalization */ struct adrx * -getadrx (const char *addrs) +getadrx (const char *addrs, int eai) { register char *bp; register struct adrx *adrxp = &adrxs2; @@ -230,13 +231,15 @@ getadrx (const char *addrs) break; } - /* - * Reject the address if key fields contain 8bit characters - */ + if (! eai) { + /* + * Reject the address if key fields contain 8bit characters + */ - if (contains8bit(mbox, NULL) || contains8bit(host, NULL) || - contains8bit(path, NULL) || contains8bit(grp, NULL)) { - strcpy(err, "Address contains 8-bit characters"); + if (contains8bit(mbox, NULL) || contains8bit(host, NULL) || + contains8bit(path, NULL) || contains8bit(grp, NULL)) { + strcpy(err, "Address contains 8-bit characters"); + } } if (err[0]) diff --git a/test/fakesmtp.c b/test/fakesmtp.c index 42d4f181..449fdec0 100644 --- a/test/fakesmtp.c +++ b/test/fakesmtp.c @@ -101,7 +101,11 @@ main(int argc, char *argv[]) smtp_state = SMTP_DATA; continue; } - if (xoauth != NULL) { + if (xoauth == NULL) { + putcrlf(conn, "250-ready"); + putcrlf(conn, "250-8BITMIME"); + putcrlf(conn, "250-SMTPUTF8"); + } else { /* XOAUTH2 support enabled; handle EHLO and AUTH. */ if (strncmp(line, "EHLO", 4) == 0) { putcrlf(conn, "250-ready"); diff --git a/test/mhl/test-rfc6532 b/test/mhl/test-rfc6532 index db3754d9..96ee873d 100755 --- a/test/mhl/test-rfc6532 +++ b/test/mhl/test-rfc6532 @@ -57,7 +57,7 @@ MIME-Version: 1.0' # check with incompatible locale # This is here as a place holder. We should either try to run the (8-bit) text # through iconv, or refuse to display it if the locale is incompatible. In the -# cAse of EAI, that means not UTF-8. +# case of EAI, that means not UTF-8. # start_test 'incompatible locale' # LC_ALL=C # run_test "$mhl -form mhl.headers `mhpath last`" \ diff --git a/test/post/test-rfc6531 b/test/post/test-rfc6531 new file mode 100755 index 00000000..b86936bc --- /dev/null +++ b/test/post/test-rfc6531 @@ -0,0 +1,73 @@ +#!/bin/sh +###################################################### +# +# Test SMTPUTF8 (RFC 6531) support. +# +###################################################### + +set -e + +if test -z "${MH_OBJ_DIR}"; then + srcdir=`dirname $0`/../.. + MH_OBJ_DIR=`cd $srcdir && pwd`; export MH_OBJ_DIR +fi + +. "${srcdir}/test/post/test-post-common.sh" + +setup_test +mhl="${MH_LIBEXEC_DIR}/mhl" + +#### Make sure that html-to-text conversion is what we expect. +require_locale en_US.utf-8 en_US.utf8 +LC_ALL=en_US.UTF-8; export LC_ALL + +#### Enable EAI in mhbuild, via the profile. +sed "s%^\(mhbuild: .*\)%\1 -headerencoding utf-8%" "$MH" >"$MH.new" +mv -f "$MH.new" "$MH" + +# check SMTPUTF8 basic +start_test 'SMTPUTF8 basic' + +cat > "${testname}.expected" < BODY=8BITMIME SMTPUTF8 +RCPT TO: +DATA +From: senderø@example.com +To: recipientæ@example.com +Subject: Blåbærsyltetøy +MIME-Version: 1.0 +Content-Type: text/plain; charset="UTF-8" +Content-Transfer-Encoding: 8bit +Date: + +Blåbærsyltetøy er veldig godt. Jeg liker blåbærsyltetøy. +Jøran. + +Text source: +Arnt Gulbrandsen and Jiankang Yao, "Email Address Internationalization", +Computing Edge 1:6, June 2015, +http://www.computer.org/cms/computing-edge/ce-jun15-final.pdf, p. 50. +. +QUIT +EOF + +cat > "${MH_TEST_DIR}/Mail/draft" < EHLO nosuchhost.example.com +<= 250-ready +<= 250-8BITMIME +<= 250-SMTPUTF8 <= 250 I'll buy that for a dollar! => MAIL FROM: +<= 250-ready +<= 250-8BITMIME +<= 250-SMTPUTF8 <= 250 I'll buy that for a dollar! => RCPT TO: +<= 250-ready +<= 250-8BITMIME +<= 250-SMTPUTF8 <= 250 I'll buy that for a dollar! => DATA <= 354 Go ahead diff --git a/uip/mhbuild.c b/uip/mhbuild.c index a0903c57..87252a7c 100644 --- a/uip/mhbuild.c +++ b/uip/mhbuild.c @@ -56,9 +56,11 @@ DEFINE_SWITCH_ENUM(MHBUILD); DEFINE_SWITCH_ARRAY(MHBUILD, switches); #undef X +/* utf-8 is for Email Address Internationalization, using SMTPUTF8. */ #define MIMEENCODING_SWITCHES \ X("base64", 0, BASE64SW) \ X("quoted-printable", 0, QUOTEDPRINTSW) \ + X("utf-8", 0, UTF8SW) \ #define X(sw, minchars, id) id, DEFINE_SWITCH_ENUM(MIMEENCODING); @@ -244,6 +246,9 @@ main (int argc, char **argv) case QUOTEDPRINTSW: header_encoding = CE_QUOTED; break; + case UTF8SW: + header_encoding = CE_8BIT; + break; default: adios (NULL, "Internal error: algorithm %s", cp); } diff --git a/uip/mhbuildsbr.c b/uip/mhbuildsbr.c index b9989934..26d8524d 100644 --- a/uip/mhbuildsbr.c +++ b/uip/mhbuildsbr.c @@ -360,15 +360,17 @@ finish_field: } m_getfld_state_destroy (&gstate); - /* - * Iterate through the list of headers and call the function to MIME-ify - * them if required. - */ - - for (hp = ct->c_first_hf; hp != NULL; hp = hp->next) { - if (encode_rfc2047(hp->name, &hp->value, header_encoding, NULL)) { - adios(NULL, "Unable to encode header \"%s\"", hp->name); - } + if (header_encoding != CE_8BIT) { + /* + * Iterate through the list of headers and call the function to MIME-ify + * them if required. + */ + + for (hp = ct->c_first_hf; hp != NULL; hp = hp->next) { + if (encode_rfc2047(hp->name, &hp->value, header_encoding, NULL)) { + adios(NULL, "Unable to encode header \"%s\"", hp->name); + } + } } /* @@ -1748,8 +1750,11 @@ build_headers (CT ct, int header_encoding) if (ct->c_descr) { np = add (DESCR_FIELD, NULL); vp = concat (" ", ct->c_descr, NULL); - if (encode_rfc2047(DESCR_FIELD, &vp, header_encoding, NULL)) - adios(NULL, "Unable to encode %s header", DESCR_FIELD); + if (header_encoding != CE_8BIT) { + if (encode_rfc2047(DESCR_FIELD, &vp, header_encoding, NULL)) { + adios(NULL, "Unable to encode %s header", DESCR_FIELD); + } + } add_header (ct, np, vp); } diff --git a/uip/post.c b/uip/post.c index 0a42ad2f..db9c0298 100644 --- a/uip/post.c +++ b/uip/post.c @@ -276,7 +276,7 @@ static char *partno = NULL; /* * static prototypes */ -static void putfmt (char *, char *, FILE *); +static void putfmt (char *, char *, int *, FILE *); static void start_headers (void); static void finish_headers (FILE *); static int get_header (char *, struct headers *); @@ -288,14 +288,14 @@ static void anno (void); static int annoaux (struct mailname *); static void insert_fcc (struct headers *, char *); static void make_bcc_file (int); -static void verify_all_addresses (int, char *, int, char *); +static void verify_all_addresses (int, int, char *, int, char *); static void chkadr (void); static void sigon (void); static void sigoff (void); static void p_refile (char *); static void fcc (char *, char *); static void die (char *, char *, ...); -static void post (char *, int, int, char *, int, char *); +static void post (char *, int, int, int, char *, int, char *); static void do_text (char *file, int fd); static void do_an_address (struct mailname *, int); static void do_addresses (int, int); @@ -306,6 +306,7 @@ int main (int argc, char **argv) { int state, compnum, dashstuff = 0, swnum, oauth_flag = 0; + int eai = 0; /* use Email Address Internationalization (EAI) (SMTPUTF8) */ char *cp, *msg = NULL, **argp, **arguments, *envelope; char buf[BUFSIZ], name[NAMESZ], *auth_svc = NULL; FILE *in, *out; @@ -602,14 +603,14 @@ main (int argc, char **argv) switch (state = m_getfld (&gstate, name, buf, &bufsz, in)) { case FLD: case FLDPLUS: - compnum++; + compnum++; cp = add (buf, NULL); while (state == FLDPLUS) { bufsz = sizeof buf; state = m_getfld (&gstate, name, buf, &bufsz, in); cp = add (buf, cp); } - putfmt (name, cp, out); + putfmt (name, cp, &eai, out); free (cp); continue; @@ -711,7 +712,7 @@ main (int argc, char **argv) /* If we are doing a "whom" check */ if (whomsw) { /* This won't work with MTS_SENDMAIL_PIPE. */ - verify_all_addresses (1, envelope, oauth_flag, auth_svc); + verify_all_addresses (1, eai, envelope, oauth_flag, auth_svc); done (0); } @@ -723,14 +724,15 @@ main (int argc, char **argv) verify_all_addresses with MTS_SENDMAIL_PIPE, but that might require running sendmail as root. Note that spost didn't verify addresses. */ - verify_all_addresses (verbose, envelope, oauth_flag, auth_svc); + verify_all_addresses (verbose, eai, envelope, oauth_flag, + auth_svc); } - post (tmpfil, 0, verbose, envelope, oauth_flag, auth_svc); + post (tmpfil, 0, verbose, eai, envelope, oauth_flag, auth_svc); } - post (bccfil, 1, verbose, envelope, oauth_flag, auth_svc); + post (bccfil, 1, verbose, eai, envelope, oauth_flag, auth_svc); (void) m_unlink (bccfil); } else { - post (tmpfil, 0, isatty (1), envelope, oauth_flag, auth_svc); + post (tmpfil, 0, isatty (1), eai, envelope, oauth_flag, auth_svc); } p_refile (tmpfil); @@ -753,7 +755,7 @@ main (int argc, char **argv) */ static void -putfmt (char *name, char *str, FILE *out) +putfmt (char *name, char *str, int *eai, FILE *out) { int count, grp, i, keep; char *cp, *pp, *qp; @@ -770,8 +772,24 @@ putfmt (char *name, char *str, FILE *out) return; } + if (! *eai) { + /* Check each header field value to see if it has any 8-bit characters. + If it does, enable EAI support. */ + if (contains8bit(str, NULL)) { + if (verbose) { + printf ("EAI/SMTPUTF8 enabled\n"); + } + + /* Enable SMTPUTF8. */ + *eai = 1; + + /* Enable passing of utf-8 setting to getname()/getadrx(). */ + enable_eai(); + } + } + if ((i = get_header (name, hdrtab)) == NOTOK) { - if (strncasecmp (name, "nmh-", 4)) { + if (strncasecmp (name, "nmh-", 4)) { fprintf (out, "%s: %s", name, str); } else { /* Filter out all Nmh-* headers, because Norm asked. They @@ -819,7 +837,8 @@ putfmt (char *name, char *str, FILE *out) } tmpaddrs.m_next = NULL; - for (count = 0; (cp = getname (str)); count++) + + for (count = 0; (cp = getname (str)); count++) { if ((mp = getm (cp, NULL, 0, error, sizeof(error)))) { if (tmpaddrs.m_next) np->m_next = mp; @@ -834,6 +853,7 @@ putfmt (char *name, char *str, FILE *out) else badmsg++; } + } if (count < 1) { if (hdr->flags & HNIL) @@ -1577,8 +1597,8 @@ do_addresses (int bccque, int talk) */ static void -post (char *file, int bccque, int talk, char *envelope, int oauth_flag, - char *auth_svc) +post (char *file, int bccque, int talk, int eai, char *envelope, + int oauth_flag, char *auth_svc) { int fd; int retval, i; @@ -1626,11 +1646,12 @@ post (char *file, int bccque, int talk, char *envelope, int oauth_flag, break; } } else { - if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch, - verbose, snoop, sasl, saslmech, user, + if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch, + verbose, snoop, sasl, saslmech, user, oauth_flag ? auth_svc : NULL, tls)) - || rp_isbad (retval = sm_winit (envelope))) + || rp_isbad (retval = sm_winit (envelope, eai))) { die (NULL, "problem initializing server; %s", rp_string (retval)); + } do_addresses (bccque, talk && verbose); if ((fd = open (file, O_RDONLY)) == NOTOK) @@ -1658,19 +1679,22 @@ post (char *file, int bccque, int talk, char *envelope, int oauth_flag, /* Address Verification */ static void -verify_all_addresses (int talk, char *envelope, int oauth_flag, char *auth_svc) +verify_all_addresses (int talk, int eai, char *envelope, int oauth_flag, + char *auth_svc) { int retval; struct mailname *lp; sigon (); - if (!whomsw || checksw) + if (!whomsw || checksw) { if (rp_isbad (retval = sm_init (clientsw, serversw, port, watch, verbose, snoop, sasl, saslmech, user, oauth_flag ? auth_svc : NULL, tls)) - || rp_isbad (retval = sm_winit (envelope))) + || rp_isbad (retval = sm_winit (envelope, eai))) { die (NULL, "problem initializing server; %s", rp_string (retval)); + } + } if (talk && !whomsw) printf (" -- Address Verification --\n"); diff --git a/uip/send.c b/uip/send.c index 4a6e0399..1e48ea3a 100644 --- a/uip/send.c +++ b/uip/send.c @@ -476,7 +476,7 @@ go_to_it: closefds (3); for (msgnum = 0; msgnum < msgp; msgnum++) { - switch (sendsbr (vec, vecp, program, msgs[msgnum], &st, 1, auth_svc)) { + switch (sendsbr (vec, vecp, program, msgs[msgnum], &st, 1, auth_svc)) { case DONE: done (++status); case NOTOK: diff --git a/uip/sendsbr.c b/uip/sendsbr.c index c4d25029..74f7dae5 100644 --- a/uip/sendsbr.c +++ b/uip/sendsbr.c @@ -88,7 +88,7 @@ sendsbr (char **vec, int vecp, char *program, char *draft, struct stat *st, break; case OK: - buildvec = argsplit(buildmimeproc, &buildprogram, &i); + buildvec = argsplit(buildmimeproc, &buildprogram, &i); buildvec[i++] = "-auto"; if (distfile) buildvec[i++] = "-dist";