]> diplodocus.org Git - minc/blob - minc
ebe447f74ff92fda0753bb2cdb3ff8899cc44e9d
[minc] / minc
1 #! /usr/local/bin/perl
2
3 # $Id$
4
5 =head1 NAME
6
7 B<minc> - Incorporate mail from a maildir into mh folders.
8
9 =head1 SYNOPSIS
10
11 B<minc> [-B<dhns>]
12
13 =head1 DESCRIPTION
14
15 B<minc> is a program for incorporating mail from a maildir to a mh
16 folder hierarchy. It takes mail from a maildir folder (not a maildir
17 folder hierarchy), checks for spam with razor
18 (http://razor.sourceforge.net/), and optionally filters mail into
19 separate mh folders.
20
21 The filtering is quite sophisticated, as it is done using real Perl
22 substitution (s//) commands.
23
24 =cut
25
26 use strict;
27 use warnings;
28
29 require 'sysexits.ph';
30
31 use Data::Dumper;
32 use File::Basename;
33 use Getopt::Std;
34 use Log::Dispatch;
35 use Log::Dispatch::File;
36 use POSIX qw(strftime WEXITSTATUS WIFEXITED);
37
38 =head1 OPTIONS
39
40 =over 4
41
42 =item B<-d>
43
44 Dump (using Data::Dumper) the FILTERS hash and exit. This is useful
45 for testing the syntax of .mincfilter.
46
47 =item B<-h>
48
49 Show help.
50
51 =item B<-n>
52
53 Dry run; do not actually incorporate the mail, but log and report to
54 stdout/stderr as normal.
55
56 =item B<-s>
57
58 Process SPAM only, then exit, leaving all non-spam messages in the
59 maildir.
60
61 =back
62
63 =cut
64
65 my $dumpfilters = 0;
66 my $run = 1;
67 my $spamonly = 0;
68
69 our $opt_d;
70 our $opt_h;
71 our $opt_n;
72 our $opt_s; # ;; # stupid cperl-mode
73
74 if (not getopts('dhns')) {
75 exit(&EX_USAGE);
76 }
77
78 if ($opt_d) {
79 $dumpfilters = 1;
80 } elsif ($opt_h) {
81 print("Sorry bub, no help.\n");
82 exit(&EX_OK);
83 } elsif ($opt_n) {
84 $run = 0;
85 }
86
87 if ($opt_s) { # ))){ # stupid cperl-mode
88 $spamonly = 1;
89 }
90
91 =head1 ENVIRONMENT
92
93 =over 4
94
95 =item HOME
96
97 Where configuration files (.mincfilter) are found. Also,
98 $HOME/Maildir is used for the maildir if MAILDIR is not set.
99
100 =item MAILDIR
101
102 Where mail is delivered.
103
104 =back
105
106 =cut
107
108 use Env qw(HOME MAILDIR);
109
110 if (not $HOME) {
111 die("HOME environment variable must be set.\n");
112 }
113 if (not $MAILDIR) {
114 $MAILDIR = "$HOME/Maildir";
115 }
116
117 =head1 FILES
118
119 =over 4
120
121 =item $HOME/.mincfilter
122
123 This file is Perl code (included via the 'require' directive) which is
124 expected to define the FILTERS hash.
125
126 =item `mhpath +`/logs/minc.log
127
128 Where minc logs what it does, unless in -n mode.
129
130 =item `mhpath +`/logs/dryrun.log
131
132 Where minc logs what it would do; used in -n mode.
133
134 =cut
135
136 our %FILTERS;
137 require "$HOME/.mincfilter";
138
139 my $mh;
140 my $logfile;
141
142 $mh = `mhpath +`;
143 chomp($mh);
144
145 if ($run) {
146 $logfile = $mh . '/logs/minc.log';
147 } else {
148 $logfile = $mh . '/logs/dryrun.log';
149 }
150
151 \f
152 ###############################################################################
153 # Logging
154
155 # debuglevels for the logger
156 use constant LOG_DEBUGLEVEL => 'info';
157 use constant SCREEN_DEBUGLEVEL => 'debug';
158
159 # characters used to wrap around the id field in the log so I can more
160 # easily parse the msg flow
161 use constant DEBUGCHAR => '%';
162 use constant INFOCHAR => '=';
163 use constant INCOMINGCHAR => '<';
164 use constant SAVECHAR => '>';
165 use constant WARNCHAR => 'W';
166 use constant ERRORCHAR => 'E';
167
168 my $logger = new Log::Dispatch;
169 $logger->add(new Log::Dispatch::File (name=>'logfile',
170 filename=>$logfile,
171 min_level=>'info',
172 mode=>'append'));
173
174 # log alias, handles getting passed vars that are undef
175 sub mylog {
176 my $level = shift;
177 my $act = shift;
178 my $timestamp = strftime('%b %e %H:%M:%S', localtime);
179 my $msg;
180 foreach my $part (@_) {
181 if (defined($part)) {
182 $msg .= $part;
183 }
184 }
185 # no newlines in the log message, thanks
186 $msg =~ s/\n/ /gm;
187 $msg = sprintf("%s %s%s %s\n", $timestamp, $act, $act, $msg);
188 $logger->log(level=>$level, message=>$msg);
189
190 if ($act eq SAVECHAR) {
191 $logger->log(level=>$level, message=>"\n");
192 }
193 }
194
195 sub logincoming { mylog('info', INCOMINGCHAR, @_); }
196 sub logsave { mylog('notice', SAVECHAR, @_); }
197 sub loginfo { mylog('info', INFOCHAR, @_); }
198 sub logdebug { mylog('debug', DEBUGCHAR, @_); }
199 sub logwarn { mylog('warning', WARNCHAR, @_); }
200 sub logerr { mylog('error', ERRORCHAR, @_); }
201
202 sub log_headers {
203 my %headers = @_;
204
205 logincoming('From: ', $headers{'return-path'});
206 logincoming('To: ', $headers{'to'});
207 logincoming('Subject: ', $headers{'subject'});
208 logincoming('Message-Id: ', $headers{'message-id'});
209 }
210
211 \f
212 ###############################################################################
213 # Utility procedures
214
215 sub _errprint {
216 printf('%s:%s', __FILE__, __LINE__);
217
218 if (@_) {
219 print(': ');
220 foreach (@_) {
221 print;
222 }
223 }
224 }
225
226 sub err {
227 my $ex = shift;
228
229 _errprint(@_);
230 print(": $!\n");
231
232 exit($ex);
233 }
234
235 sub errx {
236 my $ex = shift;
237
238 _errprint(@_);
239 print("\n");
240
241 exit($ex);
242 }
243
244 sub mkfolder {
245 my $mhfolder = shift;
246 my $folder;
247 my $target;
248 my $component;
249
250 $folder = $mh . '/' . $mhfolder;
251 $target = '';
252
253 if (not -d $folder) {
254 foreach $component (split('/', $folder)) {
255 if (defined($component) and length($component) >= 1) {
256 $target = $target . '/' . $component;
257 if (-d $target or mkdir($target)) {
258 next;
259 } else {
260 err(&EX_TEMPFAIL,
261 "Failed to create +$mhfolder ($component)");
262 }
263 }
264 }
265 }
266 }
267
268 sub getfiles {
269 my $dir = shift;
270 my @result;
271
272 if (not opendir(DIR, $dir)) {
273 err(&EX_TEMPFAIL, "Failed opendir($dir)");
274 }
275
276 # Initialize $! to 0 (success) because stupid stupid Perl provides
277 # no way to distinguish an error from an empty directory; that
278 # means setting $! to 0 and testing it afterwards is the only way
279 # to detect an error. Real Programmers don't handle errors,
280 # right? >sigh<
281 $! = 0;
282 @result = grep {
283 ($_ ne '.' and $_ ne '..')
284 and $_ = "$MAILDIR/new/$_"
285 } readdir(DIR);
286
287 if ($! != 0) {
288 err(&EX_TEMPFAIL, "Failed readdir($dir)");
289 }
290
291 closedir(DIR);
292
293 return @result;
294 }
295
296 sub get_highest_msgnum {
297 my $mhfolder = shift;
298 my $msgnum;
299 my $highest;
300
301 $highest = 0;
302
303 foreach $msgnum (getfiles($mhfolder)) {
304 # Look for integers.
305 if ($msgnum =~ /^[0-9]+$/) {
306 if ($msgnum > $highest) {
307 $highest = $msgnum;
308 }
309 }
310 }
311
312 return $highest + 1;
313 }
314
315 # We want to print the name of each list that has new mail only once,
316 # so use this hash to store the lists that have already been printed.
317 # Start the list out with SPAM already in it, since we don't care when
318 # new messages are added to it.
319 my %FOLDERS = ('SPAM'=>1);
320
321 sub store_message {
322 my $msg = shift;
323 my $mhfolder = shift;
324 my $msgnum;
325 my $try;
326 my $mhmsg;
327 my $status;
328
329 # We must do this even in -n mode because later steps fail without
330 # it. This should be harmless.
331 mkfolder($mhfolder);
332
333 $msgnum = get_highest_msgnum($mhfolder);
334 for ($try = 0; ; $try++) {
335 $msgnum++;
336 $mhmsg = "$mhfolder/$msgnum";
337
338 if (not stat($mhmsg)) {
339 if ($! == ENOENT) {
340 # Now we have a non-existent file, let's try to create
341 # it. We must create a zero-byte file first because a
342 # file my appear between our happy stat results and
343 # our later rename(2), which would clobber said file.
344 # So attempt to create a file with this name. If it
345 # succeeds, in just a bit here we'll knowingly clobber
346 # this file with the rename(2) call.
347 if (sysopen(MSG, $mhmsg, O_WRONLY | O_EXCL | O_CREAT, 0600)) {
348 close(MSG);
349 last;
350 }
351 }
352 }
353
354 # This algorithm is different from the maildir one; let's make
355 # 10 tries instead of 3.
356 if ($try == 9) {
357 errx(&EX_TEMPFAIL, "Attempted filename exists.");
358 }
359
360 # This algorithm is different; i don't think we need to sleep.
361 #sleep(2);
362 }
363
364 logsave("+$mhfolder");
365
366 if ($run) {
367 if (not rename($msg, $mhmsg)) {
368 err(&EX_TEMPFAIL, "Failed rename($msg, $mhmsg)");
369 }
370
371 # XXX: Lame! Instead, keep a hash of arrays. Keys are folder
372 # names, array elements are message numbers in that list.
373 # Then after all the messages have been sorted, run one mark
374 # command per folder, marking all messages for that folder in
375 # one sweep.
376 if ($mhfolder ne 'SPAM') {
377 $status = system("mark +$mhfolder $msgnum -sequence unseen -add");
378 if (not (WIFEXITED($status) and WEXITSTATUS($status) == 0)) {
379 errx(&EX_TEMPFAIL, "Failed to mark message unseen");
380 }
381 }
382 }
383
384 if (not $FOLDERS{$mhfolder}) {
385 print("+$mhfolder\n");
386 $FOLDERS{$mhfolder} = 1;
387 }
388 }
389
390 sub get_headers {
391 my $msg = shift;
392 my %headers;
393 my $current; # current header, used for unfolding lines
394 my $fieldname; # unmolested header name
395
396 open(MSG, $msg);
397 while (<MSG>) {
398 chomp;
399 if (length == 0) {
400 last;
401 }
402
403 if (/^\s/) {
404 # folded header continuation
405
406 if (not defined($current)) {
407 print("Malformed message, cannot parse headers.\n");
408 return ();
409 }
410
411 $headers{$current} .= $_;
412 } else {
413 ($fieldname) = split(/:/);
414 $current = lc($fieldname);
415 (undef, $headers{$current}) = split(/^$fieldname:\s*/);
416 }
417 }
418 close(MSG);
419
420 return %headers;
421 }
422
423 \f
424 ###############################################################################
425 # Spam handling
426
427 sub is_spam {
428 my $msg = shift;
429 my $err;
430 my $sig;
431 my $status;
432
433 $err = $sig = $msg;
434 $err =~ s|/new/|/err/|;
435 $sig =~ s|/new/|/sig/|;
436 if (-f $err) {
437 my $line;
438 my $message;
439
440 open(ERR, $err);
441 $line = <ERR>;
442 close(ERR);
443
444 chomp($line);
445 ($message, $!) = split(/:/, $line);
446
447 logerr("$err: $message: $!");
448
449 if ($run) {
450 if (unlink($err) != 1) {
451 err(&EX_TEMPFAIL, "Failed unlink($err)");
452 }
453 }
454 } elsif (-f $sig) {
455 # This is supposed to be a signature created with razor-check
456 # directly after delivery. Currently this isn't supported
457 # because it isn't clear to me how to get that signature back
458 # into razor-check. For now, just unlink any sig files we
459 # find and proceed with full razor-check mode.
460
461 if ($run) {
462 if (unlink($sig) != 1) {
463 err(&EX_TEMPFAIL, "Failed unlink($sig)");
464 }
465 }
466 }
467
468 if ($run) {
469 $status = system("razor-check < $msg");
470 if (not WIFEXITED($status)) {
471 err(&EX_TEMPFAIL, "Failed to run razor-check < $msg");
472 } elsif (WEXITSTATUS($status) == 0) {
473 return 1;
474 } else {
475 return 0;
476 }
477 } else {
478 return 0;
479 }
480 }
481
482 sub kill_spam {
483 my @msglist = @_;
484 my @result;
485 my $msg;
486 my $i;
487
488 @result = ();
489
490 $i = 0;
491 foreach $msg (@msglist) {
492 printf('%sChecking for spam... %6d/%d',
493 "\r", ++$i, scalar(@msglist));
494 if (is_spam($msg)) {
495 print(" SPAM\n");
496 store_message($msg, 'SPAM');
497 } else {
498 push(@result, $msg);
499 }
500 }
501 print("\nDone: ", scalar(@result), " survivors\n");
502
503 return @result;
504 }
505
506 \f
507 ###############################################################################
508 # Filtering
509
510 sub find_mh_folder {
511 my $msg = shift;
512 my %headers;
513 my $header;
514 my $contents;
515 my $regex;
516 my $subst;
517 my $try;
518
519 %headers = get_headers($msg);
520 if (not %headers) {
521 return 'inbox';
522 }
523
524 log_headers(%headers);
525
526 foreach $header (keys(%FILTERS)) {
527 $contents = $headers{lc($header)};
528
529 if (defined($contents)) {
530 foreach $regex (keys(%{$FILTERS{$header}})) {
531 $subst = $FILTERS{$header}->{$regex};
532
533 $try = '$contents =~';
534 $try .= " s|$regex|$subst|is";
535 if (eval $try) {
536 return $contents;
537 }
538 }
539 }
540 }
541
542 return 'inbox';
543 }
544
545 sub filter_mail {
546 my @msglist = @_;
547 my $msg;
548 my $mhfolder;
549
550 foreach $msg (@msglist) {
551 $mhfolder = find_mh_folder($msg);
552 store_message($msg, $mhfolder);
553 }
554 }
555
556 \f
557 MAIN: {
558 my @msglist;
559
560 if ($dumpfilters) {
561 $Data::Dumper::Indent = 1;
562 print(Dumper(\%FILTERS));
563 exit(&EX_OK);
564 }
565
566 @msglist = kill_spam(getfiles("$MAILDIR/new"));
567
568 if (not $spamonly) {
569 filter_mail(@msglist);
570 }
571 }
572
573 __END__
574
575 =head1 EXAMPLES
576
577 %FILTERS =
578 ('List-Id:'=>
579 {'.*<([[:graph:]]+)-([[:graph:]]+)\.lists\.sourceforge\.net>.*'=>'l/$1/$2',
580 '.*<([[:graph:]]+?)(-list)?\.freedesktop\.org>.*'=>'l/freedesktop/$1',
581 '.*<swig\.cs\.uchicago\.edu>.*'=>'l/swig'},
582
583 'List-Post:'=>
584 {'<mailto:([[:graph:]]+)+@([[:graph:]]+)\.apache\.org>'=>'l/apache/$2/$1'},
585
586 'To:'=>
587 {'root'=>'root'});
588
589 The first List-Id filter is a surprisingly effective one which creates
590 appropriate folder names (such as l/htdig/updates and l/rox/devel) for
591 all Sourceforge lists to which i subscribe. Certainly there are lists
592 hosted at Sourceforge which do not work well with this, and this is
593 where it is important to remember that the first match is the one
594 uses. Simply put the more specific rules before this one.
595
596 The next List-Id example is simple. The swig example demonstrates
597 that the folder name does not have to use a portion of the matched
598 header; you can just hardcode the folder name.
599
600 The List-Post example is a nice one. Most ASF projects have their own
601 apache.org subdomain, with mailing lists hosted there. So, given a
602 list such as dev@httpd.apache.org, this filter will create the folder
603 name l/apache/httpd/dev.
604
605 =head1 AUTHORS
606
607 Written by Eric Gillespie <epg@pretzelnet.org> with logging code
608 stolen from Adam Lazur <adam@lazur.org>.
609
610 Design by Eric Gillespie and Doug Porter <dsp@waterspout.com>.
611
612 =cut