UNTESTED: Refactor the job parallelization stuff into a generic Jobs package.

[flac-archive] / fa-flacd

diff --git a/fa-flacd b/fa-flacd
index ec264c317e47df68d16c1823a94fe41ed2fe3334..b10e0c9c210a2f296f3f2ad03abe2f3ade417587 100755 (executable)
--- a/fa-flacd
+++ b/fa-flacd
@@ -1,33 +1,90 @@
 #! /usr/bin/env perl
 
 # $Id$
 #! /usr/bin/env perl
 
 # $Id$

+# $URL$

 
 =head1 NAME
 
 
 =head1 NAME
 

-B<fa-flacd>, B<fa-rip>, B<fa-tags> - archive CDs to single FLAC files
+B<fa-flacd> - archive CDs to single FLAC files

 
 =head1 SYNOPSIS
 
 
 =head1 SYNOPSIS
 

-B<fa-flacd> [B<-jv>]
+B<fa-flacd> [B<-j> I<jobs>] [B<-v>]

 
 

-B<fa-rip>
+=cut

 
 

-B<fa-tags> ID TRACKCOUNT OFFSET [OFFSET ...] LENGTH
+package Jobs;

 
 

-=cut
+use strict;
+use warnings;
+
+use POSIX ':sys_wait_h';
+
+our @jobs;
+our @finished;
+
+sub reaper {
+    while ((my $pid = waitpid(-1, WNOHANG)) > 0) {
+        push(@finished, [$pid, $?]);
+    }
+
+    # XXX if $pid == -1 handle errors?
+
+    $SIG{CHLD} = \&reaper;
+}
+
+sub newjob {
+    my $f = shift;
+    my %o = @_;
+    my $pid;
+
+    $SIG{CHLD} = \&reaper;
+    if (not $o{'debug'}) {
+        $pid = fork();
+        if (not defined($pid)) {
+            die("fork: $!");
+        }
+    }
+
+    if ($o{'debug'} or $pid == 0) {
+        $SIG{CHLD} = 'DEFAULT';
+        exit($f->());
+    }
+
+    push(@jobs, $pid);
+
+    return $pid;
+}
+
+sub deljob {
+    my $i = shift;
+    my $j;
+
+    my ($pid, $status) = @{$finished[$i]};
+
+    for ($j = 0; $j <= $#jobs; $j++) {
+        $pid == $jobs[$j] and splice(@jobs, $j, 1) and last;
+    }
+
+    splice(@finished, $i, 1);
+
+    return ($pid, $status);
+}
+
+\f
+################################################################################
+package main;

 
 use strict;
 use warnings;
 
 use File::Basename;
 
 use strict;
 use warnings;
 
 use File::Basename;

-use Getopt::Std; $Getopt::Std::STANDARD_HELP_VERSION = 1;
+use Getopt::Long qw(:config gnu_getopt no_ignore_case);

 use POSIX ':sys_wait_h';
 use POSIX ':sys_wait_h';

+use Pod::Usage;

 
 

-our $VERSION = 1;
-
+my $debug;

 my $verbose;
 my $verbose;

-my @jobs;
-my @finished;

 
 sub verbose {
     $verbose and print(STDERR $_) for @_;
 
 sub verbose {
     $verbose and print(STDERR $_) for @_;


@@ -41,6 +98,7 @@ sub get_tags {
     my $value;
     my $artist;
     my $album;
     my $value;
     my $artist;
     my $album;

+    my $discnum;

     my @tags;
 
     verbose("Opening tags file $fn\n");
     my @tags;
 
     verbose("Opening tags file $fn\n");


@@ -51,17 +109,65 @@ sub get_tags {
 
         ($tag, $value) = split(/=/, $_, 2);
 
 
         ($tag, $value) = split(/=/, $_, 2);
 

-        if (/^ARTIST=/) {
+        if (/^ARTIST=/i) {

             $artist = $value;
             verbose("ARTIST $artist from $fn\n");
             $artist = $value;
             verbose("ARTIST $artist from $fn\n");

-        } elsif (/^ALBUM=/) {
+        } elsif (/^ALBUM=/i) {

             $album = $value;
             $album = $value;

-            verbose("ALBUM $album from $fn\n");
+            verbose("ALBUM $album from $fn\n"); # cperl-mode sucks "
+        } elsif (/^DISCNUMBER=/i) {
+            $discnum = int($value);
+            verbose("DISCNUMBER $discnum from $fn\n");

         }
     }
     close(TAGS) or die("close($fn): $!");
 
         }
     }
     close(TAGS) or die("close($fn): $!");
 

-    return ($artist, $album, @tags);
+    return ($artist, $album, $discnum, @tags);
+}
+
+sub track_tags {
+    my $h = shift;
+    my @result;
+
+    while (my ($key, $vall) = each(%$h)) {
+        for my $val (@$vall) {
+            push(@result, "$key=$val")
+        }
+    }
+
+    return @result;
+}
+
+sub run_flac {
+    my $infile = shift;
+    my $cue = shift;
+    my $outfile = shift;
+
+    my @cue;
+    if (defined($cue)) {
+        @cue = ('--cuesheet', $cue);
+    }
+
+    verbose("Running flac\n");
+    my $status = system('flac', '-o', "$outfile.flac-tmp",
+                     '--delete-input-file', '-V', '--no-padding', '--best',
+                     @cue,
+                     map({ ('-T', $_) } @_),
+                     $infile);
+    if (WIFEXITED($status)) {
+        if (($status = WEXITSTATUS($status)) != 0) {
+            die("flac exited with status $status");
+        }
+    } elsif (WIFSIGNALED($status)) {
+        die("flac killed with signal ", WTERMSIG($status));
+    } elsif (WIFSTOPPED($status)) {
+        die("flac stopped with signal ", WSTOPSIG($status));
+    } else {
+        die("Major horkage on system(flac): \$? = $? \$! = $!");
+    }
+
+    rename("$outfile.flac-tmp", "$outfile.flac")
+      or die("rename($outfile.flac-tmp, $outfile.flac): $!");

 }
 
 # Process the fa-rip output in the directory DIR.
 }
 
 # Process the fa-rip output in the directory DIR.


@@ -69,120 +175,142 @@ sub flac {
     my $dir = shift;
     my $artist;
     my $album;
     my $dir = shift;
     my $artist;
     my $album;

+    my $discnum;

     my @tags;
     my @tags;

-    my $status;
+    my $single_file = -e "$dir/wav";
+    my $outdir;
+    my $outfile;
+    my $outlog;
+    my @files;

 
     verbose("Renaming $dir/tags\n");
     rename("$dir/tags", "$dir/using-tags")
       or die("rename($dir/tags, $dir/using-tags): $!");
 
 
     verbose("Renaming $dir/tags\n");
     rename("$dir/tags", "$dir/using-tags")
       or die("rename($dir/tags, $dir/using-tags): $!");
 

-    ($artist, $album, @tags) = get_tags("$dir/using-tags");
+    ($artist, $album, $discnum, @tags) = get_tags("$dir/using-tags");
+    for ($artist, $album) {
+        s|/|_|g;
+    }

 
     verbose("mkdir($artist)\n");
     -d $artist or mkdir($artist) or die("mkdir($artist): $!");
 
 
     verbose("mkdir($artist)\n");
     -d $artist or mkdir($artist) or die("mkdir($artist): $!");
 

+    if (not $single_file) {
+        $outdir = "$artist/$album";
+        verbose("mkdir($outdir)\n");
+        -d "$outdir" or mkdir("$outdir") or die("mkdir($outdir): $!");
+    }
+

     verbose("chdir($dir)\n");
     chdir($dir) or die("chdir($dir): $!");
 
     verbose("chdir($dir)\n");
     chdir($dir) or die("chdir($dir): $!");
 

-    verbose("Running flac\n");
-    $status = system('flac', '-o', "../$artist/$album.flac-tmp",
-                     '--delete-input-file', '-V', '--cuesheet',
-                     'cue', '--no-padding', '--best',
-                     map({ ('-T', $_) } @tags),
-                     'wav');
-    if (WIFEXITED($status) and ($status = WEXITSTATUS($status)) != 0) {
-        die("flac: $status");
-    } elsif (WIFSIGNALED($status)) {
-        die("flac killed with signal ", WTERMSIG($status));
-    } elsif (WIFSTOPPED($status)) {
-        die("flac stopped with signal ", WSTOPSIG($status));
+    if ($single_file) {
+        $outfile = $album;
+        defined($discnum) and $outfile .= " (disc $discnum)";
+        run_flac('wav', 'cue', "../$artist/$outfile", @tags);
+        $outlog = "../$artist/$outfile.log";
+        @files = ("$artist/$outfile.flac");
+    } else {
+        # Go over @tags, store all [n] tags in a list keyed by n in
+        # %tracks_to_tags, store all ARTIST (not ARTIST[n]) tags in
+        # @disc_artist, and leave the rest in @tags.
+        my %tracks_to_tags;
+        my @disc_artist;
+        my @tmp;
+        for my $tag (@tags) {
+            if ($tag =~ /^([^[]+)\[(\d+)]=(.*)/) {
+                push(@{$tracks_to_tags{$2}->{$1}}, $3);
+            } elsif ($tag =~ /^ARTIST=/) {
+                push(@disc_artist, $tag);
+            } else {
+                push(@tmp, $tag);
+            }
+        }
+        @tags = @tmp;
+
+        for my $tracknum (sort(map(int, keys(%tracks_to_tags)))) {
+            my $title = join(' ', map(split, @{$tracks_to_tags{$tracknum}->{'TITLE'}}));
+            $title =~ s|/|_|g;
+            $outfile = join('/',
+                            $outdir,
+                            join(' ',
+                                 (defined($discnum)
+                                  ? sprintf('%02d', $discnum)
+                                  : ()),
+                                 sprintf('%02d', $tracknum),
+                                 $title));
+
+            # If we have ARTIST[n] tags for this track, set
+            # @track_artist to the empty list; they will go in along
+            # with the other [n] tags.
+            my @track_artist;
+            if (exists($tracks_to_tags{$tracknum}->{'ARTIST'})) {
+                @track_artist = ();
+            } else {
+                @track_artist = @disc_artist;
+            }
+
+            run_flac(sprintf('track%02d.cdda.wav', $tracknum), undef,
+                     "../$outfile",
+                     @track_artist,
+                     @tags,
+                     "TRACKNUMBER=$tracknum",
+                     track_tags($tracks_to_tags{$tracknum}));
+            push(@files, "$outfile.flac");
+        }
+        $outlog = "../$outdir/log";

     }
 
     verbose("Cleaning up $dir\n");
     unlink('using-tags') or die("unlink(using-tags): $!");
     unlink('cue') or die("unlink(cue): $!");
     }
 
     verbose("Cleaning up $dir\n");
     unlink('using-tags') or die("unlink(using-tags): $!");
     unlink('cue') or die("unlink(cue): $!");

-    rename('toc', "../$artist/$album.toc")
-      or die("rename(toc, ../$artist/$album.toc): $!");
-    rename('log', "../$artist/$album.log")
-      or die("rename(log, ../$artist/$album.log): $!");
+    rename('log', $outlog)
+      or die("rename(log, $outlog): $!");

     chdir('..') or die("chdir(..): $!");
     chdir('..') or die("chdir(..): $!");

-    rmdir($dir) or die("rmdir($dir): $!");
-
-    rename("$artist/$album.flac-tmp", "$artist/$album.flac")
-      or die("rename($artist/$album.flac-tmp, $artist/$album.flac): $!");
-
-    return 0;
-}

 
 

-sub reaper {
-    my $pid;
-
-    while (($pid = waitpid(0, WNOHANG)) > 0) {
-        push(@finished, [$pid, $?]);
-    }
-
-    $SIG{CHLD} = \&reaper;
-}
-
-sub newjob {
-    my $dir = shift;
-    my $pid;
-
-    $pid = fork();
-    if (not defined($pid)) {
-        die("fork: $!");
-    } elsif ($pid == 0) {
-        $SIG{CHLD} = 'IGNORE';
-        open(STDERR, ">$dir/log") or die("open(STDERR, >$dir/log): $!");
-        exit(flac($dir));
+    if (-x "$dir/post-processor") {
+        verbose(join(' ', "Running ./$dir/post-processor", @files), "\n");
+        system("./$dir/post-processor", @files);
+        unlink("$dir/post-processor") or die("unlink($dir/post-processor): $!");

     }
 
     }
 

-    verbose("new job $pid for $dir\n");
-    return $pid;
-}
-
-sub deljob {
-    my $i = shift;
-    my $j;
-    my $pid;
-    my $status;
-
-    $pid = $finished[$i][0];
-    $status = $finished[$i][1];
-
-    verbose("$pid finished (");
-    if (WIFEXITED($status)) {
-        verbose('exited ', WEXITSTATUS($status));
-    } elsif (WIFSIGNALED($status)) {
-        verbose('signalled ', WTERMSIG($status));
-    } elsif (WIFSTOPPED($status)) {
-        verbose('stopped ', WSTOPSIG($status));
-    }
-    verbose(")\n");
-
-    for ($j = 0; $j <= $#jobs; $j++) {
-        $pid == $jobs[$j] and splice(@jobs, $j, 1) and last;
-    }
+    rmdir($dir) or die("rmdir($dir): $!");

 
 

-    splice(@finished, $i, 1);
+    return 0;

 }
 
 sub flacloop {
     my $MAXJOBS = shift;
     my $i;
 }
 
 sub flacloop {
     my $MAXJOBS = shift;
     my $i;

-    my $j;

 
 
     $SIG{CHLD} = \&reaper;
     while (1) {
         if (scalar(@jobs) <= $MAXJOBS) {
             foreach $i (glob('*/tags')) {
 
 
     $SIG{CHLD} = \&reaper;
     while (1) {
         if (scalar(@jobs) <= $MAXJOBS) {
             foreach $i (glob('*/tags')) {

-                push(@jobs, newjob(dirname($i))) <= $MAXJOBS or last;
+                my $dir = dirname($i);
+                my $pid =
+                  Jobs::newjob(sub {
+                                   open(STDERR, ">$dir/log")
+                                     or die("open(STDERR, >$dir/log): $!");
+                                   return flac($dir);
+                               }, 'debug'=>$debug);
+                verbose("new job $pid for $dir\n");
+                @Jobs::jobs <= $MAXJOBS or last;

             }
         }
 
         for ($i = 0; $i <= $#finished; $i++) {
             }
         }
 
         for ($i = 0; $i <= $#finished; $i++) {

-            deljob($i);
+            my ($pid, $status) = Jobs::deljob($i);
+            verbose("$pid finished (");
+            if (WIFEXITED($status)) {
+                verbose('exited with status ', WEXITSTATUS($status));
+            } elsif (WIFSIGNALED($status)) {
+                verbose('killed with signal ', WTERMSIG($status));
+            } elsif (WIFSTOPPED($status)) {
+                verbose('stopped with signal ', WSTOPSIG($status));
+            }
+            verbose(")\n");

         }
 
         verbose(scalar(@jobs), " jobs\n");
         }
 
         verbose(scalar(@jobs), " jobs\n");


@@ -191,18 +319,19 @@ sub flacloop {
 }
 
 MAIN: {
 }
 
 MAIN: {

-    my %opts;
-
-    $opts{'j'} = 4;
-    $opts{'v'} = 0;
-    if (not getopts('j:v', \%opts)) {
-        print("usage: flacd [-jN -v]\n");
-        exit(2);
-    }
-
-    $verbose = $opts{'v'};
-
-    flacloop($opts{'j'});
+    my $jobs;
+    my $help;
+
+    $jobs = 4;
+    GetOptions(
+               'debug|X' => \$debug,
+               'jobs|j=i' => \$jobs,
+               'verbose|v' => \$verbose,
+               'help|h|?' => \$help,
+              ) or pod2usage();
+    $help and pod2usage(-exitstatus=>0, -verbose=>1);
+
+    flacloop($jobs);

 }
 
 \f
 }
 
 \f


@@ -210,29 +339,23 @@ __END__
 
 =head1 DESCRIPTION
 
 
 =head1 DESCRIPTION
 

-B<fa-flacd>, B<fa-rip>, and B<fa-tags> together comprise
-B<flac-archive>, a system for archiving audio CDs to single FLAC
-files.  B<fa-flacd> is the guts of the system.  It runs in the
-directory where the audio archives are stored, scanning for new CDs to
-encode and rename; it never exits.  B<fa-rip> generates the inputs for
-B<fa-flacd>: the ripped WAV file, Vorbis tags, and a cuesheet.
-B<fa-tags> is not meant to be run directly; B<fa-rip> uses it to
-generate the candidate Vorbis tags.
-
-All three programs expect to be run from the same directory.  They use
-that directory to manage directories named by artist and by disc ID.
-Intermediate files are written to the disc ID directory.  B<fa-flacd>
-processes the disc ID directories into per-album files in the artist
-directories.
-
-=head2 FA-FLACD
-
-B<fa-flacd> does not exit; it runs until the user kills it.  Every 5
-seconds it scans its current directory for directories with a file
-called "tags" and creates a processing job for each one.  The number
-of jobs B<fa-flacd> attempts to run is controlled by the B<-j> option
-and defaults to 4.  B<fa-flacd> will print diagnostic output when the
-B<-v> option is given.
+B<fa-flacd> and B<fa-rip> together comprise B<flac-archive>, a system
+for archiving audio CDs to single FLAC files.  B<fa-flacd> is the guts
+of the system.  It runs in the directory where the audio archives are
+stored, scanning for new ripped CDs to encode and rename; it never
+exits.  B<fa-rip> generates the inputs for B<fa-flacd>: the ripped WAV
+file, Vorbis tags, and a cuesheet.
+
+Both programs expect to be run from the same directory.  They use that
+directory to manage directories named by artist.  Intermediate files
+are written to temporary directories here.  B<fa-flacd> processes the
+temporary directories into per-album files in the artist directories.
+
+Every 5 seconds, B<fa-flacd> scans its current directory for
+directories with a file called "tags" and creates a processing job for
+each one.  The number of jobs B<fa-flacd> attempts to run is
+controlled by the B<-j> option and defaults to 4.  B<fa-flacd> will
+print diagnostic output when the B<-v> option is given.

 
 A processing job first renames the directory's "tags" file to
 "using-tags" so that B<ra-flacd> will not try to start another job for
 
 A processing job first renames the directory's "tags" file to
 "using-tags" so that B<ra-flacd> will not try to start another job for


@@ -241,49 +364,32 @@ so a new job will not be started until the user corrects the error
 condition and renames "using-tags" back to "tags".  Next, it encodes
 the "wav" file to a FLAC file, using the "cue" file for the cuesheet
 and "using-tags" for Vorbis tags.  Any diagnostic output is saved in
 condition and renames "using-tags" back to "tags".  Next, it encodes
 the "wav" file to a FLAC file, using the "cue" file for the cuesheet
 and "using-tags" for Vorbis tags.  Any diagnostic output is saved in

-the "log" file.  Finally, the "cue" and "log" files are moved to the
-artist directory (and named by album) and the ID directory is removed.
-
-=head2 FA-RIP
-
-B<fa-rip> uses C<cd-discid(1)> to retrieve the disc ID and track
-information.  It creates a directory named by ID for storage of its
-intermediate files.  It passes the C<cd-discid(1)> output as
-command-line arguments to B<fa-tags> in the background.  It then uses
-C<cdrdao(1)> to create the "cue" file in the background.  Finally, it
-execs C<cdparanoia(1)> to rip the CD to the "wav" file.
-
-In order for this CD to be processed by B<fa-flacd>, the user must
-create a "tags" file.  This is usually done by renaming one of the
-candidate-tags files and deleting the others.
-
-=head2 FA-TAGS
-
-B<fa-tags> uses C<cddb-tool(1)> (from the B<abcde> package) to
-populate candidate-tags files.  These are numbered in the order of
-entries read from CDDB, e.g. candidate-tags-1, candidate-tags-2, etc.
-B<fa-tags> also creates candidate-tags-0, which has the correct fields
-for this CD (including correct number of TITLE= lines), but with all
-fields blank.
-
-B<fa-tags> expects the output of C<cd-discid(1)> as command-line
-arguments.  That is, the disc ID, number of tracks, list of track
-offsets, and total length of the CD in seconds.
-
-=head1 ENVIRONMENT
+the "log" file.  Finally, B<fa-flacd> moves the "cue" and "log" files
+to the artist directory (named by album) and removes the temporary
+directory.
+
+If the temporary directory contains an executable file named
+"post-processor", B<fa-flacd> executes that file with the relative
+path to the output FLAC file as an argument.  The output files are in
+their final location when "post-processor" starts.  Possible uses are
+running B<flac2mp3>, moving the output files to a different location,
+removing the lock file, or adding to a database.  The standard input,
+output, and error streams are inherited from B<fa-flacd>, so they may
+be connected to anything from a tty to /dev/null.  This means that you
+may want to redirect these streams, if you want to save them or do any
+logging.
+
+=head1 OPTIONS

 
 =over 4
 
 
 =over 4
 

-=item CDDBURL
+=item B<-j> [B<--jobs>] I<jobs>

 
 

-B<fa-tags> uses this to retrieve candidate Vorbis tags.  Defaults to
-"http://freedb.freedb.org/~cddb/cddb.cgi".
+Run up to I<jobs> jobs instead of the default 4.

 
 

-=item CDDEV
+=item B<-v> [B<--verbose>]

 
 

-B<fa-rip> uses this to rip audio and save the cuesheet for a CD.  It
-makes some effort to check some common device names for FreeBSD,
-Linux, and NetBSD by default.
+Print diagnostic information.

 
 =back
 
 
 =back