Index: /trunk/ippScripts/Build.PL
===================================================================
--- /trunk/ippScripts/Build.PL	(revision 23249)
+++ /trunk/ippScripts/Build.PL	(revision 23250)
@@ -80,4 +80,6 @@
         scripts/summit_copy.pl
         scripts/ipp_image_path.pl
+        scripts/dist_component.pl
+        scripts/dist_advancerun.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',
Index: /trunk/ippScripts/scripts/dist_advancerun.pl
===================================================================
--- /trunk/ippScripts/scripts/dist_advancerun.pl	(revision 23250)
+++ /trunk/ippScripts/scripts/dist_advancerun.pl	(revision 23250)
@@ -0,0 +1,152 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Parse the command-line arguments
+my ($dist_id, $stage, $stage_id, $clean, $outroot);
+my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'stage=s'        => \$stage,      # raw, chip, warp, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
+           'outroot=s'      => \$outroot,    # "directory" for outputs
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outroot",
+           -exitval => 3) unless
+    defined $dist_id and
+    defined $stage and
+    defined $stage_id and
+    defined $outroot;
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+# Look for programs we need
+my $missing_tools;
+my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
+my $regtool   = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+my $chiptool   = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $camtool   = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $faketool   = can_run('faketool') or (warn "Can't find faketool" and $missing_tools = 1);
+my $warptool   = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+my $difftool   = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $stacktool   = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+if ($missing_tools) {
+    &my_die("Can't find required tools.", $dist_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+my $tool;
+if ($stage eq "raw") {
+    $tool = "regtool";
+} elsif ($stage eq "chip") {
+    $tool = "chiptool";
+} elsif ($stage eq "camera") {
+    $tool = "camtool";
+} elsif ($stage eq "fake") {
+    $tool = "faketool";
+} elsif ($stage eq "warp") {
+    $tool = "warptool";
+} elsif ($stage eq "stack") {
+    $tool = "stacktool";
+} elsif ($stage eq "diff") {
+    $tool = "difftool";
+} else {
+    &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+# XXX Should we create a file rule for this
+my $outfile = "$outroot/dbinfo.$stage.$stage_id.mdc";
+
+{
+    my $id_arg = "-$stage" . "_id";
+    my $command = "$tool -exportrun $id_arg $stage_id -outfile $outfile";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
+}
+
+# set distRun.stage = 'full'
+{
+    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
+}
+
+
+exit 0;
+
+### Pau.
+
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $dist_id = shift;    # Magic DS identifier
+    my $exit_code = shift;      # Exit code to add
+
+    my $command = "$disttool -updaterun";
+    $command   .= " -dist_id $dist_id";
+    $command   .= " -code $exit_code";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $dist_id");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    carp($msg);
+    exit $exit_code;
+}
+
+__END__
Index: /trunk/ippScripts/scripts/dist_component.pl
===================================================================
--- /trunk/ippScripts/scripts/dist_component.pl	(revision 23250)
+++ /trunk/ippScripts/scripts/dist_component.pl	(revision 23250)
@@ -0,0 +1,339 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Digest::MD5::File qw( file_md5_hex );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
+my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
+my ($outroot, $run_state, $data_state, $magicked);
+my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,     # camera for evaluating file rules
+           'stage=s'        => \$stage,      # raw, chip, warp, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
+           'component=s'    => \$component,  # the class_id or skycell_id
+           'path_base=s'    => \$path_base,  # path_base of the input
+           'chip_path_base=s'=> \$chip_path_base,  # path base for camera stage (to enable us to find the mask filefor raw images)
+           'state=s'        => \$run_state,  # state of the run
+           'data_state=s'   => \$data_state, # data_state for this component
+           'magicked=s'     => \$magicked,   # data_state for this component
+           'outroot=s'      => \$outroot,    # "directory" for outputs
+           'clean=s'        => \$clean,      # create clean distribution
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outroot",
+           -exitval => 3) unless
+    defined $dist_id and
+    defined $camera and
+    defined $stage and
+    defined $stage_id and
+    defined $component and
+    defined $path_base and
+    defined $outroot;
+
+if (($stage eq 'raw') and !defined $chip_path_base) {
+    pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
+}
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+$ipprc->define_camera($camera);
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $basename = basename($path_base);
+
+#
+# are we creating a clean distribution?
+if (defined($clean) and ($clean eq "T")) {
+    $clean = 1;
+} else {
+    $clean = 0;
+}
+
+# create the output directories if it is not a nebulous path and it doesn't exist
+if (index($outroot, "neb://") != 0) {
+    if (! -e $outroot ) {
+        my $code = system "mkdir -p $outroot";
+        &my_die("cannot create output directory $outroot", $dist_id, $component,
+                $code >> 8) if $code;
+    }
+}
+
+my $file_list = get_file_list($clean, $stage, $path_base);
+
+&my_die("failed to compute product list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$file_list;
+
+&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !scalar @$file_list;
+
+
+# set up directory for temporary files
+
+my $tmpdir  = "$outroot/tmpdir.$dist_id.$component.$$";
+if (-e $tmpdir) {
+    if (-d $tmpdir) {
+        my $rc = system "rm -r $tmpdir"; 
+        &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
+    } else {
+        unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component,
+                       $PS_EXIT_UNKNOWN_ERROR);
+
+my @base_list;
+foreach my $file (@$file_list) {
+    my $base = basename($file);
+    push @base_list, $base;
+    my $path = $ipprc->file_resolve($file);
+    symlink $path, "$tmpdir/$base";
+}
+
+if (!$clean) {
+
+    # Here is where we determine whether or not a GPC1 image can be released.
+
+    if ($magicked and ($magicked eq "F")) {
+        $magicked = 0;
+    }
+
+    if ($camera eq "GPC1" and !$magicked) {
+        &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
+    }
+
+    # run streaksrelease to set masked pixels to NAN
+    my ($image, $mask, $variance, $class_id);
+    if ($stage eq "raw") {
+        $class_id = $component;
+        $image = $path_base;
+        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $class_id);
+    } elsif ($stage eq "chip") {
+        $class_id = $component;
+        $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
+        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+        $variance = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+    } elsif ($stage eq "warp") {
+        $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
+        $variance = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
+    } elsif ($stage eq "diff") {
+        $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
+        $variance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
+    } else {
+        &my_die("not ready for stage: $stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    push @base_list, $image;
+    push @base_list, $mask if $mask;
+    push @base_list, $variance if $variance;
+
+    my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
+    $command .= " -class_id $class_id" if $class_id;
+    $command .= " -mask $mask" if $mask;
+    $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask);
+    $command .= " -weight $variance" if $variance;
+    unless (defined $no_op) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+        }
+    } else {
+        print "skipping command $command\n";
+    }
+}
+
+
+my $file_name;
+my $bytes;
+my $md5sum;
+# create the tarfile
+{
+    # XXX TODO: create a file rule for the tar file name
+    my $tbase = basename($path_base);
+    $tbase .= ".$component" if $component;
+    $file_name = "$tbase.tgz";
+    my $tarfile = "$outroot/$file_name";
+
+    my $command = "tar -C $tmpdir -czhf $tarfile .";
+
+    unless (defined $no_op) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+        }
+    } else {
+        print "skipping command $command\n";
+    }
+
+    # tell the module not to die on error
+    $Digest::MD5::File::NOFATALS = 1;
+    $md5sum = file_md5_hex($tarfile);
+    &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
+
+    my @finfo = stat($tarfile);
+    &my_die("unable to stat $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !@finfo;
+    $bytes = $finfo[7];
+
+    if (!$save_temps) {
+        # delete the temp directory and it's contents
+        system "rm -r $tmpdir";
+    }
+}
+{
+    my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component";
+    $command .= " -name $file_name -bytes $bytes -md5sum $md5sum";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    unless (defined $no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+        }
+    } else {
+        print "skipping command $command\n";
+    }
+}
+
+exit 0;
+
+### Pau.
+
+
+sub get_file_list {
+    my $clean = shift;
+    my $stage = shift;
+    my $path_base = shift;
+
+    my @file_list;
+    if ($stage eq "raw") {
+        push @file_list, $path_base;
+    } else {
+        # TODO: these data will eventually come from the CONFIG dump
+
+        my $fh;
+        open $fh, "/data/ipp004.0/home/bills/ipp/ippScripts/scripts/clean.mdc" or
+            &my_die("cannot find clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+
+        my @data = <$fh>;
+        my $clean_mdc = join "", @data;
+
+        my $mdlist = $mdcParser->parse($clean_mdc) or 
+                &my_die("failed to parse clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $product_lists = parse_md_list($mdlist) or
+                &my_die("failed to parse metadata list", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+
+
+        my $product_list;
+        foreach my $list (@$product_lists) {
+
+            my $list_name = $list->{STAGE};
+            if (lc ($list_name) eq lc($stage) ) {
+                $product_list = $list;
+            }
+        }
+        &my_die("failed to parse find product list for stage $stage", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$product_list;
+
+        # resolve each file rule and add it to the file_list
+        foreach my $rule (keys %$product_list) {
+            next if $rule eq "STAGE";
+            my $keep_on_clean = $product_list->{$rule};
+            my $fn = $ipprc->filename($rule, $path_base, $component) or 
+                &my_die("Missing entry from camera config: $rule", $dist_id, $component, 
+                    $PS_EXIT_CONFIG_ERROR);
+    #        printf "%-16.16s\t%s\t%s\n",  $rule, $clean, $fn;
+            if (!$clean or $keep_on_clean) {
+                push @file_list, $fn;
+            }
+        }
+    }
+
+    return \@file_list;
+}
+
+
+
+
+sub file_check
+{
+    my $file = shift;           # Name of file
+    &my_die("Unable to find output file: $file", $dist_id, $component, $PS_EXIT_SYS_ERROR) unless
+        $ipprc->file_exists($file);
+}
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $dist_id = shift;    # Magic DS identifier
+    my $component = shift;      # class_id or skycell_id
+    my $exit_code = shift;      # Exit code to add
+
+    my $command = "$disttool -addprocessedcomponent";
+    $command   .= " -dist_id $dist_id";
+    $command   .= " -component $component";
+    $command   .= " -code $exit_code";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $dist_id $component");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    carp($msg);
+    exit $exit_code;
+}
+
+__END__