Changeset 24244 for branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_advancerun.pl

Timestamp:

May 26, 2009, 1:59:32 PM (17 years ago)

Author:

beaumont

Message:

merged with trunk

Location:

branches/cnb_branches/cnb_branch_20090301

Files:

• . (modified) (1 prop)
• ippScripts/scripts/dist_advancerun.pl (modified) (9 diffs)

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_advancerun.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
-use File::Basename qw( basename );
+use File::Basename qw( dirname);
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -28 +28 @@
 
 # Parse the command-line arguments
-my ($dist_id, $stage, $stage_id, $clean, $outroot);
+my ($dist_id, $stage, $stage_id, $outdir, $clean);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -35 +35 @@
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
-           'outroot=s'      => \$outroot,    # "directory" for outputs
+           'clean'          => \$clean,      # exporting a clean run
+           'outdir=s'       => \$outdir,     # "directory" for outputs
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -45 +46 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outroot",
+pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outdir",
            -exitval => 3) unless
     defined $dist_id and
     defined $stage and
     defined $stage_id and
-    defined $outroot;
+    defined $outdir;
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -70 +71 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-my $tool;
+my $tool_cmd;
+my $list_mode;
+my $component_key;
 if ($stage eq "raw") {
-    $tool = "regtool";
+    $tool_cmd = "$regtool -exp_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "chip") {
-    $tool = "chiptool";
+    $tool_cmd = "$chiptool -chip_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "camera") {
-    $tool = "camtool";
+    $tool_cmd = "$camtool -cam_id";
+    $list_mode = "-processedexp";
+    $component_key = "";
 } elsif ($stage eq "fake") {
-    $tool = "faketool";
+    $tool_cmd = "$faketool -fake_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "warp") {
-    $tool = "warptool";
+    $tool_cmd = "$warptool -warp_id";
+    $list_mode = "-warped";
+    $component_key = "skycell_id";
 } elsif ($stage eq "stack") {
-    $tool = "stacktool";
+    $tool_cmd = "$stacktool -stack_id";
+    $list_mode = "-sumskyfile";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
-    $tool = "difftool";
+    $tool_cmd = "$difftool -diff_id";
+    $list_mode = "-diffskyfile";
+    $component_key = "skycell_id";
 } else {
     &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
 }
 
-# XXX Should we create a file rule for this
-my $outfile = "$outroot/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $id_arg = "-$stage" . "_id";
-    my $command = "$tool -exportrun $id_arg $stage_id -outfile $outfile";
+$tool_cmd .= " $stage_id";
+
+# XXX should we create a file rule for this?
+my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
+
+{
+    my $command = "$tool_cmd -exportrun -outfile $dbinfo_file";
+    $command .= " -clean" if defined $clean;
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -104 +123 @@
     }
 }
-
-# set distRun.stage = 'full'
-{
-    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+{
+    my $command = "$tool_cmd $list_mode";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -116 +134 @@
         &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
     }
+    if (@$stdout_buf == 0) {
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $components = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    open MANIFEST, ">$dirinfo" or
+        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $destdir;
+    foreach my $c (@$components) {
+        # take the workdir from the first component
+        if (!$destdir) {
+            my $workdir = $c->{workdir};
+            if ($workdir) {
+                $destdir = stripvolume($workdir, $stage);
+            } elsif ($stage eq 'raw') {
+                $destdir = 'none';
+            } else {
+                &my_die("workdir not found for open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+            }
+            print MANIFEST "destdir METADATA\n";
+            print MANIFEST "\t" , "destdir", "\tSTR\t", $destdir, "\n";
+            print MANIFEST "END\n\n";
+            print MANIFEST "components METADATA\n";
+        }
+        my $component = $c->{$component_key} ? $c->{$component_key} : "exposure";
+        my $path;
+        if ($stage eq 'raw') {
+            $path = $c->{uri};
+        } else {
+            $path = $c->{path_base};
+        }
+        &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
+        my $component_dir = find_componentdir($destdir, $path); 
+#        print MANIFEST "$component METADATA\n";
+        print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
+    }
+    print MANIFEST "END\n\n";
+    close MANIFEST or 
+        &my_die("Unable to close dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+{
+    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
 }
 
@@ -123 +196 @@
 ### Pau.
 
+sub stripvolume
+{
+    my $path = shift;
+    my $stage = shift;
+    my @segments;
+
+    # workdir isn't what we want for raw stage
+    return "none" if ($stage and ($stage eq 'raw'));
+
+    my $scheme = file_scheme($path);
+    my $tail;
+    if ($scheme) {
+        # strip off scheme://
+        $tail = substr($path, length($scheme) + 3);
+    } elsif (substr($path, 0, 1) eq '/') {
+        $tail = substr($path, 1);
+        $scheme = "";
+    }
+    # remove any leading / that are left
+    while ((substr($tail, 0, 1) eq '/')) {
+        $tail = substr($tail, 1);
+    }
+
+    if (($scheme eq 'neb') or ($scheme eq 'path')) {
+        my $volume;
+        ($volume, @segments) = split '/', $tail;
+
+    } elsif (!$scheme or ($scheme eq 'file')) {
+
+        # XXX Here we're assuming the /data/ipp??? structure. This won't be true when data is forwarded
+        # by remote sites. We need a way to configure this
+        my $volume;
+
+        # data/ippxxx/dirs
+        (undef, $volume, @segments) = split '/', $tail;
+    } else {
+        die( "unexpected workdir value: $path\n");
+    }
+
+    return caturi(@segments);
+}
+
+sub find_componentdir
+{
+    my $destdir = shift;
+    my $path = shift;
+
+    my $result;
+    if ($destdir eq 'none') {
+        $result = stripvolume($path);
+    } else {
+        # find location of destdir in the path
+        my $i = index($path, $destdir);
+
+        $result = substr($path, $i + length($destdir) + 1);
+
+        while (substr($result, 0, 1) eq '/') {
+            $result = substr($result, 1);
+        }
+    }
+    return dirname($result);
+}
 
 sub my_die
@@ -134 +269 @@
     my $command = "$disttool -updaterun";
     $command   .= " -dist_id $dist_id";
-    $command   .= " -code $exit_code";
+    $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;