Changeset 27838 for branches/tap_branches/ippScripts/scripts/magic_destreak_cleanup.pl

Timestamp:

May 3, 2010, 8:41:49 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/tap_branches

Files:

• . (modified) (1 prop)
• ippScripts/scripts/magic_destreak_cleanup.pl (modified) (15 diffs)

branches/tap_branches/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -8 +8 @@
 use Sys::Hostname;
 my $host = hostname();
+my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -30 +31 @@
 my $missing_tools;
 my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -61 +63 @@
 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 
-if (0) {
-my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
-&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer;
-
-my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
-if ($@ or not defined $nebulous) {
-    &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR);
-}
-}
 
 $dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname;
@@ -86 +79 @@
 my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n";
 
-my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, magicRun.inverse"
-         . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id)"
+my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, camRun.reduction AS cam_reduction, magicRun.inverse"
+         . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN camRun USING(cam_id)"
          . " WHERE magic_ds_id = $magic_ds_id";
 my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id";
-
-# my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 my $stmt1 = $dbh->prepare($q1);
@@ -103 +94 @@
 my $stage_id = $run->{stage_id};
 my $cam_path_base = $run->{cam_path_base};
+my $cam_reduction = $run->{cam_reduction};
+$cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL');
 my $inverse = $run->{inverse};
+
+
+&my_die("cleanup not supported for camera stage", $magic_ds_id, $PS_EXIT_PROG_ERROR) if $stage eq "camera";
 
 
@@ -109 +105 @@
 &my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw";
 
+my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use
+&my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
+
 
 my $stmt2 = $dbh->prepare($q2);
@@ -114 +113 @@
 
 my $num_components = 0;
+my @components;
+# save the data, so we can disconnect from the database
+# deleting can take awhile
 while (my $comp = $stmt2->fetchrow_hashref()) {
+        push @components, $comp;
         $num_components++;
-
+}
+
+$stmt2->finish();
+$dbh->disconnect() or warn $dbh->errstr;
+
+# We no longer clean up the camera stage mask files, but the code was left in place in case
+# we change our minds.
+my $cleanup_cam_mask = 0;
+
+my $dynamicMasks;               # Use dynamic masks?
+foreach my $comp (@components) {
         my $component = $comp->{component};
         my $backup_path_base = $comp->{backup_path_base};
@@ -124 +137 @@
 
         if ($stage eq "chip") {
-            # we use the mask output from the camera stage for input and replace
-            # the output of the chip stage with that mask as well.
-            my $cammask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
+            # Check to see if we're using dynamic masks
+            if ($cleanup_cam_mask && !defined $dynamicMasks) {
+                # Get the PSASTRO recipe
+                my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform ppConfigDump: $error_code", $magic_ds_id, $component,
+                            $PS_EXIT_CONFIG_ERROR);
+                }
+                my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+                my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config doc", $magic_ds_id, $component,
+                            $PS_EXIT_CONFIG_ERROR);
+
+                $dynamicMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
+            }
 
             if ($backup_path_base) {
                 $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $component);
-                $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $component);
+
+                if ($dynamicMasks) {
+                    my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
+                    # This is kludgey but correct
+                    $bmask = dirname($backup_path_base) . "/SR_" . basename($mask);
+                    $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $component);
+                } else {
+                    $bmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $component);
+                }
                 $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $component);
-                # This is kludgey but correct
-                
-                my $prefix;
-                if ($backup_path_base =~ /\.SR/) {
-                    $prefix = "";
-                } else {
-                    $prefix = "SR_";
-                }
-
-                $bmask   = dirname($backup_path_base) . "/" . $prefix . basename($cammask);
             }
             if ($recovery_path_base) {
                 $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component);
-                $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
+
+                if ($dynamicMasks) {
+                    my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
+                    # This is kludgey but correct
+                    $rmask = dirname($recovery_path_base) . "/" . basename($mask);
+                    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
+                } else {
+                    $rmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
+                }
                 $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component);
-                # This is kludgey but correct
-                $rmask   = dirname($recovery_path_base) . "/" . basename($cammask);
             }
         } elsif ($stage eq "camera") {
@@ -160 +192 @@
                 $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
                 $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
-            } 
+            }
             if ($recovery_path_base) {
                 $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
@@ -166 +198 @@
                 $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
                 $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base);
-            } 
+            }
         } elsif ($stage eq "diff") {
             my $name = "PPSUB.OUTPUT";
@@ -183 +215 @@
         }
 
-        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
 
         if ($stage eq "diff" and $inverse) {
@@ -199 +231 @@
                 $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
             }
-            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+            # undef is not necessary, it's just to keep the lists in the same order.
+            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources);
         }
 }
+
+$dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n";
 
 if (!$no_update and ($num_components > 0)) {
@@ -211 +246 @@
 if (!$no_update) {
     my $result = $dbh->do("UPDATE magicDSRun SET state = 'cleaned' WHERE magic_ds_id = ?", undef, $magic_ds_id);
-    my_die("attempt to update magicDSRun failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
+    my_die("attempt to update magicDSRun.state failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
 } else {
     print STDERR "skipping update of magicDSRun\n";
@@ -225 +260 @@
                 if (!$no_update) {
                     print STDERR "deleting $file\n" if $verbose;
-                    $ipprc->file_delete($file) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR);
+                    $ipprc->file_delete($file, 1) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR);
                     } else {
                     print STDERR "skipping delete $file\n";
@@ -246 +281 @@
     $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
 
-    my $command = "$magicdstool -updaterun -fault $exit_code";
+    my $command = "$magicdstool -updaterun -set_state failed_cleanup";
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -dbname $dbname" if defined $dbname;