Changeset 23352 for branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_cleanup.pl

Timestamp:

Mar 17, 2009, 12:08:50 PM (17 years ago)

Author:

beaumont

Message:

merged with head

Location:

branches/cnb_branches/cnb_branch_20090301

Files:

• . (modified) (1 prop)
• ippScripts/scripts/ipp_cleanup.pl (modified) (14 diffs)

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_cleanup.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions
-
 # Parse the command-line arguments
 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist);
 GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
-           'camera|i=s'     => \$camera,    # user-supplied camera name
-           'stage_id=s'     => \$stage_id,  # id for this stage (only needed for certain stages)
-           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
-           'path_base=s'    => \$path_base, # basename for files
-           'dbname|d=s'     => \$dbname,    # Database name
+           'camera|i=s'     => \$camera,    # user-supplied camera name
+           'stage_id=s'     => \$stage_id,  # id for this stage (only needed for certain stages)
+           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
+           'path_base=s'    => \$path_base, # basename for files
+           'dbname|d=s'     => \$dbname,    # Database name
            'verbose'        => \$verbose,   # Print to stdout
-           'no-op'          => \$no_op,     # pretend but don't actually inject
-           'helplist'       => \$helplist   # give help listing
-           ) or pod2usage( 2 );
-
-pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage", 
-           -exitval => 2) if defined $helplist;
-
-pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]", 
-           -exitval => 2 ) if scalar @ARGV;
+           'no-op'          => \$no_op,     # pretend but don't actually inject
+           'helplist'       => \$helplist   # give help listing
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage",
+           -exitval => 2) if defined $helplist;
+
+pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]",
+           -exitval => 2 ) if scalar @ARGV;
 
 pod2usage( -msg => "Required options:--camera (name) --stage (stage) --mode (mode)",
-           -exitval => 3) unless 
+           -exitval => 3) unless
     defined $camera and
     defined $stage and
     defined $mode;
 
-# $mode must be one of "goto_cleaned" or "goto_purged"
-unless (($mode eq "goto_cleaned") || ($mode eq "goto_purged")) {
-    die "invalid cleanup mode $mode\n";    
-}
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up", $stage_id, $PS_EXIT_CONFIG_ERROR); # this is used for PATH, NEB filename conversions
+
+# $mode must be one of "goto_cleaned", "goto_scrubbed", or
+# "goto_purged" goto_cleaned and goto_scrubbed both result in
+# 'cleaned' on success ('scrubbed' allows chips without config files
+# to be cleaned; they cannot be recovered, but the small data is left
+# behind). XXX make 'scrubbed' a data_state?
+unless (($mode eq "goto_cleaned") || ($mode eq "goto_scrubbed") || ($mode eq "goto_purged")) {
+    die "invalid cleanup mode $mode\n";
+}
+
+my $error_state;
+if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
+if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
+if ($mode eq "goto_purged")   { $error_state = "error_purged";   }
+
 
 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1);
@@ -55 +65 @@
 }
 
-$ipprc->define_camera($camera);
-
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 # choice of files to delete depends on the stage
 if ($stage eq "chip") {
-    
+
     die "--stage_id required for stage chip\n" if !$stage_id;
     ### select the imfiles for this entry
@@ -78 +86 @@
         &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     }
+
+    # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
+    if (@$stdout_buf == 0)  {
+        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state new";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+        }
+        exit 0;
+    }
+
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
@@ -87 +110 @@
     # loop over all of the imfiles, determine the path_base and class_id for each
     foreach my $imfile (@$imfiles) {
-        my $class_id = $imfile->{class_id};
-        my $path_base = $imfile->{path_base};
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
         my $status = 1;
 
         # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
         if ($mode eq "goto_cleaned") {
             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
@@ -124 +148 @@
                 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
             }
-        
+
             # actual command to delete the files
             $status = &delete_files (\@files);
         }
 
-        if ($status)  {
-            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
+        if ($status)  {
+            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedimfile";
@@ -136 +160 @@
                 $command .= " -tocleanedimfile";
             }
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+        } else {
+
+            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state";
             $command .= " -dbname $dbname" if defined $dbname;
 
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
-            }
-        } else {
-            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1";
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
-            }
-        }
-    }
-
-} elsif ($stage eq "camera") {
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+
+if ($stage eq "camera") {
     die "--stage_id required for stage camera\n" if !$stage_id;
     # this stage uses 'camtool'
@@ -210 +238 @@
 
     if ($status)  {
-        my $command = "$camtool -cam_id $stage_id -updaterun";
+        my $command;
         if ($mode eq "goto_cleaned") {
-            $command .= " -state cleaned";
-        } else {
-            $command .= " -state purged";
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
+        }
+        if ($mode eq "goto_scrubbed") {
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
+        }
+        if ($mode eq "goto_purged") {
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state purged";
         }
         $command .= " -dbname $dbname" if defined $dbname;
@@ -224 +256 @@
         }
     } else {
-        my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1";
+        # since 'camera' has only a single imfile, we can just update the run
+        my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -236 +269 @@
     }
     exit 0;
-} elsif ($stage eq "warp") {
+}
+
+if ($stage eq "warp") {
     die "--stage_id required for stage warp\n" if !$stage_id;
     # this stage uses 'warptool'
@@ -295 +330 @@
         }
 
-        if ($status)  {
-            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
+        if ($status)  {
+            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedskyfile";
@@ -302 +337 @@
                 $command .= " -tocleanedskyfile";
             }
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
-            }
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+            }
          } else {
-            # XXX: -updateskyfile mode does not exist, need to add it
-            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1";
+            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
             $command .= " -dbname $dbname" if defined $dbname;
 
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
-            }
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+            }
             exit $PS_EXIT_UNKNOWN_ERROR;
-        }
+        }
     }
     exit 0;
@@ -329 +363 @@
 # left TODO
 # fake : faketool : -pendingcleanupimfile (loop over imfiles)
-# stack: stacktool : -pendingcleanupskyfile (loop over skyfiles)
-# diff:  difftool : -pendingcleanupskyfile
+# stack: stacktool : -pendingcleanupskyfile
+# diff:  difftool : -pendingcleanupskyfile (loop over skyfiles)
 
 die "ipp_cleanup.pl -stage $stage not yet implemented\n";
 
-sub delete_files 
+sub delete_files
 {
     my $files = shift; # reference to a list of files to unlink
-    
-    # this script is, of course, very dangerous.  
+
+    # this script is, of course, very dangerous.
     foreach my $file (@$files) {
-        print STDERR "unlinking $file\n";
+        print STDERR "unlinking $file\n";
         $ipprc->file_delete($file);
     }
@@ -346 +380 @@
 }
 
-sub addFilename 
+sub addFilename
 {
     my $files      = shift; # reference to a list of files to unlink
@@ -359 +393 @@
 }
 
+# XXX we currently do not set the error state in the db on my_die
 sub my_die
 {