Changeset 30586 for branches/czw_branch/20101203/ippScripts

Timestamp:

Feb 11, 2011, 4:29:48 PM (15 years ago)

Author:

watersc1

Message:

Trying to merge back into trunk.

Location:

branches/czw_branch/20101203

Files:

• . (modified) (1 prop)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/addstar_run.pl (modified) (4 diffs)
• ippScripts/scripts/dist_advancerun.pl (modified) (8 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (8 diffs)
• ippScripts/scripts/dist_component.pl (modified) (5 diffs)
• ippScripts/scripts/dist_defineruns.pl (modified) (4 diffs)
• ippScripts/scripts/dist_make_fileset.pl (modified) (3 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (3 diffs)
• ippScripts/scripts/ipp_cleanup.pl (modified) (4 diffs)
• ippScripts/scripts/magic_cleanup.pl (copied) (copied from trunk/ippScripts/scripts/magic_cleanup.pl )
• ippScripts/scripts/magic_destreak.pl (modified) (3 diffs)
• ippScripts/scripts/magic_destreak_cleanup.pl (modified) (2 diffs)
• ippScripts/scripts/magic_destreak_defineruns.pl (modified) (3 diffs)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (9 diffs)
• ippScripts/scripts/nightly_science.pl (modified) (21 diffs)
• ippScripts/scripts/publish_file.pl (modified) (4 diffs)
• ippScripts/scripts/register_exp.pl (modified) (1 diff)
• ippScripts/scripts/register_imfile.pl (modified) (19 diffs)

branches/czw_branch/20101203/ippScripts/Build.PL

@@ -58 +58 @@
         scripts/magic_tree.pl
         scripts/magic_process.pl
+        scripts/magic_cleanup.pl
         scripts/magic_destreak.pl
         scripts/magic_destreak_revert.pl

branches/czw_branch/20101203/ippScripts/scripts/addstar_run.pl

@@ -71 +71 @@
     defined $camera;
 if ($minidvodb && !defined($minidvodb_group)) {
-                my_die( "missing minidvodb_group", $add_id, 3 );
-            }
+                my_die( "missing minidvodb_group", $add_id, 3 );
+            }
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
@@ -113 +113 @@
 
 if (defined $dvodbReal) {
-        if ($minidvodb) {
-            
-            if (!defined($minidvodb_name)) {
-                #take the active one, if it's not defined on the command line
-                #reverts would have this already set, for example.
-                my $command = "addtool -listminidvodbrun ";
-                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
-                $command .= " -state 'active' -limit 1";
-                $command .= " -dbname $dbname" if defined $dbname;
-                print $command;
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
-                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
-
-                my $components = parse_md_list($metadata) or
-                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
-                my $comp = $$components[0];
-                $minidvodb_path = $comp->{minidvodb_path};
-                $minidvodb_name = $comp->{minidvodb_name};
-        
-                if (!defined($minidvodb_path)) {
-                    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
-                } 
-                if (!defined($minidvodb_name)) {
-                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
-                }
-            }
-            # tack on the minidvodb part to the db.
-#           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
-#we don't need this now that I fixed the paths     
-            
-        }
+        if ($minidvodb) {
+
+            #if (!defined($minidvodb_name)) {
+                #take the active one, if it's not defined on the command line
+                #reverts would have this already set, for example.
+                my $command = "addtool -listminidvodbrun ";
+                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
+                $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+                $command .= " -state 'active' -limit 1";
+                $command .= " -dbname $dbname" if defined $dbname;
+                print $command;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
+
+                my $components = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
+                my $comp = $$components[0];
+                $minidvodb_path = $comp->{minidvodb_path};
+                $minidvodb_name = $comp->{minidvodb_name};
+
+                if (!defined($minidvodb_path)) {
+                    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
+                }
+                if (!defined($minidvodb_name)) {
+                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
+                }
+            #}
+            # tack on the minidvodb part to the db.
+#           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
+#we don't need this now that I fixed the paths
+
+        }
         else {
 
@@ -153 +154 @@
 
 unless ($no_op) {
-            print $dvodbReal;
+            print $dvodbReal;
 
         ## addstar can either save the full set of detections, or just
@@ -222 +223 @@
         $command .= " -addprocessedexp";
         $command .= " -fault $exit_code";
-        $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
+        $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
         $command .= " -path_base $outroot" if defined $outroot;
         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+        $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);

branches/czw_branch/20101203/ippScripts/scripts/dist_advancerun.pl

@@ -23 +23 @@
 use PS::IPP::Config 1.01 qw( :standard );
 
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -29 +28 @@
 
 # Parse the command-line arguments
-my ($dist_id, $stage, $stage_id, $outdir, $clean);
+my ($dist_id, $stage, $stage_id, $outdir, $clean, $camera);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
-           'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'dist_id=s'      => \$dist_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -53 +53 @@
     defined $stage_id and
     defined $outdir;
+
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -134 +136 @@
 }
 
-# XXX should we create a file rule for this?
-my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $command = "$tool_cmd $exportarg -outfile $dbinfo_file";
+
+# work around the fact that $ipprc->file_create does not actually create a file on disk
+# unless the scheme is nebulous
+sub create_file {
+    my $rule = shift;
+    my $path_base = shift;
+    my $ref = shift;
+
+    my $file;
+
+    my $error;
+    $file = $ipprc->prepare_output($rule, $path_base, undef, 1, \$error)
+        or &my_die("Unable to prepare outut for $rule", $dist_id, $PS_EXIT_SYS_ERROR);
+
+    my $scheme = file_scheme($file);
+    $scheme = "" if !$scheme;
+
+    my $resolved;
+    if ($scheme) {
+        $ipprc->file_create($file) 
+            or &my_die("Unable to create $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        $resolved = $ipprc->file_resolve($file) 
+            or &my_die("Unable to resolve $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        if ($scheme eq 'neb') {
+            &my_die("$resolved not found", $dist_id, $PS_EXIT_SYS_ERROR) unless ($resolved and -e $resolved);
+    }
+    } else {
+        $resolved = $file;
+    }
+    $$ref = $resolved;
+
+    return $file
+}
+
+my $dbinfo_root = "$outdir/dbinfo.$stage.$stage_id";
+my $resolved;
+my $dbinfo_file = create_file("DIST.OUTPUT.DBINFO", $dbinfo_root, \$resolved);
+
+{
+    my $command = "$tool_cmd $exportarg -outfile $resolved";
     $command .= " -clean" if ((defined $clean) and ($stage ne "raw"));
     $command .= " -dbname $dbname" if defined $dbname;
@@ -149 +188 @@
     }
 }
-my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+
+my $dirinfo_root = "$outdir/dirinfo.$stage.$stage_id";
+my $dirinfo = create_file("DIST.OUTPUT.DIRINFO", $dirinfo_root, \$resolved);
+
 {
     my $command = "$tool_cmd $list_mode";
@@ -168 +210 @@
         &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
-    open MANIFEST, ">$dirinfo" or
-        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    open MANIFEST, ">$resolved" or
+        &my_die("Unable to open dirinfo file $resolved",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
     my $destdir;
@@ -197 +239 @@
         &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
         my $component_dir = find_componentdir($destdir, $path);
-#        print MANIFEST "$component METADATA\n";
         print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
     }
@@ -217 +258 @@
     }
 }
-
 
 exit 0;

branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl

@@ -16 +16 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename );
 use Digest::MD5::File qw( file_md5_hex );
@@ -71 +71 @@
 # Parse the command-line arguments
 my ($camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $alt_path_base, $clean);
-my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
+my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -88 +88 @@
            'magicked'       => \$magicked,   # magicked state for this component
            'alt_path_base=s'=> \$alt_path_base,  # path to alternate inputs
-           'outdir=s'       => \$outdir,     # "directory" for outputs
+           'outroot=s'      => \$outroot,    # outroot
            'prefix=s'       => \$prefix,     # "prefix" to apply to filenames
            'clean'          => \$clean,      # create clean distribution
@@ -99 +99 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outdir",
+pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outroot",
            -exitval => 3) unless
     defined $camera and
@@ -106 +106 @@
     defined $component and
     defined $path_base and
-    defined $outdir;
+    defined $outroot;
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -116 +116 @@
 $ipprc->define_camera($camera);
 
-# create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outdir, "neb://") != 0) {
-    if (! -e $outdir ) {
-        my $code = system "mkdir -p $outdir";
-        &my_die("cannot create output directory $outdir", $component,
-                $code >> 8) if $code;
-    }
-}
+$ipprc->outroot_prepare($outroot);
 
 # Get the list of data products for this component
@@ -138 +131 @@
 # set up directory for temporary files
 
-my $tmpdir  = "$outdir/tmpdir.$component.$$";
-if (-e $tmpdir) {
-    if (-d $tmpdir) {
-        my $rc = system "rm -r $tmpdir";
-        &my_die("cannot rm $tmpdir return code: $rc", $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
-    } else {
-        unlink $tmpdir or &my_die("cannot delete $tmpdir", $component, $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
-mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $component,
-                       $PS_EXIT_UNKNOWN_ERROR);
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+$temproot = "/tmp" if !defined $temproot;
+&my_die("directory for TEMP.DIR $temproot does not exist", $component, $PS_EXIT_CONFIG_ERROR) if ! -e $temproot;
+
+my $tmpdir  = tempdir("$temproot/dist.XXXX", CLEANUP => !$save_temps);
 
 #
@@ -349 +336 @@
 if ($num_files) {
     # create the tarfile
-    # XXX TODO: create a file rule for the tar file name
-    my $tbase = basename($path_base);
-    $tbase .= ".$component" if $component;
-    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
-    my $tarfile = "$outdir/$file_name";
+#    my $tbase = basename($path_base);
+#    $tbase .= ".$component" if $component;
+#    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
+    my $error;
+    my $output_tarfile;
+    my $rule;
+    if ($stage eq 'chip' or $stage eq 'chip_bg' or $stage eq 'raw') {
+        $rule = "DIST.OUTPUT.CHIP.BUNDLE";
+    } else {
+        $rule = "DIST.OUTPUT.BUNDLE";
+    }
+    $output_tarfile = $ipprc->prepare_output($rule, $outroot, $component, 1, \$error)
+            or &my_die("Failed to prepare output tarfile: $error", $component, $error);
+    $file_name = basename($output_tarfile);
+
+
+    my $scheme = file_scheme($output_tarfile);
+
+    my $tarfile;
+    if ($scheme) {
+        $ipprc->file_create($output_tarfile) 
+            or &my_die("Failed to create $output_tarfile: $error", $component, $error);
+        $tarfile = $ipprc->file_resolve($output_tarfile);
+
+        if ($scheme eq 'neb') {
+            &my_die("output file $output_tarfile not found", $component, $PS_EXIT_SYS_ERROR)
+                unless ($tarfile and -e $tarfile);
+        }
+    } else {
+        # no scheme the filename and the resolved filename are the same
+        $tarfile = $output_tarfile;
+    }
 
     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";

branches/czw_branch/20101203/ippScripts/scripts/dist_component.pl

@@ -21 +21 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-
 use PS::IPP::Config 1.01 qw( :standard );
-
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -77 +74 @@
     defined $outdir;
 
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 $ipprc->redirect_output($logfile) if $logfile;
+
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+
+$temproot = "/tmp" if !defined $temproot;
 
 if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
@@ -85 +87 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
+my ($rf, $rf_name) = tempfile("$temproot/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
 close $rf;
+
+my $basename = basename($path_base);
+my $outroot = "$outdir/$basename";
 
 my ($file_name, $bytes, $md5sum);
@@ -92 +97 @@
     my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id";
     $command .= " --results_file $rf_name";
-    $command .= " --component $component --path_base $path_base --outdir $outdir";
+    $command .= " --component $component --path_base $path_base --outroot $outroot";
     $command .= " --chip_path_base $chip_path_base" if $chip_path_base;
     $command .= " --state $run_state" if defined $run_state;
@@ -136 +141 @@
     &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum;
 }
+
 
 {

branches/czw_branch/20101203/ippScripts/scripts/dist_defineruns.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $dist_root, $no_magic);
+my ($stage, $stage_limit, $dist_root, $workdir, $no_magic);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -49 +49 @@
            'stage=s'        => \$stage,      # stage to queue
            'label=s'        => \@labels,     # labels
+           'workdir=s'      => \$workdir,     # workdir
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
-           'dist_root=s'    => \$dist_root,  # root of distribution work area
+#           'dist_root=s'    => \$dist_root,  # root of distribution work area
            'no_magic'       => \$no_magic,   # queue runs without requiring magic (for testing only)
            'dbname=s'       => \$dbname,     # Database name
@@ -66 +67 @@
 $ipprc->redirect_output($logfile) if $logfile;
 
-if (!$dist_root) {
-    $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
-    &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+if (!$workdir) {
+    print "workdir not supplied will use DISTRIBUTION_ROOT\n";
+    # old method where we set workdir based on a config file
+    if (!$dist_root) {
+        $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
+        &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+    }
+
+    my ($day, $month, $year) = (gmtime)[3,4,5];
+    my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+
+    $workdir = $dist_root . "/$datestr";
+
+    print "workdir is $workdir\n";
 }
-
-my ($day, $month, $year) = (gmtime)[3,4,5];
-my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
-
-my $workdir = "$dist_root/$datestr";
-
-print "workdir is $workdir\n";
 
 # if stage is not supplied as an argument, loop over all stages
@@ -90 +95 @@
         my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label";
         $command .= " -no_magic" if $no_magic;
-        $command .= " -dry_run" if $no_update;
+        $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
         $command .= " -set_label $label";

branches/czw_branch/20101203/ippScripts/scripts/dist_make_fileset.pl

@@ -96 +96 @@
 # make sure that the database info file for this run exists
 my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc";
-if (! -e "$dbinfo_file" ) {
+if (! $ipprc->file_exists($dbinfo_file) ) {
     &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -103 +103 @@
 # make sure that the dirinfo file for this run exists
 my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc";
-if (! -e "$dirinfo_file" ) {
+if (!$ipprc->file_exists($dirinfo_file)) {
     &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -277 +277 @@
     $command .= " -dbname $dbname" if $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        print STDERR "Unable to perform $command error_code: $error_code\n";
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            print STDERR "Unable to perform $command error_code: $error_code\n";
+        }
+    } else {
+        print STDERR "skipping $command\n";
     }
     exit $fault;

branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl

@@ -74 +74 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
 }
 
 my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to parse metadata config doc", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
 
 my $burntoolStateGood = 999;
@@ -87 +87 @@
 }
 if ($burntoolStateGood == 999) {
-    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
+    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
 }
 my $outState = -1 * abs($burntoolStateGood);
@@ -178 +178 @@
         my $exp_id = shift;
         my $class_id = shift;
-        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1);
+        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state 0 -set_state pending_burntool",1);
     }
     printf STDERR "$message\n";

branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl

@@ -167 +167 @@
 #            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
 #            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
+            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
+            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
@@ -429 +429 @@
 
                 # delete the temporary image datafiles
-                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
-                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
-                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
+                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id, 1);
+                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
+                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
 #            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
                 if ($mode eq "goto_purged") {
@@ -706 +706 @@
             my @files = ();
             # delete the temporary image datafiles
-            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
-
-            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
+            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
+
+            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id, 1);
 
             addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
@@ -2048 +2048 @@
     my $path_base  = shift; # base filename
     my $class_id   = shift; # class_id, if needed
+    my $recovery   = shift; # is there is a recovery file to clean?
 
     my $file = $ipprc->filename($filerule, $path_base, $class_id);
 
     push @$files, $file;
+    
+    if ($recovery) {
+        # need to clean up the recovery file (the pixels censored by streaksremove)
+        $file = $ipprc->recovery_filename($file);
+        push @$files, $file;
+    }
     return 1;
 }

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl

@@ -65 +65 @@
            'cam_reduction=s'=> \$cam_reduction,  # reduction class from camera stage (for chip and raw)
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
-           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'recoveryroot=s' => \$recoveryroot,# "prefix" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
            'magicked=s'     => \$magicked,   # magicked state of the run
@@ -190 +190 @@
 my $recovery_path_base;
 if ($recoveryroot) {
-    if (!inNebulous($recoveryroot)) {
-        if (! -e $recoveryroot ) {
-            my $code = system "mkdir -p $recoveryroot";
-            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
-                    $code >> 8) if $code;
-        }
-    }
-    # note: trailing / is necessary
-    $recovery_path_base = "$recoveryroot/$basename/";
+    # recoveryroot is a path to prepend to the basenames of the input files
+    if (inNebulous($recoveryroot)) {
+        # if recoveryroot is a nebulous path we ignore the actual path and put the files in the
+        # the same "directory" as the input files
+        $recoveryroot = "$dirname/REC_";
+        $recovery_path_base = $recoveryroot . $basename;
+    } else {
+        # otherwise we put the files in recoveryroot.
+        # Regardless, we prefix the basename with 'REC_'
+        $ipprc->outroot_prepare($recoveryroot);
+        $recovery_path_base = "$recoveryroot/REC_$basename";
+    }
 }
 
@@ -601 +604 @@
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -component $component";
+    $command .= " -backup_path_base $backup_path_base" if $backup_path_base;
+    $command .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
     $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -233 +233 @@
                 $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
                 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
-                $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+                # bills 2011-01-24
+                # don't clean up the uncensored sources file
+                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
             if ($recovery_path_base) {
@@ -251 +253 @@
                 $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
                 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
-                $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+                # bills 2011-01-24
+                # don't clean up the uncensored sources file
+                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
             if ($recovery_path_base) {

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $workdir);
+my ($stage, $stage_limit, $workdir, $recoveryroot);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -51 +51 @@
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
            'workdir=s'      => \$workdir,    # output destination
+           'recoveryroot=s' => \$recoveryroot, # recovery pixels destination
            'dbname=s'       => \$dbname,     # Database name
            'verbose'        => \$verbose,    # Print stuff?
@@ -78 +79 @@
     foreach my $label (@labels) {
         my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label";
+        $command .= " -recoveryroot $recoveryroot" if $recoveryroot;
         $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl

@@ -39 +39 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked, $run_state);
-my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
+my ($outroot, $recovery_path_base, $replace, $release, $bytes, $md5sum);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -53 +53 @@
            'cam_reduction=s'=> \$cam_reduction,  # reduction class of the associated camera run
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
-           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'recovery_path_base=s' => \$recovery_path_base,# "directory" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
            'bothways=s'     => \$bothways,   # run has inverse files (bothways diff)
@@ -130 +130 @@
 
 # default value is "NULL" do not use it
-if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
-    $recoveryroot = undef;
+if (defined($recovery_path_base) and ($recovery_path_base eq "NULL")) {
+    $recovery_path_base = undef;
 }
 
@@ -149 +149 @@
 }
 
-my $recovery_path_base;
-if ($recoveryroot) {
-    # note: trailing / is necessary
-    $recovery_path_base = "$recoveryroot/$basename/";
-}
-
 
 my ($image, $mask, $ch_mask, $weight, $sources, $astrom);
 my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
+my ($rimage, $rmask, $rch_mask, $rweight);
 
 if ($stage eq "raw") {
@@ -214 +209 @@
     $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
     $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
+
+    $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $class_id);
+    # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
+    $rmask   = dirname($recovery_path_base) . "/REC_" . basename($mask);
+    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $class_id);
+    $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $class_id);
 } elsif ($stage eq "camera") {
     $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
@@ -226 +227 @@
     $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
+    $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
+    $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
+    $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
 } elsif ($stage eq "diff") {
     my $name = "PPSUB.OUTPUT";
@@ -236 +240 @@
     $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename($name, $recovery_path_base);
+        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+    }
 }
 
 revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+if ($recovery_path_base) {
+    delete_recovery_files($rimage, $rmask, $rweight, $rch_mask);
+}
 
 if ($stage eq "diff" and $bothways) {
@@ -251 +263 @@
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename($name, $recovery_path_base);
+        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+        delete_recovery_files($rimage, $rmask, $rweight);
+    }
 }
 
@@ -426 +444 @@
 
 
-sub file_check
-{
-    my $file = shift;           # Name of file
-    &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
-        $ipprc->file_exists($file);
+sub delete_recovery_files
+{
+    foreach my $file (@_) {
+        if ($ipprc->file_exists($file)) {
+            if (!$ipprc->file_delete($file)) {
+                &my_die("failed to delete recovery file $file", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
+    }
 }
 

branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl

@@ -44 +44 @@
 my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual);
 my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
-my ( $registration_status, $burntool_status, $observing_status);
+my ( $registration_status, $burntool_status, $observing_status, $old_date);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old);
@@ -66 +66 @@
     'registraion_status'   => \$registration_status,
     'burntool_status'      => \$burntool_status,
+    'old_date=s'           => \$old_date,
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
@@ -134 +135 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $date, $PS_EXIT_SYS_ERROR);
 }
 
@@ -278 +279 @@
 
 $metadata_out{nsObservingState} = &get_observing_state($date);
+$metadata_out{nsRegistrationState} = &get_registration_state($date);
 #
 # Mode selection
@@ -396 +398 @@
     exit(0);
 }
-exit(10);
+
 if (defined($check_confirm_stacks) || defined($test_mode)) {
     $metadata_out{nsStackState} = 'CONFIRM_STACKING';
@@ -409 +411 @@
     exit(0);
 }
-
+exit(10);
 if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
@@ -1097 +1099 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    my $is_processing = 0;
+    my $is_registering;
+    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
+        $is_registering = 0;
+    }
+    else {
+        $is_registering = 1;
+    }
+
     foreach my $target (sort (keys %science_config)) {
         if ($science_config{$target}{STACKABLE} == 1) {
@@ -1105 +1118 @@
                         print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
                     }
+                    $is_processing = 1;
                     $metadata_out{nsStackState} = 'FORCETOWARP';
                     next;
@@ -1120 +1134 @@
                     next;
                 }
+                $Npotential++;
                 if ($Nalready != 0) {
+                    $Nqueued++;
                     if ($debug == 1) {
                         print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
@@ -1137 +1153 @@
                     }
                     stack_queue($date,$target,$filter);
+                    $Nqueued ++;
                 }
                 if (defined($pretend)) {
@@ -1152 +1169 @@
         }
     }
+    $metadata_out{nsStackPotential} = $Npotential;
+    $metadata_out{nsStackQueued}    = $Nqueued;
+    if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)&&($is_registering == 0)) {
+        $metadata_out{nsStackState} = 'FINISHED_STACKS';
+    }   
 }
 
@@ -1231 +1253 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     my $data_ref = $db->selectall_arrayref( $sth );
-
-    return($#{ $data_ref } + 1);
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
 }
 
@@ -1272 +1301 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nnoexp     = 0;
+    my $is_processing = 0;
+    my $is_registering;
+    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
+        $is_registering = 0;
+    }
+    else {
+        $is_registering = 1;
+    }
+
     foreach my $target (sort (keys %science_config)) {
         if ($science_config{$target}{DIFFABLE} == 1) {
             foreach my $filter (@filter_list) {
+                $Npotential ++;
                 my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
                 if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
@@ -1280 +1321 @@
                         print STDERR "execute_diffs: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
                     }
-                    $metadata_out{nsDiffState} = 'FORCETOWARP';
-                    next;
-                }
+                    $is_processing = 1;
+                }
                 if ($NprocChips == 0) {
+                    $Nnoexp ++;
                     if ($debug == 1) {
                         print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n";
@@ -1311 +1352 @@
         }
     }
+    if ($debug == 1) {
+        print "$metadata_out{nsObservingState} $metadata_out{nsDiffPotential} $metadata_out{nsDiffQueued} $is_processing $is_registering\n";
+    }
+
+    if ($metadata_out{nsObservingState} eq 'END_OF_NIGHT') {
+        if ($is_processing == 1) {
+            $metadata_out{nsDiffState} = 'DIFFING';
+        }
+        elsif ($is_registering == 0) {
+            if ($Npotential == $Nnoexp) {
+                $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+            }
+            elsif ($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued}) {
+                $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+            }
+        }
+        else {
+            $metadata_out{nsDiffState} = 'DIFFING';
+        }
+    }
+    else {
+        $metadata_out{nsDiffState} = 'DIFFING';
+    }
+
+#     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#     } 
+
+#     if ($is_processing == 1) {
+#       $metadata_out{nsDiffState} = 'DIFFING';
+#     }
 }
 
@@ -1326 +1398 @@
 
     my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
     
     foreach my $object_row (@{ $object_ref }) {
         my $this_object = shift @{ $object_row };
-        my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
-        $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
-        $input_sth .= " ORDER BY dateobs ";
+#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+#       $input_sth .= " ORDER BY dateobs ";
         
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
         my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            $comment_hash{$this_comment} = $this_exp_id;
+        }
         
         if (($#{ $warps } + 1) % 2 != 0) {
-            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
-            next;
+            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                print STDERR ": I should declare an exposure to be faulty.\n";
+                my @keep_warps = ();
+#               print "@{ $warps }\n";
+                foreach my $this_warp (@{ $warps }) {
+                    my $this_comment = ${ $this_warp }[3];
+                    my $this_exp_id  = ${ $this_warp }[0];
+                    if ($comment_hash{$this_comment} == $this_exp_id) {
+                        push @keep_warps, $this_warp;
+                    }
+                    else {
+                        print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
+                    }
+                }
+                @{ $warps } = @keep_warps;
+#               print "@{ $warps }\n";
+            }
         }
         
         while ($#{ $warps } > -1) {
             my $input_warp = shift @{ $warps };
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            
             my $template_warp = shift @{ $warps };
-            my $input_exp_id = ${ $input_warp }[0];
+
             my $template_exp_id = ${ $template_warp }[0];
             
             my $input_warp_id = ${ $input_warp }[1];
             my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
             
+            $Npotential++;
+
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed\n";
+                next;
+            }
+
             if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
-                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
+                $Nqueued++;
+                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
                 next;
             }
@@ -1366 +1490 @@
                 $cmd .= ' -pretend ';
                 print STDERR "diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
             }
             
@@ -1375 +1500 @@
                     &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
                 }
-            }
-        }
-    }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }        
+
 }
             
@@ -1493 +1625 @@
         }
     }   
-
+#    print "$datetime $eon_dt\n";
     if (DateTime->compare($datetime,$eon_dt) < 1) {
         return("OBSERVING");
@@ -1501 +1633 @@
     }
 }
+
+# This basically does the end of night check, but does it "the hard way," to prevent the time from fooling us.
+sub get_registration_state {
+    my $date = shift;
+
+    foreach my $eon (keys %eon_config) {
+        my $command = "$regtool -processedexp -simple ";
+        $command .= " -dbname $dbname ";
+        $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
+        $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT});
+        $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE});
+        $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE});
+        $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT});
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run ( command => $command, verbose => $verbose );
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+        }
+        my @eon_exposures = split /\n/, (join '', @$stdout_buf);
+        if ($#eon_exposures >= 0) {
+            return("REGISTERED");
+        }
+    }   
+    return("NOT_REGISTERED");
+}
+
+    
 
 sub get_tool_parameters {
@@ -1620 +1781 @@
     my $stage = shift; # stage name
     my $stage_id = shift; #  identifier
+    my $date = shift;
     my $exit_code = shift; # Exit code
-    # outputImage and path_base are globals
 
     carp($msg);

branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
+my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir, $need_magic );
 my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
 my ( $output_format );
@@ -54 +54 @@
     'fileset=s'         => \$fileset,     # Fileset name
     'workdir=s'         => \$workdir,     # Working directory
+    'need-magic'        => \$need_magic,  # do we require censored detections?
     'dbname=s'          => \$dbname,    # Database name
     'verbose'           => \$verbose,   # Print to stdout
@@ -166 +167 @@
     foreach my $comp ( @$components ) {
         my $path_base = $comp->{path_base}; # Base name for file
+        if (!$need_magic and $comp->{magicked}) {
+            # This client is authorized to receive uncensored detections
+            # Get the uri for the "backup" files
+            print "Using uncensored input from $path_base\n";
+            $path_base = $ipprc->destreaked_filename($path_base);
+        }
         next if defined $comp->{quality} and $comp->{quality} > 0;
         print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment};
@@ -181 +188 @@
             $filename = $ipprc->filename( "PSPHOT.OUT.CMF.MEF", "$path_base.pos" );
         }
-        $filename = $ipprc->file_resolve($filename);
+        &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename);
+        my $resolved = $ipprc->file_resolve($filename);
+        
+        &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved;
+
+        $filename = $resolved;
 
         my $data = { zp => $zp,

branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl

@@ -348 +348 @@
     if (defined $exp_id and not $no_update) {
         my $command = "$regtool -addprocessedexp -exp_id $exp_id -fault $exit_code";
+        $command .= " -exp_tag $exp_tag";
+        $command .= " -exp_name UNKNOWN";
+        $command .= " -inst UNKNOWN";
+        $command .= " -telescope UNKNOWN";
+        $command .= " -telescope UNKNOWN";
+        $command .= " -filelevel UNKNOWN";
         $command .= " -hostname $host" if defined $host;
         $command .= " -dbname $dbname" if defined $dbname;

branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl

@@ -10 +10 @@
 my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host at $date\n\n";
+my $cmd_line = join ' ', @ARGV;
+print "Starting script $0 $cmd_line on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -39 +40 @@
 
 my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile);
+my ($sunrise, $sunset);
 GetOptions(
     'caches'           => \$cache,
@@ -47 +49 @@
     'bytes=s'          => \$bytes,
     'md5sum=s'         => \$md5sum,
+    'sunrise=s'        => \$sunrise,
+    'sunset=s'         => \$sunset,
     'dbname|d=s'       => \$dbname,    # Database name
     'verbose'          => \$verbose,   # Print to stdout
@@ -54 +58 @@
 ) or pod2usage( 2 );
 
-my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
+my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -65 +69 @@
     defined $uri;
 
+unless (defined($sunset)) {
+    $sunset = '03:30:00';
+}
+unless (defined($sunrise)) {
+    $sunrise = '17:30:00';
+}
 
 my $RECIPE = "REGISTER"; # Recipe to use for ppStats
@@ -84 +94 @@
 
 my $cmdflags;
-my $burntoolStateGood;
+my $burntoolStateTarget;
+my $burntoolStateCurrent;
 # Run ppStats on the input file
 {
@@ -101 +112 @@
     print "STDOUT:\n$out1";
     print "STDERR:\n$err1";
-    &my_die("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
+    &my_die_for_add("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
 
     print "[Running " . join(' ', @command2) . "]\n";
@@ -110 +121 @@
     print "STDOUT:\n$out2";
     print "STDERR:\n$err2";
-    &my_die("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
+    &my_die_for_add("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
     chomp $out2;
     $cmdflags = $out2;
 
     # Manually parse the burntool_state entry.
-    my $burntoolState = 0;
+    $burntoolStateCurrent = 0;
     my $isGPC1 = 0;
-    $burntoolStateGood = 0;
+    $burntoolStateTarget = 0;
     foreach my $line (split /\n/, $out1) {
         if ($line =~ /FPA.BURNTOOL.APPLIED/) {
             $line =~ s/^\s+//;
-            $burntoolState = (split /\s+/, $line)[2];
+            $burntoolStateCurrent = (split /\s+/, $line)[2];
         }
         if ($line =~ /FPA.CAMERA/) {
@@ -131 +142 @@
     }
     if ($isGPC1 != 1) {
-        $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool.
-    }
-    elsif (($isGPC1 == 1) && ($burntoolState == 1)) {
-#       print STDERR "In the good region: >>$burntoolState<<\n";
+        $burntoolStateCurrent = 0; # If it's not GPC1, you shouldn't have run burntool.
+    }
+    elsif (($isGPC1 == 1) && ($burntoolStateCurrent == 1)) {
+#       print STDERR "In the good region: >>$burntoolStateCurrent<<\n";
         my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -148 +159 @@
             if ($line =~ /BURNTOOL.STATE.GOOD/) {
                 $line =~ s/^\s+//;
-                $burntoolStateGood = (split /\s+/, $line)[2];
+                $burntoolStateTarget = (split /\s+/, $line)[2];
                 last;
             }
         }
-        $burntoolState = $burntoolStateGood; # Positive because this has the header table.
-
-    }
-    $cmdflags .= " -burntool_state $burntoolState ";
+        # XXX why was this being equated??
+        # $burntoolState = $burntoolStateGood; # Positive because this has the header table.
+
+    }
+    $cmdflags .= " -burntool_state $burntoolStateTarget ";
 }
 
 $now_time = localtime();
 printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose;
+printf STDERR "\nburntool state current: %d target: %d\n", $burntoolStateCurrent, $burntoolStateTarget;
 
 # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
-if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die_for_add ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die_for_add ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die_for_add ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die_for_add ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die_for_add ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
 
 my $command = "$regtool -addprocessedimfile";
@@ -177 +191 @@
 $command .= " -hostname $host" if defined $host;
 $command .= " -dbname $dbname" if defined $dbname;
-$command .= " -data_state check_burntool";
+if (abs($burntoolStateCurrent) == $burntoolStateTarget) {
+    printf STDERR "This has already been burned.\n";
+    $command .= " -data_state full";
+}
+elsif (is_daytime($dateobs,$sunrise,$sunset)) {
+    printf STDERR "This is a daytime exposure.\n";
+    $command .= " -data_state full";
+}
+else {
+    printf STDERR "Need to check burntool.\n";
+    $command .= " -data_state check_burntool";
+}
 $command .= " $cmdflags";
 
@@ -187 +212 @@
 my $ra        = &value_for_flag($cmdflags, 0.0, "-ra");
 my $dec       = &value_for_flag($cmdflags, 0.0, "-decl");
-my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
+
 
 # if the needed data is available, pass it to sunmoon:
@@ -216 +241 @@
 # lock file?
 # if exp_type = DARK and date > MIDNIGHT HST { wait }
-# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); 
+# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log");
 
 $now_time = localtime();
@@ -236 +261 @@
 # We now have an imfile in the database, check if we can burntool it.  If not, continue on.
 
-{ 
+if ((abs($burntoolStateCurrent) != $burntoolStateTarget)&&
+    (!is_daytime($dateobs,$sunrise,$sunset))) {
     my $mdcParser  = PS::IPP::Metadata::Config->new;
 
@@ -243 +269 @@
     $check_date =~ s/T.*$//;
     my $exp_name   = $tmp_exp_name;
-    
+
 
     my $bt_check_command = "$regtool -checkburntoolimfile ";
     $bt_check_command .= " -class_id $class_id ";
-    $bt_check_command .= " -date $check_date ";
-    $bt_check_command .= " -valid_burntool $burntoolStateGood ";
+    $bt_check_command .= " -dateobs_begin ${check_date}T${sunset} ";
+    $bt_check_command .= " -dateobs_end ${check_date}T${sunrise} ";
+    $bt_check_command .= " -valid_burntool $burntoolStateTarget ";
     $bt_check_command .= " -exp_name $exp_name ";
     $bt_check_command .= " -dbname $dbname" if defined $dbname;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run ( command => $bt_check_command, verbose => $verbose);
+        run ( command => $bt_check_command, verbose => $verbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform regtool: $error_code", 
-                $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR);
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die_for_update("Unable to perform regtool: $error_code",
+                           $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
     }
 
@@ -264 +291 @@
     my $exposures;
     while ( scalar @whole > 0 ) {
-        my $value = shift @whole;
-        push @single, $value;
-        if ($value =~ /^\s*END\s*$/) {
-            push @single, "\n";
-            
-            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
-            &my_die("Unable to parse output from regtool", 
-                    $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless
-                        defined $list;
-            push @{ $exposures }, @$list;
-            @single = ();
-        }
-    }
-    
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
+            &my_die_for_update("Unable to parse output from regtool",
+                               $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless
+                                   defined $list;
+            push @{ $exposures }, @$list;
+            @single = ();
+        }
+    }
+
     # We only care about the final entry, as that contains the exposure we are.
-    
+
     my $regtool_update = "$regtool -updateprocessedimfile ";
     $regtool_update .= "-dbname $dbname " if defined $dbname;
@@ -286 +313 @@
     my $burntool_data = pop(@{ $exposures });
     if ($burntool_data->{burnable} == 0) {
-        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
-        unless ($no_update) {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
-                exit($error_code);
-            }
-        } else {
-            print "skipping command: $command\n";
-        }
+        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
+                exit($error_code);
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
     }
     else {
-        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
-        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
-        $apply_command .= " --this_uri $burntool_data->{uri} ";
-        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
-        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
-        $apply_command .= " --verbose " if $verbose;
-        print "$apply_command\n";
-        unless ($no_update) {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
-                exit($error_code);
-            }
-        }
-    }    
+        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
+        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
+        $apply_command .= " --this_uri $burntool_data->{uri} ";
+        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
+        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
+        $apply_command .= " --verbose " if $verbose;
+        print "$apply_command\n";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die_for_update("Unable to perform ipp_apply_burntool_single.pl: $error_code",
+                               $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+        }
+    }
 
 
@@ -353 +380 @@
 }
 
-sub my_die
+sub is_daytime
+{
+    my $dateobs = shift;
+    my $sunset  = shift;
+    my $sunrise = shift;
+
+    my ($date,$time);
+
+    if ($dateobs =~ /T/) {
+        ($date,$time) = split /T/, $dateobs;
+    }
+    else {
+        ($date,$time) = split / /, $dateobs;
+    }
+    my ($hour,$minute,$second) = split /\:/, $time; # /;
+    my ($ss_hour,$ss_minute,$ss_second) = split /\:/, $sunset; # /;
+    my ($sr_hour,$sr_minute,$sr_second) = split /\:/, $sunrise; # /;
+    if ($second =~ /Z/) {
+        $second =~ s/Z//;
+    }
+#     print "this exposure: $hour $minute $second\n";
+#     print "sunset:        $ss_hour $ss_minute $ss_second\n";
+#     print "sunrise:       $sr_hour $sr_minute $sr_second\n";
+#     printf "Hss: %d Mss: %d Sss: %d\n",($hour >= $ss_hour),($minute >= $ss_minute),($second >= $ss_second);
+#     printf "Hsr: %d Msr: %d Ssr: %d\n",($hour <= $sr_hour),($minute <= $sr_minute),($second <= $sr_second);
+
+    if (($hour > $ss_hour)&&($hour <= 24)) {
+        return(1); # After sunset by more than an hour, before midnight
+    }
+    elsif ($hour == $ss_hour) {
+        if ($minute > $ss_minute) {
+            return(1); # After sunset by more than a minute
+        }
+        elsif ($minute == $ss_minute) {
+            if ($second >= $ss_second) {
+                return(1); # After sunset by more than a second
+            }
+            else {
+                return(0);
+            }
+        }
+        else {
+            return(0);
+        }
+    }
+    elsif (($hour < $sr_hour)&&($hour >= 0)) {
+        return(1); # Before sunrise by more than an hour, but after midnight
+    }
+    elsif ($hour == $sr_hour) {
+        if ($minute < $sr_minute) {
+            return(1); # Before sunrise by more than a minute
+        }
+        elsif ($minute == $sr_minute) {
+            if ($second <= $sr_second) {
+                return(1); # Before sunrise by more than a second
+            }
+            else {
+                return(0);
+            }
+        }
+        else {
+            return(0);
+        }
+    }
+    else {
+        return(0); # We should never get here.
+    }
+}
+
+sub my_die_for_add
 {
     my $msg = shift; # Warning message on die
@@ -384 +480 @@
     exit $exit_code;
 }
+sub my_die_for_update
+{
+    my $msg = shift; # Warning message on die
+    my $exp_id = shift;
+    my $class_id = shift;
+    my $exit_code = shift;
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp($msg);
+    if (defined $exp_id && defined $tmp_class_id and not $no_update) {
+        my $command = "$regtool -updateprocessedimfile";
+        $command .= " -exp_id $exp_id";
+        $command .= " -class_id $class_id";
+        $command .= " -fault $exit_code";
+        $command .= " -hostname $host" if defined $host;
+        $command .= " -dbname $dbname" if defined $dbname;
+        print "Running: $command\n";
+        system($command);
+    }
+    exit $exit_code;
+}
 
 # Pau.