Changeset 28017 for branches/czw_branch/20100427/ippScripts/scripts

Timestamp:

May 18, 2010, 5:06:01 PM (16 years ago)

Author:

watersc1

Message:

Almost working version of the mask-stats/software revision code.

Almost.

Location:

branches/czw_branch/20100427

Files:

• . (modified) (1 prop)
• ippScripts/scripts/automate_stacks.pl (modified) (16 diffs)
• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/diff_skycell.pl (modified) (1 diff)
• ippScripts/scripts/ipp_cleanup.pl (modified) (5 diffs)
• ippScripts/scripts/magic_destreak.pl (modified) (8 diffs)
• ippScripts/scripts/magic_destreak_cleanup.pl (modified) (4 diffs)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (8 diffs)
• ippScripts/scripts/magic_process.pl (modified) (4 diffs)
• ippScripts/scripts/make_burntool_pcontrol.pl (modified) (11 diffs)
• ippScripts/scripts/receive_advance.pl (modified) (2 diffs)
• ippScripts/scripts/register_exp.pl (modified) (1 diff)
• ippScripts/scripts/register_imfile.pl (modified) (1 diff)
• ippScripts/scripts/staticsky.pl (copied) (copied from trunk/ippScripts/scripts/staticsky.pl )

branches/czw_branch/20100427/ippScripts/scripts/automate_stacks.pl

@@ -56 +56 @@
 # Grab options
 my ( $date, $camera, $dbname, $logfile, $verbose, $manual);
-my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only);
+my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $check_mode);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
@@ -70 +70 @@
     'isburning'            => \$isburning,
     'force_stack_count'    => \$force_stack_count,
+    'check'                => \$check_mode,
     'test_mode'            => \$test_mode,
     'this_target_only=s'   => \$this_target_only,
@@ -116 +117 @@
     defined $queue_chips or defined $queue_stacks or
     defined $check_chips or defined $check_stacks or
-    defined $test_mode or defined $clean_old;
+    defined $test_mode or defined $clean_old or defined $check_mode;
 
 # Configurable parameters from our config file.
@@ -127 +128 @@
 my %stackable_list = ();
 my %reduction_class = ();
-my $retention_time = 9000;
-
+my %clean_commands = ();
+my %clean_retention = ();
+my %noclean_list = ();
 # Grab the configuration data.
 my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
@@ -141 +143 @@
 my $metadata = $mdcParser->parse(join "", @$stdout_buf);
 foreach my $entry (@{ $metadata }) {
-    if (${ $entry }{name} eq 'RETENTION_TIME') {
-        $retention_time = ${ $entry }{value};
-    }
+    if (@{ $entry }{name} eq 'CLEAN_MODES') {
+        my @mode_data = @{ ${ $entry }{value} };
+        my $this_mode = '';
+        foreach my $mentry (@mode_data) {
+            if (${ $mentry }{name} eq 'MODE') {
+                $this_mode = ${ $mentry }{value};
+            }
+            elsif (${ $mentry }{name} eq 'COMMAND') {
+                $clean_commands{$this_mode} = ${ $mentry }{value};
+            }
+            elsif (${ $mentry }{name} eq 'RETENTION_TIME') {
+                $clean_retention{$this_mode} = ${ $mentry }{value};
+            }
+        }
+    }
+#    if (${ $entry }{name} eq 'RETENTION_TIME') {
+#        $retention_time = ${ $entry }{value};
+#    }
     elsif (${ $entry }{name} eq 'FILTERS') {
         push @filter_list, ${ $entry }{value};
@@ -173 +190 @@
                 $reduction_class{$this_target} = ${ $tentry }{value};
             }
+            elsif (${ $tentry }{name} eq 'NOCLEAN') {
+                $noclean_list{$this_target} = ${ $tentry }{value};
+            }
         }
     }
@@ -218 +238 @@
 if (defined($test_mode)) {
     $debug = 1;
-}
-
-if (defined($check_registration) || defined($test_mode)) {
+    my $z;
+    foreach $z (@target_list) {
+        print "TARGET: $z $tessID_list{$z} $obsmode_list{$z} $object_list{$z} $comment_list{$z} $stackable_list{$z} $reduction_class{$z}\n";
+    }
+    foreach $z (@filter_list) {
+        print "FILTER: $z\n";
+    }
+    foreach $z (keys (%clean_commands)) {
+        print "CLEAN: $z $clean_commands{$z} $clean_retention{$z}\n";
+    }
+    
+}
+
+if (defined($check_registration) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'NEW';
     my ($Nsummit_exp,$Nfaults) = check_summit_copy($date);
     return_metadata($date);
-    unless (defined($test_mode)) { exit(0); }
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
 }
 if (defined($define_burntool) || defined($test_mode)) {
@@ -237 +268 @@
     unless (defined($test_mode)) { exit(0); }
 }
-if (defined($check_chips) || defined($test_mode)) {
+if (defined($check_chips) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'QUEUECHIPS';
     &execute_chips($date,"pretend");
@@ -244 +275 @@
     }
     return_metadata($date);
-    unless (defined($test_mode)) { exit(0); }
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
 }
 if (defined($queue_chips)) {
@@ -252 +283 @@
     exit(0);
 }
-if (defined($check_stacks) || defined($test_mode)) {
+if (defined($check_stacks) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'TOWARP';
     &execute_stacks($date,"pretend");
@@ -259 +290 @@
     }
     return_metadata($date);
-    unless (defined($test_mode)) { exit(0); }
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
 }
 if (defined($queue_stacks)) {
     $metadata_out{nsState} = 'STACKING';
     &execute_stacks($date);
-    if ($metadata_out{nsState} == 'QUEUESTACKING') {
+    if ($metadata_out{nsState} eq 'QUEUESTACKING') {
         $metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR';
     }
@@ -424 +455 @@
     }
 
-    if ($#{ $stderr_buf } > -1) {
-        $metadata_out{nsState} = 'ERROR';
-        return(1);
-    }
+#    if ($#{ $stderr_buf } > -1) {
+#        $metadata_out{nsState} = 'ERROR';
+#        return(1);
+#    }
 
     my $N = 0;
@@ -433 +464 @@
     foreach my $row (@burntool_entries) {
         my ($trash,$start,$end);
-        ($trash,$trash,$start,$end) = (split /\s+/,$row);
-
+        (undef,$trash,$start,$end) = (split /\s+/,$row);
+        if ($trash ne 'burntool') {
+            $metadata_out{nsState} = 'ERROR';
+            return(1);
+        }
         for (my $class_counter = 0; $class_counter < 60; $class_counter++) {
             $metadata_out{"bt${N}Begin"} = $start;
@@ -731 +765 @@
     my $date = shift;
     my $target = shift;
+    my $mode = shift;
+
+    my $command = $clean_commands{$mode};
+    my $retention_time = $clean_retention{$mode};
 
     my ($year,$month,$day) = split /-/,$date;
@@ -740 +778 @@
 
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
-
-    my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group ";
+    my $args = $command;
+    $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     if ($debug == 1) {
         $args .= ' -pretend ';
@@ -752 +790 @@
     my $pretend = shift;
 
-    foreach my $target (@target_list) {
-        my ($cleaning_date,$args) = construct_cleantool_args($date,$target);
-
-        my $command = $chiptool . $args;
-        print STDERR "$command\n";
-        if (!(defined($pretend) || $debug == 1)) {
+    foreach my $mode (keys (%clean_commands)) {
+        foreach my $target (@target_list) {
+            if (exists($noclean_list{$target})) {
+                next;
+            }
+            my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
+
+            print STDERR "$command\n";
+            if (!(defined($pretend) || $debug == 1)) {
 #           print STDERR "BEAR IS DRIVING!?\n";
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run ( command => $command, verbose => $verbose );
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-            }
-        }
-        $command = $warptool . $args;
-        print STDERR "$command\n";
-        if (!(defined($pretend) || $debug == 1)) {
-#           print STDERR "BEAR IS DRIVING $pretend $debug!?\n";
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run ( command => $command, verbose => $verbose );
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-            }
-        }
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run ( command => $command, verbose => $verbose );
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                }
+            }
+        }
     }
     return(0);

branches/czw_branch/20100427/ippScripts/scripts/camera_exp.pl

@@ -280 +280 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            print STDERR (join "\n", @$stderr_buf);
             &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code);
         }

branches/czw_branch/20100427/ippScripts/scripts/diff_skycell.pl

@@ -399 +399 @@
             $command .= " -hostname $host" if defined $host;
             $command .= " -path_base $outroot" if defined $outroot;
-            $command .= " -dbname $dbname" if defined $dbname;
         } else {
             $command .= " -updatediffskyfile";
         }
+        $command .= " -dbname $dbname" if defined $dbname;
         run(command => $command, verbose => $verbose);
     }

branches/czw_branch/20100427/ippScripts/scripts/ipp_cleanup.pl

@@ -192 +192 @@
             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
             $command .= " -dbname $dbname" if defined $dbname;
             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -309 +309 @@
         # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
         my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
         $command .= " -dbname $dbname" if defined $dbname;
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -447 +447 @@
             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
             $command .= " -dbname $dbname" if defined $dbname;
             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -581 +581 @@
             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
             $command .= " -dbname $dbname" if defined $dbname;
             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -731 +731 @@
             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
             $command .= " -dbname $dbname" if defined $dbname;
             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

branches/czw_branch/20100427/ippScripts/scripts/magic_destreak.pl

@@ -42 +42 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
-my ($outroot, $recoveryroot);
+my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
@@ -62 +62 @@
            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
+           'magicked=s'     => \$magicked,   # magicked state of the run
            'release'        => \$release,    # NAN masked pixels for release
            'save-temps'     => \$save_temps, # Save temporary files?
@@ -72 +73 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot",
+pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot --magicked",
            -exitval => 3) unless
     defined $magic_ds_id and
@@ -83 +84 @@
     defined $uri and
     defined $path_base and
-    defined $outroot;
+    defined $outroot and
+    defined $magicked;
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
@@ -210 +212 @@
                     &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
 
-            $temp_dir = tempdir( CLEANUP => !$save_temps);
             ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
 
@@ -219 +220 @@
                 } else {
                     # diff run must have been cleaned up, need to create this skycell file on the fly
+                    if (!defined $temp_dir ) {
+                        $temp_dir = tempdir( CLEANUP => !$save_temps);
+                    }
                     my $skycell_id = $skycell->{skycell_id};
                     $skycell_uri = "$temp_dir/$skycell_id";
@@ -248 +252 @@
             my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
+                    # note verbose == 0 to avoid polluting log files with almost always useless information
+                run(command => $command, verbose => 0);
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -267 +272 @@
         $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
 
-        if ($dynamicMasks) {
+        # if magicked is non-zero we are updating a previously magicked component. In this case we don't
+        # touch the camera mask
+        if (!$magicked and $dynamicMasks) {
             $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
             $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);

branches/czw_branch/20100427/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -63 +63 @@
 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 
-if (0) {
-my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
-&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer;
-
-my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
-if ($@ or not defined $nebulous) {
-    &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR);
-}
-}
 
 $dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname;
@@ -108 +99 @@
 
 
+&my_die("cleanup not supported for camera stage", $magic_ds_id, $PS_EXIT_PROG_ERROR) if $stage eq "camera";
+
+
 &my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_cleaned";
 &my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw";
@@ -130 +124 @@
 $dbh->disconnect() or warn $dbh->errstr;
 
+# We no longer clean up the camera stage mask files, but the code was left in place in case
+# we change our minds.
+my $cleanup_cam_mask = 0;
+
 my $dynamicMasks;               # Use dynamic masks?
 foreach my $comp (@components) {
@@ -140 +138 @@
         if ($stage eq "chip") {
             # Check to see if we're using dynamic masks
-            if (!defined $dynamicMasks) {
+            if ($cleanup_cam_mask && !defined $dynamicMasks) {
                 # Get the PSASTRO recipe
                 my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";

branches/czw_branch/20100427/ippScripts/scripts/magic_destreak_revert.pl

@@ -38 +38 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction);
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked);
 my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
@@ -55 +55 @@
            'replace=s'      => \$replace,    # replace the input images with the results.
            'bothways=s'     => \$bothways,   # run has inverse files (bothways diff)
+           'magicked=s'     => \$magicked,   # magicked state of the run
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -64 +65 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot",
+pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot --magicked",
            -exitval => 3) unless
     defined $magic_ds_id and
@@ -72 +73 @@
     defined $component and
     defined $path_base and
+    defined $magicked and
     defined $outroot;
 
@@ -77 +79 @@
 
 &my_die("cam_path_base is required for chip stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'chip' and !$cam_path_base);
+
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
@@ -169 +173 @@
         my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
+            # note verbose == 0 to keep from polluting log file with lots of usually useless information
+            run(command => $command, verbose => 0);
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -182 +187 @@
     }
 
+    $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
+    $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+
     # we use the mask output from the camera stage for input and replace
     # the output of the chip stage with that mask as well.
-    $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
-    $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
-
-    if ($dynamicMasks) {
+    # Note that when destreaking as part of an update $magicked is non-zero.
+    # In this case we do not touch the camera stage mask so there is no need to revert it
+    if (!$magicked and $dynamicMasks) {
         $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
         $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
@@ -195 +202 @@
 
     $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id);
-    # This is kludgey but correct
+    # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
     $bmask   = dirname($backup_path_base) . "/SR_" . basename($mask);
     $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);

branches/czw_branch/20100427/ippScripts/scripts/magic_process.pl

@@ -99 +99 @@
     }
 
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+    $inputs = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR);
-
-    $inputs = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR);
 }
 
 
 my @outputs;
+my $inverse;                    # Using inverse diff?
 ### Do the heavy lifting
 {
@@ -263 +261 @@
                 cat_list_to_list($mfh, $in_path_base, "mask.list");
                 cat_list_to_list($wfh, $in_path_base, "weight.list");
+
+                if ($innode->{inverse}) {
+                    $inverse = 1;
+                }
             }
             close $in_fh;
@@ -330 +332 @@
     }
 }
+
 if ($node eq "root") {
     my $streaks_file = "$outroot.streaks";
@@ -348 +351 @@
     &run_verifystreaks($baseroot);
 
-    my $command = "$magictool -addmask";
-    $command   .= " -magic_id $magic_id";
-    $command   .= " -uri $streaks_file";
-    $command   .= " -streaks $num_streaks";
-    $command   .= " -dbname $dbname" if defined $dbname;
-
-    # Add the processed file to the database
-    unless ($no_update) {
+    my $exp_id;                 # Exposure identifier
+    my $cam_path;               # Camera stage path_base
+    {
+        my $command = "magictool -exposure -magic_id $magic_id";
+        $command .= " -inverse" if defined $inverse;
+        $command .= " -dbname $dbname" if defined $dbname;
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            # This isn't going to work because our result was already entered.
-            &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
-        }
-    } else {
-        print "Skipping command: $command\n";
+        my $exposures = $mdcParser->parse_list(join "", @$stdout_buf);
+        my $exp = $$exposures[0]; # Exposure of interest (should only be one)
+
+        $exp_id = $exp->{exp_id};
+        $cam_path = $exp->{path_base};
+    }
+
+    {
+        my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
+        my $streaks = "$outroot.streaks";                           # Streaks file
+        my $clusters = "$baseroot.verify/${exp_id}_clusterPos.txt"; # Clusters file
+
+        my $command = "ppCoord -astrom $astrom -streaks $streaks";
+        if ($ipprc->file_exists($clusters)) {
+            $command .= " -clusters $clusters";
+        }
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        open my $coords, "> $outroot.coords";
+        print $coords join("", @$stdout_buf);
+        close $coords;
+    }
+
+
+    {
+        my $command = "$magictool -addmask";
+        $command   .= " -magic_id $magic_id";
+        $command   .= " -uri $streaks_file";
+        $command   .= " -streaks $num_streaks";
+        $command   .= " -dbname $dbname" if defined $dbname;
+
+        # Add the processed file to the database
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                # This isn't going to work because our result was already entered.
+                &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
+            }
+        } else {
+            print "Skipping command: $command\n";
+        }
     }
 }

branches/czw_branch/20100427/ippScripts/scripts/make_burntool_pcontrol.pl

@@ -1 @@
-#!/usr/bin/perl -w
+#!/usr/bin/env perl 
 # script to generate a pcontrol.pro file to run burntool on a set of data.
 
 use DBI;
+use warnings;
 use DateTime;
 use Getopt::Std;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use IPC::Cmd 0.36 qw( can_run run);
 
 use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
@@ -35 +39 @@
                    ) or die "Unable to connect to database $DBI::errstr\n";
 
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+if ($missing_tools) {
+    die "Cannot find required tools.";
+}
 # Determine what the value of "BURNTOOL.STATE.GOOD" currently is:
 $burntoolStateGood = 999;
-open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n";
+open(LAZY,"$ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n";
 while(<LAZY>) {
     chomp;
@@ -46 +54 @@
 }
 close(LAZY);
-#unless( $burntoolStateGood == 999) {
-#    print STDERR "GOOD == $burntoolStateGood\n";
-#}
-
-
-# List of which obs_modes are "science" and which are something else.
-%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1,
-            'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 1, ' ' => 0);
+
+# Read the nightly science config file and use that to determine which data is science
+my $verbose = 0;
+my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run(command => $conf_cmd, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
+}
+
+my @target_list;
+my %object_list;
+my $comment_list;
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+foreach my $entry (@{ $metadata }) {
+    if (${ $entry }{name} eq 'TARGETS') {
+        my @target_data = @{ ${ $entry }{value} };
+        my $this_target = '';
+        
+        foreach my $tentry (@target_data) {
+            if (${ $tentry }{name} eq 'NAME') {
+                $this_target = ${ $tentry }{value};
+                push @target_list, $this_target;
+                $obsmode_list{$this_target} = '.*';
+                $object_list{$this_target} = '.*';
+                $comment_list{$this_target} = '.*';
+            }
+            elsif (${ $tentry }{name} eq 'OBSMODE') {
+                $obsmode_list{$this_target} = ${ $tentry }{value};
+                $obsmode_list{$this_target} =~ s/%//;
+            }
+            elsif (${ $tentry }{name} eq 'OBJECT') {
+                $object_list{$this_target} = ${ $tentry }{value};
+                $object_list{$this_target} =~ s/%//;
+            }
+            elsif (${ $tentry }{name} eq 'COMMENT') {
+                $comment_list{$this_target} = ${ $tentry }{value};
+                $comment_list{$this_target} =~ s/%//;
+            }
+        }
+    }
+}
 if (exists($opt{P})) {
-    $science{Unknown} = 1;
+    push @target_list, 'PR';
+    $obsmode_list{'PR'} = 'Unknown';
+    $object_list{'PR'} = '.*';
+    $comment_list{'PR'} = '.*';
 }
 
@@ -119 +167 @@
     
     # Find _ALL_ observations within the date range we care about
-    $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment FROM rawExp WHERE " .
+    $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment,object FROM rawExp WHERE " .
         "dateobs >= '${date_min}' AND dateobs <= '${date_max}' order by dateobs";
 
@@ -133 +181 @@
     foreach $row_ref (@{ $data_ref }) {
 
-        ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref };
+        ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment,$object) = @{ $row_ref };
         # Fix nulls in the database
         if (!defined($obs_mode)) {
             $obs_mode = ' ';
         }
+        if (!defined($object)) {
+            $object = ' ';
+        }
         if (!defined($comment)) {
             $comment = ' ';
@@ -144 +195 @@
         ($date,$time) = split / /, $dateobs;
         ($year,$month,$day) = split /-/, $date;
-        ($hour,$minute,$second) = split /:/, $time;
+        ($hour,$minute,$second) = split /:/, $time; # / # This stops emacs run-on syntax highlights.
         
         $dt = DateTime->new( year   => $year, month  => $month, day    => $day,
@@ -155 +206 @@
         $et = $dt->epoch();
 
-
-        if (!exists($science{$obs_mode})) {
-            $science{$obs_mode} = 0;
-        }
-        if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))||
-            (($science{$obs_mode} == 0)&&(
-                                          ($comment =~ /MD/)||($comment =~ /ThreePi/)||
-                                          ($comment =~ /STS/i)||($comment =~ /M31/)||($comment =~ /Stellar Transit/)||
-                                          ($comment =~ /CAL/i)||($comment =~ /SS/)||($comment =~ /SVS/)||
-                                          ($comment =~ /Sweetspot/)||($comment =~ /virgo/i)||($comment =~ /kepler/)||
-                                          ($comment =~ /sdss/))))                                         
-        {
+        if (is_science($exp_type,$obs_mode,$object,$comment)) {
             if ($start_s[-1] == 0) {
                 $start_s[-1] = $et - 1800;
@@ -194 +234 @@
     # Scan again and count how many exposures are between windows
     foreach $row_ref (@{ $data_ref }) {
-        ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref };
+        ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment,$object) = @{ $row_ref };
         if (!defined($obs_mode)) {
             $obs_mode = ' ';
@@ -204 +244 @@
         ($date,$time) = split / /, $dateobs;
         ($year,$month,$day) = split /-/, $date;
-        ($hour,$minute,$second) = split /:/, $time;
+        ($hour,$minute,$second) = split /:/, $time; # / # another emacs syntax highlight bug.
         
         $dt = DateTime->new( year   => $year, month  => $month, day    => $day,
@@ -215 +255 @@
         $et = $dt->epoch();
 
-        if (!exists($science{$obs_mode})) {
-            die "No mode >>$obs_mode<<\n";
-        }
-        
         for ($i = 0; $i <= $#start_s; $i++) {
-            if (($et < $start_s[$i])&&($science{$obs_mode} == 0)) {
+            if (($et < $start_s[$i])&&(is_science($exp_type,$obs_mode,$object,$comment) == 0)) {
                 if ($i == 0) {
                     $skips[$i]++;
@@ -278 +314 @@
 unless(exists($opt{b}) || ($N_ranges == 0)) {
     print_epilogue();
+}
+
+
+# Subroutine to use the configuration file data and the exposure data to determine whether or not something is "science"
+sub is_science {
+#    if (is_science($exp_type,$obs_mode,$object,$comment)) {
+    my ($exp_type,$obs_mode,$object,$comment) = @_;
+    my $return_value = 0;
+    unless($object) {
+        $object = '';
+    }
+    if ($exp_type eq 'OBJECT') {
+        foreach my $target (@target_list) {
+#           print STDERR "$target $obsmode_list{$target} $exp_type $obs_mode $object $comment\n";
+            if (($obs_mode =~ /$obsmode_list{$target}/)&&
+                ($object   =~ /$object_list{$target}/)&&
+                ($comment  =~ /$comment_list{$target}/)) {
+                $return_value = 1;
+            }
+#           print ">$return_value $exp_type $obs_mode $object $comment $target $obsmode_list{$target} $object_list{$target} $comment_list{$target}\n";
+            if ($return_value) {
+                return($return_value);
+            }
+        }
+    }
+    return(0);
 }
 

branches/czw_branch/20100427/ippScripts/scripts/receive_advance.pl

@@ -60 +60 @@
     &my_die( "Unable to set up", $fileset_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
-# update the fileset entry
-
-if ($dbinfo_uri and ($dbinfo_uri ne "NULL")) {
+my $import_run_to_db = 0;
+if ($import_run_to_db and $dbinfo_uri and ($dbinfo_uri ne "NULL")) {
     my $filename = basename($dbinfo_uri);
     my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest
@@ -102 +101 @@
 }
 
+# update the fileset entry
 # All done
 {

branches/czw_branch/20100427/ippScripts/scripts/register_exp.pl

@@ -127 +127 @@
 $command .= " -end_stage $end_stage" if defined $end_stage;
 $command .= " -tess_id   $tess_id"   if defined $tess_id;
+$command .= " -state full";
 $command .= " $cmdflags";
 

branches/czw_branch/20100427/ippScripts/scripts/register_imfile.pl

@@ -174 +174 @@
 $command .= " -hostname $host" if defined $host;
 $command .= " -dbname $dbname" if defined $dbname;
+$command .= " -data_state full";
 $command .= " $cmdflags";