Changeset 36072 for branches/eam_branches/ipp-20130711

Timestamp:

Aug 30, 2013, 4:52:14 PM (13 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20130711

Files:

• Nebulous (modified) (1 prop)
• ippMonitor (modified) (1 prop)
• ippScripts/scripts/background_chip.pl (modified) (3 diffs)
• ippScripts/scripts/chip_imfile.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/dist_advancerun.pl (modified) (2 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (6 diffs)
• ippScripts/scripts/dist_make_fileset.pl (modified) (2 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/ipp_cleanup.pl (modified) (1 diff)
• ippScripts/scripts/lap_science.pl (modified) (9 diffs, 1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/nightly_science.pl (modified) (4 diffs)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/receive_advance.pl (modified) (3 diffs)
• ippScripts/scripts/receive_file.pl (modified) (2 diffs)
• ippScripts/scripts/register_imfile.pl (modified) (2 diffs)
• ippScripts/scripts/skycell_jpeg.pl (modified) (8 diffs)
• ippScripts/scripts/stack_skycell.pl (modified) (1 diff)
• ippScripts/scripts/warp_skycell.pl (modified) (1 prop)
• ippTasks (modified) (1 prop)
• ippTasks/diff.pro (modified) (4 diffs)
• ippTasks/dist.pro (modified) (1 diff)
• ippTasks/minidvodb.pro (modified) (1 diff)
• ippTasks/nightly_science.pro (modified) (1 diff)
• ippTasks/register.pro (modified) (1 diff)
• ippTasks/survey.pro (modified) (3 diffs)
• ippTasks/warp.pro (modified) (4 diffs)
• ippToPsps (modified) (1 prop)
• ippToPsps/config/changes.txt (copied) (copied from trunk/ippToPsps/config/changes.txt )
• ippToPsps/config/ippToPspsDbSchema.sql (modified) (1 diff)
• ippToPsps/jython/Makefile.am (modified) (1 diff)
• ippToPsps/jython/detectionbatch.py (modified) (1 diff)
• ippToPsps/jython/gpc1db.py (modified) (2 diffs)
• ippToPsps/jython/ipptopspsdb.py (modified) (2 diffs)
• ippToPsps/jython/queue.py (modified) (1 diff)
• ippToPsps/jython/removepending.py (copied) (copied from trunk/ippToPsps/jython/removepending.py )
• psconfig (modified) (1 prop)
• psconfig/tagsets/ipp-3.0.dist (modified) (1 diff)
• tools/fixburntool (modified) (1 diff)
• tools/releasecalib (copied) (copied from trunk/tools/releasecalib )
• tools/releasecalib/scripts (copied) (copied from trunk/tools/releasecalib/scripts )
• tools/releasecalib/scripts/calibudpateprocess.txt (copied) (copied from trunk/tools/releasecalib/scripts/calibudpateprocess.txt )
• tools/releasecalib/scripts/extractinfo (copied) (copied from trunk/tools/releasecalib/scripts/extractinfo )
• tools/releasecalib/scripts/updatecalib (copied) (copied from trunk/tools/releasecalib/scripts/updatecalib )
• tools/rundiffskycell.pl (modified) (3 diffs)

branches/eam_branches/ipp-20130711/ippScripts/scripts/background_chip.pl

@@ -34 +34 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -134 +135 @@
 }
 my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id);
-my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
+my $in_bg;
+if (1) {
+    $in_bg = $ipprc->filename("PPIMAGE.BACKMDL", $cam_path_base, $class_id);
+}
+else {
+    $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
+}
 my $in_pattern = $ipprc->filename("PPIMAGE.PATTERN", $in_path, $class_id);
 my $in_config = $ipprc->filename("PPIMAGE.CONFIG", $in_path, $class_id);
@@ -220 +227 @@
 }
 
+my $do_binned_images = 1;   # Generate the binned images that are useful for making JPEGs
+if ($do_binned_images) {
+    my $command = "$ppImage -file $out_image -mask $out_mask $outroot -recipe PPIMAGE PPIMAGE_PA -Db PHOTOM F";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppImage: $error_code", $chip_bg_id, $class_id, $error_code);
+    }
+}
+
+
 if (!$quality and !$no_op) {
     &my_die("Couldn't find expected output file: $out_image", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image);

branches/eam_branches/ipp-20130711/ippScripts/scripts/chip_imfile.pl

@@ -233 +233 @@
     if ($do_photom and ($run_state eq 'update')) {
         # If previous psf file is ok skip photometry
-        if ($dump_config || rerun_photometry($outroot, $class_id)) {
+        if (rerun_photometry($outroot, $class_id)) {
             carp "Will rerun photometry\n";
         } else {
@@ -348 +348 @@
             }
         }
+        # resolve uri
+        my $uriReal = $ipprc->file_resolve( $uri );
+        # If instance is not found or if it doesn't exist try running neb-repair on it 
+        # Note: file_exists returns false if the file exists but has zero size
+        if (!$uriReal or !$ipprc->file_exists($uriReal)) {
+            my $repair_cmd = "$nebrepair $uri";
+            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+            unless ($repair_success) {
+                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+            $uriReal = $ipprc->file_resolve( $uri );
+        }
+            
+        &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
 
         # if recipe option PPIMAGE:APPLY.BURNTOOL is set and the burntool information is stored in
@@ -379 +393 @@
                                               UNLINK => !$save_temps, SUFFIX => '.fits' );
 
-            # get the UNIX version of the (possible) neb: or path: filename
-            my $uriReal = $ipprc->file_resolve( $uri );
-                # Catch errors here
-            if (!$uriReal) {
-                my $repair_cmd = "$nebrepair $uri";
-                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
-                unless ($repair_success) {
-                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-                }
-                $uriReal = $ipprc->file_resolve( $uri );
-            }
-                
-            &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
-
             # funpack into the temp file.
             my $funpack_cmd = "$funpack -S $uriReal > $tempName";
@@ -492 +492 @@
     if ($no_compress_image) {
         $command .= " -R PPIMAGE.CHIP FITS.TYPE NONE";
+    }
+    if ($run_state eq 'update' and metadataLookupBool($recipeData, 'MASK.STATS')) {
+        $command .= " -Db PPIMAGE:MASK.STATS FALSE ";
     }
     $command .= $burntoolArguments if $burntoolArguments;

branches/eam_branches/ipp-20130711/ippScripts/scripts/dist_advancerun.pl

@@ -115 +115 @@
     $list_mode = "-result";
     $component_key = "skycell_id";
+} elsif ($stage eq "skycal") {
+    $tool_cmd = "$staticskytool -skycal_id";
+    $list_mode = "-skycalresult";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";
@@ -134 +138 @@
 } elsif ($stage eq 'warp_bg') {
     $exportarg = '-exportwarp';
+} elsif ($stage eq 'skycal') {
+    $exportarg = '-exportskycalrun';
 }
 

branches/eam_branches/ipp-20130711/ippScripts/scripts/dist_bundle.pl

@@ -53 +53 @@
                       'PPSTACK.OUTPUT.MASK' => 'mask',
                       'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
-my %sky_cleaned   = ( 'PSPHOT.STACK.OUTPUT.IMAGE' => 'image',
-                      'PSPHOT.STACK.OUTPUT.MASK' => 'mask',
-                      'PSPHOT.STACK.OUTPUT.VARIANCE' => 'variance' );
+my %empty_cleaned = ();
 
 
@@ -110 +108 @@
     defined $outroot;
 
+$no_magic = 1;
+
 if ($stage eq 'raw' and !$clean and !$no_magic) {
     
@@ -134 +134 @@
 my $file_list = get_file_list($stage, $component, $path_base, $clean, $num_sky_inputs);
 
-if (($stage ne 'raw') and ($stage ne 'fake') and !$poor_quality) {
+if (($stage ne 'raw') and ($stage ne 'fake') and ($stage ne 'stack_summary') and !$poor_quality) {
     # If the file list is empty it is an error because we should at least get a config dump file
     # except for fake stage which doesn't do anything yet
@@ -460 +460 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
-    } elsif ($stage eq "sky") {
-        $type = $sky_cleaned{$rule};
+    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary') {
+        $type = $empty_cleaned{$rule};
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -546 +546 @@
         $config_file_rule = "PPSTACK.CONFIG";
     } elsif ($stage eq "sky") {
-        if (1 or $num_sky_inputs > 1) {
-            $config_file_rule = "PSPHOT.STACK.CONFIG";
-        } else {
-            $config_file_rule = "PSPHOT.SKY.CONFIG";
-        }
+        $config_file_rule = "PSPHOT.STACK.CONFIG";
+    } elsif ($stage eq "skycal") {
+        $config_file_rule = "PSASTRO.CONFIG";
+    } elsif ($stage eq "stack_summary") {
+        # stack_summary stage does not use a config dump file.
+        return build_stack_summary_file_list($path_base, \@file_list);
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -665 +666 @@
     if (!$num_inputs) {
         $num_inputs = "undefined" if !defined $num_inputs;
-        &my_die("unexpected number of static sky inputs $num_inputs",  $PS_EXIT_PROG_ERROR, $component, $error_code);
+        &my_die("unexpected number of static sky inputs $num_inputs",  $component, $error_code);
     }
 
     return $num_inputs;
+}
+
+# For stack_summary stage, we don't have a config dump file to give us the list of files.
+# For now run neb-ls on the path base and take everything that matches.
+sub build_stack_summary_file_list {
+    my ($path_base, $file_list) = @_;
+
+    my $scheme = file_scheme($path_base);
+
+    if (!$scheme or $scheme ne 'neb') {
+        &my_die("Building bundles for stack_summary not supported for no nebulous path_base.",
+            $component, $PS_EXIT_PROG_ERROR);
+    }
+
+    my $command = "neb-ls $path_base" . '%';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $component, $error_code);
+    }
+    my $nebls_output = join "", @$stdout_buf;
+
+    my @files = split "\n", $nebls_output;
+
+    my $i = 0;
+    foreach my $f (@files) {
+        my %file;
+        $file{file_rule} = sprintf 'RULE.%d', $i++;
+        $file{name} = 'neb://any/' . $f;
+        push @$file_list, \%file;
+    }
+
+    return $file_list;
 }
 

branches/eam_branches/ipp-20130711/ippScripts/scripts/dist_make_fileset.pl

@@ -223 +223 @@
     my $dbname = shift;
 
-    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky') {
+    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal') {
         return "";
     }
@@ -252 +252 @@
     } elsif ($stage eq 'warp_bg') {
         $query = "SELECT exp_name FROM warpBackgroundRun JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE warp_bg_id = $stage_id";
+
     } else {
-        &my_die("$stage is invalid value for stage\n");
+        &my_die("$stage is invalid value for stage", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
     }
 

branches/eam_branches/ipp-20130711/ippScripts/scripts/ipp_apply_burntool_single.pl

@@ -45 +45 @@
     defined $dbname;
 
-unless (defined $camera) {
-    $camera = "GPC1";
-}
+# unless (defined $camera) {
+#    $camera = "GPC1";
+# }
 
 my $missing_tools;
@@ -146 +146 @@
             }
             if (($addable == 1)&&($#continue_exp_ids < $continue)) {
-                $this_exp_id = $entry->{exp_id};
-                $this_uri    = $entry->{uri};
-                $this_state  = $entry->{imfile_state};
+                $this_exp_id = $entry->{exp_id} ? $entry->{exp_id} : 0;
+                $this_uri    = $entry->{uri} ? $entry->{uri} : 'NULL';
+                $this_state  = $entry->{imfile_state} ? $entry->{imfile_state} : 'NULL';
                 print ">> $this_exp_id $this_uri $this_state\n";
                 if ($this_state eq 'pending_burntool') {
@@ -177 +177 @@
     $outTable  =~ s/fits$/burn.tbl/;
     my $outTableReal = $ipprc->file_resolve($outTable, 1);
+    if (!$outTableReal) {
+        &my_die("failed to resolve output file: $outTable",$exp_id,$class_id);
+    }
     
     my $previousTable;
@@ -184 +187 @@
         $previousTable =~ s/fits$/burn.tbl/;
         $previousTableReal = $ipprc->file_resolve($previousTable, 0);
+        if (!$previousTableReal) {
+            &my_die("failed to resolve input file: $previousTable",$exp_id,$class_id);
+        }
     }
     

branches/eam_branches/ipp-20130711/ippScripts/scripts/ipp_cleanup.pl

@@ -933 +933 @@
             addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id);
 
+            addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id);
+
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
                 addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id);
-                addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id);
                 addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id);
                 # Commented out to match warp files.

branches/eam_branches/ipp-20130711/ippScripts/scripts/lap_science.pl

@@ -33 +33 @@
 # Global configuration constants that probably should be read from elsewhere.  
 my $qstack_threshold     = 0.05; # Only make a quickstack if more than 5% of the exposures require it.
-my $minimum_stack_inputs = 4;    # We can avoid magicking stack inputs if we have more than this number.
+my $minimum_stack_inputs = 2;    # We can avoid magicking stack inputs if we have more than this number.
 my $do_updates_pv2       = 1;    # Do updates using the calls needed for PV2.  
 
@@ -309 +309 @@
     my $magicDS_update_cmd  = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label";
     $magicDS_update_cmd    .= " -dbname $dbname " if defined $dbname;
+    my $warptool_preupdate_cmd = "warptool -revertwarped -warp_id $warp_id -fault 26";
+    $warptool_preupdate_cmd .= " -dbname $dbname " if defined $dbname;
     my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label";
     $warptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
@@ -314 +316 @@
     # only need to update the data if the warps are not full
     if ($warp->{state} ne 'full') { 
-        if ($chip->{state} eq 'cleaned') {
+        if (($chip->{state} eq 'cleaned')||  # Exposure is cleaned
+            (($chip->{state} eq 'update')&&($chip->{label} ne $label))) {  # Exposure is owned by an interloper.
             ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $chiptool_update_cmd, verbose => $verbose);
@@ -341 +344 @@
         }
 
+
+        ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $warptool_preupdate_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform warptool --revertwarped: $error_code", $exposure->{exp_id}, $data_group);
+        }
+
         ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $warptool_update_cmd, verbose => $verbose);
@@ -364 +375 @@
     if (S64_IS_NOT_NULL($exposure->{chip_id})) {
         if (($exposure->{chip_state} eq 'cleaned')||
+            ($exposure->{chip_state} eq 'update')||
             ($exposure->{chip_state} eq 'goto_cleaned')||
             ($exposure->{chip_state} eq 'error_cleaned')) {
@@ -777 +789 @@
 #           $can_qstack ++;
             $can_diff ++;
-            $exposure->{data_state} = 'to_diff';
+            ## BEGIN These two disable LAP stacks
+            $are_magicked = $total_exposures;
+            $have_diff ++;
+            ## END
+#           $exposure->{data_state} = 'to_diff';
+            $exposure->{data_state} = 'full';
         }
         if (($exposure->{diff_id})&&
@@ -858 +875 @@
                     }
                 }
-                elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic
+                elsif ($are_magicked >= $total_exposures) { # We have made all of the diffs, and propagated through magic
                     print "STATUS: We would create all remaining stacks possible, but no longer require magic.\n";
 #                   my $Nstacks_made = -1;
@@ -1025 +1042 @@
 #    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
     $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
-    $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} ";
+    $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
     $command .= " $warps ";
 
@@ -1106 +1123 @@
 #    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
     $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
-    $command .= " -min_num 2 -min_new 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} ";
+    $command .= " -min_num 2 -min_new 2 -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
     $command .= " $warps ";
 

branches/eam_branches/ipp-20130711/ippScripts/scripts/nightly_science.pl

@@ -1562 +1562 @@
             }
             
-            my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
-            $cmd .= "-input_label $label  -template_label $label ";
+            my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
+            $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
             $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
             $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
@@ -1699 +1699 @@
             my $new_data_group = "${data_group}.multi";
             my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
-            $cmd .= "-input_label $label  -template_label $label ";
+            $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
             $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
             $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $new_data_group ";
@@ -1756 +1756 @@
     my $new_dist_group = "${dist_group}.offnight";
     my $check_cmd = "$difftool -dbname $dbname  -definewarpwarp ";
-    $check_cmd .= "-input_label $label  -template_label $label ";
+    $check_cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
     $check_cmd .= "-set_workdir $workdir  -set_dist_group $new_dist_group  -set_data_group $new_data_group ";
     $check_cmd .= " -mintimediff 40000 ";
@@ -1778 +1778 @@
         unless (multi_date_verify_uniqueness_diff($diff->{input_warp_id},$diff->{template_warp_id},$date,$target)) {
             # If we don't already have a diff with these inputs, make a diff with these inputs.
-            my $cmd = "$difftool -dbname $dbname -definewarpwarp ";
+            my $cmd = "$difftool -dbname $dbname -definewarpwarp -good_frac 0.1 ";
             $cmd   .= "-input_label $label -template_label $label ";
             $cmd   .= "-warp_id $diff->{input_warp_id} -template_warp_id $diff->{template_warp_id} ";

branches/eam_branches/ipp-20130711/ippScripts/scripts/receive_advance.pl

@@ -65 +65 @@
     my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest
     my $tool_name;
+    my $tool_mode = '-importrun';
     if ($stage eq "raw") {
         $tool_name = "regtool";
@@ -73 +74 @@
     } elsif ($stage eq "sky") {
         $tool_name = "staticskytool";
+    } elsif ($stage eq "skycal") {
+        $tool_name = "staticskytool";
+        $tool_mode = '-importskycal';
     } else {
         $tool_name = "${stage}tool";
@@ -81 +85 @@
     &my_die("Unable to resolve $dbinfo_uri\n", $PS_EXIT_UNKNOWN_ERROR) unless $file;
 
-    my $command = "$tool -importrun -infile $file"; # Command to execute
+    my $command = "$tool $tool_mode -infile $file"; # Command to execute
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

branches/eam_branches/ipp-20130711/ippScripts/scripts/receive_file.pl

@@ -223 +223 @@
     } elsif ($runType eq 'staticskyRun') {
         $stage = 'sky';
-        # XXX: This should be skycell, but the distribution code uses exposure
         $comp_name = 'skycell_id';
-#        $current_component = $comp_name;
+    } elsif ($runType eq 'skycalRun') {
+        $stage = 'skycal';
+        $comp_name = 'skycell_id';
     } else {
         &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
@@ -238 +239 @@
     }
 
-    if ($stage eq 'sky') {
-        # the dbinfo file for a skyRun only has one component and it doesn't contain
+    if ($stage eq 'sky' or $stage = 'skycal') {
+        # the dbinfo file for staticskyRun and skycalRun only have one component and they don't contain
         # skycell_id which is the way components are listed in the dirinfo file.
         my @ids = keys %$components;

branches/eam_branches/ipp-20130711/ippScripts/scripts/register_imfile.pl

@@ -291 +291 @@
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run ( command => $bt_check_command, verbose => $verbose);
+        run ( command => $bt_check_command, verbose => 0); 
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -341 +341 @@
         $apply_command .= " --class_id $class_id --exp_id $exp_id ";
         $apply_command .= " --this_uri $burntool_data->{uri} ";
+        $apply_command .= " --camera GPC1 "; # hack, but we're only going to get here if we're GPC1.
         $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
         $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";

branches/eam_branches/ipp-20130711/ippScripts/scripts/skycell_jpeg.pl

@@ -17 +17 @@
 my ($help,$masks);
 my ($tempFile,$tempName,$maskFile,$maskName);
+$verbose = 0;
 my $ppSkycell = can_run('ppSkycell') or (warn "Can't find ppSkycell" and $missing_tools = 1);
 my $warptool  = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
@@ -38 +39 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
 pod2usage(
-    -msg => "Required options: --stage --stage_id --path_base\nHelpful options: --camera --dbname",
+    -msg => "Required options: --stage --stage_id --outroot\nHelpful options: --camera --dbname",
     -exitval => 3,
     ) unless
@@ -100 +101 @@
             }           
         }
-        print "$skycell_id $projection_cell\n"; 
+#       print "$skycell_id $projection_cell\n"; 
         my $file = $ipprc->filename("PSWARP.OUTPUT", $path_base, $skycell_id);
-        print "$file $state $quality\n";
+#       print "$file $state $quality\n";
         my $f_fh = $tangents{$projection_cell}{FILE};
         print $f_fh "$file\n";
         if ($masks) {
             my $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
-            print "$mask\n";
+#           print "$mask\n";
             my $m_fh = $tangents{$projection_cell}{MFILE};
             print $m_fh "$mask\n";
@@ -118 +119 @@
         }
         $command .= " ${outroot}.${projection_cell} ";
-        print "$command\n";
+#       print "$command\n";
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
         unless ($success) {
@@ -197 +198 @@
             }           
         }
-        print "$skycell_id $projection_cell\n"; 
+#       print "$skycell_id $projection_cell\n"; 
         my $file = $ipprc->filename("PPSUB.OUTPUT", $path_base, $skycell_id);
-        print "$file $state $quality\n";
+#       print "$file $state $quality\n";
         my $f_fh = $tangents{$projection_cell}{FILE};
         print $f_fh "$file\n";
         if ($masks) {
             my $mask = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
-            print "$mask\n";
+#           print "$mask\n";
             my $m_fh = $tangents{$projection_cell}{MFILE};
             print $m_fh "$mask\n";
@@ -215 +216 @@
         }
         $command .= " ${outroot}.${projection_cell} ";
-        print "$command\n";
+#       print "$command\n";
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
         unless ($success) {
@@ -297 +298 @@
                 $tangents{$projection_cell}{$key}{FILE} = $tempFile;
                 $tangents{$projection_cell}{$key}{NAME} = $tempName;
+                $tangents{$projection_cell}{$key}{N}    = 0;
             }
         }
         foreach my $key (keys %products) {
-            print "$skycell_id $projection_cell\n";     
+#           print "$skycell_id $projection_cell\n";     
             my $file = $ipprc->filename($products{$key}, $path_base, $skycell_id);
-            print "$file $state $quality\n";
+#           print "$file $state $quality\n";
             my $f_fh = $tangents{$projection_cell}{$key}{FILE};
-            print $f_fh "$file\n";
+            if ($ipprc->file_exists($file)) {
+                print $f_fh "$file\n";
+                $tangents{$projection_cell}{$key}{N} ++;
+            }
         }
     }
@@ -322 +327 @@
                 $command .= " -Di BIN1 1 -Di BIN2 1 ";
             }
-            elsif ($key eq 'image') {
-                $command .= " -masks $tangents{$projection_cell}{mask}{NAME} ";
-            }
-            elsif ($key eq 'mask') { 
-                next; # This should be made with the images.
-            }
-            print "$command\n";
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
+#           elsif ($key eq 'image') {
+#               $command .= " -masks $tangents{$projection_cell}{mask}{NAME} ";
+#           }
+#           elsif ($key eq 'mask') { 
+#               next; # This should be made with the images.
+#           }
+            else {
+                # Append the image list to other objects, in case the WCS information is unpopulated
+                $command .= " -wcsref $tangents{$projection_cell}{image}{NAME} ";
+            }
+            if ($tangents{$projection_cell}{$key}{N} > 0) {
+                print "$command\n";
+                ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#                   print "Was unable to perform ppSkycell $stage_id $error_code $key\n";
+                    &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
+                }
             }
         }

branches/eam_branches/ipp-20130711/ippScripts/scripts/stack_skycell.pl

@@ -299 +299 @@
         $command .= " -F PPSTACK.OUTPUT.VARIANCE PPSTACK.OUTPUT.VARIANCE.NOCOMP";
         $command .= " -F PPSTACK.OUTPUT.EXPWT PPSTACK.OUTPUT.EXPWT.NOCOMP";
+        $command .= " -F PPSTACK.OUTPUT.EXP PPSTACK.OUTPUT.EXP.NOCOMP";
         $command .= " -F PPSTACK.UNCONV PPSTACK.UNCONV.NOCOMP";
         $command .= " -F PPSTACK.UNCONV.VARIANCE PPSTACK.UNCONV.VARIANCE.NOCOMP";
         $command .= " -F PPSTACK.UNCONV.EXPWT PPSTACK.UNCONV.EXPWT.NOCOMP";
+        $command .= " -F PPSTACK.UNCONV.EXP PPSTACK.UNCONV.EXP.NOCOMP";
     }
     if ($output_logflux) {

branches/eam_branches/ipp-20130711/ippTasks/diff.pro

@@ -47 +47 @@
     active true
   end
-  task diff.summary.load
-    active true
-  end
-  task diff.summary.run
-    active true
-  end
 end
 
@@ -67 +61 @@
   end
   task diff.revert
-    active false
-  end
-  task diff.summary.load
-    active false
-  end
-  task diff.summary.run
     active false
   end
@@ -499 +487 @@
 task           diff.summary.load
   host         local
+  active       false
 
   periods      -poll $LOADPOLL
@@ -559 +548 @@
   periods      -exec $RUNEXEC
   periods      -timeout 60
+  active       false
 
   task.exec

branches/eam_branches/ipp-20130711/ippTasks/dist.pro

@@ -31 +31 @@
 list DIST_STAGE -add "warp_bg"
 list DIST_STAGE -add "sky"
+list DIST_STAGE -add "skycal"
 
 $currentStage = 0

branches/eam_branches/ipp-20130711/ippTasks/minidvodb.pro

@@ -449 +449 @@
     stderr $LOGDIR/minidvodb.premerge.run.log
 
-    $run = minidvodb_premerge.pl --camera $CAMERA --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
+    #still buggy
+
+    $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
     
   if ($DB:n == 0)

branches/eam_branches/ipp-20130711/ippTasks/nightly_science.pro

@@ -370 +370 @@
   periods         -poll $LOADPOLL
   periods         -exec $LOADEXEC
-  periods         -timeout 480
+  periods         -timeout 1000
   npending        1
 

branches/eam_branches/ipp-20130711/ippTasks/register.pro

@@ -659 +659 @@
     stderr $LOGDIR/register.burntool.run.log
 
-    $run = ipp_apply_burntool_single.pl --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI --continue 10
+    $run = ipp_apply_burntool_single.pl --camera $CAMERA --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI --continue 10
     if ("$PREVIOUS_URI" != "NULL")
       $run = $run --previous_uri $PREVIOUS_URI

branches/eam_branches/ipp-20130711/ippTasks/survey.pro

@@ -583 +583 @@
 
 macro survey.add.relstack
-  if ($0 != 3)
-    echo "USAGE: survey.add.relstack (label) (releasename)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.relstack (label) (releasename) (stacktype)"
     break
   end
@@ -590 +590 @@
   book setword SURVEY_RELSTACK $1 LABEL $1
   book setword SURVEY_RELSTACK $1 RELEASE_NAME $2
+  book setword SURVEY_RELSTACK $1 STACK_TYPE $3
   book setword SURVEY_RELSTACK $1 STATE PENDING
 end
@@ -1846 +1847 @@
     book setword SURVEY_RELSTACK $label STATE DONE
     book getword SURVEY_RELSTACK $label RELEASE_NAME -var RELEASE_NAME
+    book getword SURVEY_RELSTACK $label STACK_TYPE -var STACK_TYPE
 
     # XXX: Can we use the survey task for stack types other than nightly?
-    $run = releasetool -definerelstack -label $label -release_name $RELEASE_NAME -set_state processed -set_stack_type nightly
+    $run = releasetool -definerelstack -label $label -release_name $RELEASE_NAME -set_state processed -set_stack_type $STACK_TYPE
 
     if ($DB:n == 0)

branches/eam_branches/ipp-20130711/ippTasks/warp.pro

@@ -58 +58 @@
     active true
   end
-  task warp.summary.load
-    active true
-  end
-  task warp.summary.run
-    active true
-  end
 end
 
@@ -87 +81 @@
   end
   task warp.revert.warped
-    active false
-  end
-  task warp.summary.load
-    active false
-  end
-  task warp.summary.run
     active false
   end
@@ -545 +533 @@
 task           warp.summary.load
   host         local
-
+  active       false
   periods      -poll $LOADPOLL
   periods      -exec $LOADEXEC
@@ -605 +593 @@
   periods      -exec $RUNEXEC
   periods      -timeout 60
+  active       false
 
   task.exec

branches/eam_branches/ipp-20130711/ippToPsps/config/ippToPspsDbSchema.sql

@@ -127 +127 @@
   `active` tinyint(1) default '1',
   `parallel` tinyint(1) default '0',
+  `P2_smf_version` varchar(16) default NULL,
+  `ST_cmf_version` varchar(16) default NULL, 
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;

branches/eam_branches/ipp-20130711/ippToPsps/jython/Makefile.am

@@ -30 +30 @@
   pslogger.py \
   queue.py \
+  removepending.py \ 
   scratchdb.py \
   setupScratchDb.py \

branches/eam_branches/ipp-20130711/ippToPsps/jython/detectionbatch.py

@@ -54 +54 @@
                batchID,
                "P2", 
-               gpc1Db.getCameraStageSmf(camID),
+               gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version),
                useFullTables)
 

branches/eam_branches/ipp-20130711/ippToPsps/jython/gpc1db.py

@@ -212 +212 @@
     Gets a camera-stage smf for this cam_id
     '''
-    def getCameraStageSmf(self, camID):
-
+    def getCameraStageSmf(self, camID, smfversion):
+        self.logger.infoPair("using", smfversion)
         self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID))
 
@@ -237 +237 @@
         # list all smf files if a neb path
         files = []
-        if path.startswith("neb"):
-
-            # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
-            f=os.popen("neb-ls -p "+path+"%smf")
-            for i in f.readlines():
-                files.append(i.rstrip())
-
-        # or not a neb path
-        else:
-            files = glob.glob(path + ".smf")
+
+
+        #there are a couple of states for smfversion
+
+        # "use_new" => we have reprocessed the data, use the .smf file
+
+        # "not_reproc" => this has not been reprocessed, use the .smf file
+
+        # "use_original" => we *may* have reprocessed the data, 
+        #    use (file).smf.original if camRun.state = 'full' and if there is a .smf and .orig
+        #    use (file).smf if camRun.state = 'full' and if there is only a .smf
+        #    fault out if camRun.state != 'full' -- that is a race condition we don't want.
+       
+        
+        if (smfversion == "not_reproc") or (smfversion == "use_new"):
+            if path.startswith("neb"):
+
+                # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+                f=os.popen("neb-ls -p "+path+"%smf")
+                for i in f.readlines():
+                    files.append(i.rstrip())
+
+                # or not a neb path
+            else:
+                files = glob.glob(path + ".smf")
+
+        if (smfversion == "use_original"):
+            # get state of camRun
+            sql = "SELECT state \
+               FROM camProcessedExp \
+               JOIN camRun USING(cam_id) \
+               WHERE camRun.cam_id = %d" % camID
+            try:
+                rs = self.executeQuery(sql)
+                rs.first()
+            except:
+                self.logger.errorPair("Can't query for camRun.state for cam_id",  str(camID))
+                return None
+
+            try:
+                # get state of camRun.cam_id
+                camrunstate = rs.getString(1)
+            except:
+                self.logger.errorPair("No state found for cam_id", str(camID) )
+                return None
+            
+            if (camrunstate == "full"):
+              self.logger.infoPair("camRun.state is ",camrunstate)   
+                 # if full find smf files
+              if path.startswith("neb"):
+
+                 # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+                 fsmf=os.popen("neb-ls -p "+path+"%smf")
+                 fsmfs = fsmf.readlines()
+                 fsmforig=os.popen("neb-ls -p "+path+"%smf.original")
+                 fsmforigs = fsmforig.readlines()
+                 smfCount = len(fsmfs)
+                 smfOrigCount = len (fsmforigs)
+
+                 if (smfCount > 0  and smfOrigCount > 0):
+                     #smf and original  so use original
+                     self.logger.infoPair("using:", path+"%smf.original")
+                     for i in fsmforigs:
+                         files.append(i.rstrip())
+
+                 if (smfCount > 0 and smfOrigCount == 0):
+                     self.logger.infoPair("using:", path+"%smf")
+                     #smf file and no smf.original use .smf
+                     for i in fsmfs:
+                         files.append(i.rstrip())
+
+                 #if not smf dont' find files ??)        
+                 # or not a neb path
+
+              else:
+                # this is broken for non neb - on the other hand we probably won't do .original if so
+                files = glob.glob(path + ".smf")
+
+
+
+            # if .smf and .original use .original
+            else:
+                self.logger.infoPair("camRun.state not full, can't ingest", str(camID) )
+                return None
+
 
         # XXX EAM : test output

branches/eam_branches/ipp-20130711/ippToPsps/jython/ipptopspsdb.py

@@ -1109 +1109 @@
         ,queue_OB \
         ,parallel \
+        ,P2_smf_version \
+        ,ST_cmf_version \
         FROM skychunk \
         WHERE name = '" + self.skychunk.name + "'"
@@ -1155 +1157 @@
 
             if rs.getInt(22) == 1: self.skychunk.parallel = True
+            self.skychunk.P2_smf_version = rs.getString(23)
+            self.skychunk.ST_cmf_version = rs.getString(24)
 
             if self.skychunk.parallel: print "USING parallel"

branches/eam_branches/ipp-20130711/ippToPsps/jython/queue.py

@@ -71 +71 @@
             for batchType in self.skychunk.batchTypes: 
                 self.logger.infoPair("BatchType",batchType)
-                # self.logger.infoTitle("Previous failed datastore loads")
-                # self.publishAnyUnpublishedBatches(batchType)
+                #self.logger.infoTitle("Previous failed datastore loads")
+                #self.publishAnyUnpublishedBatches(batchType)
 
                 # get totals for whole area to check if there is anything to do

branches/eam_branches/ipp-20130711/psconfig/tagsets/ipp-3.0.dist

@@ -64 +64 @@
   YYYYY  ippScripts             ipp-2-9          -0
   YYYYY  ippTasks               ipp-2-9          -0
-# YYYYY  ippToPsps              ipp-2-9          -0
+  NNYYN  ippToPsps              ipp-2-9          -0
           
   YYYYY  ippconfig              ipp-2-9          -0

branches/eam_branches/ipp-20130711/tools/fixburntool

@@ -39 +39 @@
 
 $regtool  .= " -dbname gpc1 -exp_id $exp_id -class_id $class_id";
-$burntool .= " --dbname gpc1";
+$burntool .= " --dbname gpc1 --camera GPC1";
 
 

branches/eam_branches/ipp-20130711/tools/rundiffskycell.pl

@@ -25 +25 @@
 
 my $dbname = "gpc1";
-my ($diff_id, $skycell_id, $threads, $update, $no_redirect);
+my ($diff_id, $skycell_id, $threads, $update, $no_redirect, $outdir);
 my $zaplog;
 
@@ -33 +33 @@
     'threads=i'         => \$threads,
     'no-redirect-output'  => \$no_redirect,
+    'outdir=s'          => \$outdir,
     'zaplog'            => \$zaplog,
     'update'            => \$update,
@@ -64 +65 @@
 die "path_base not found\n" if !$path_base;
 die "diff_skyfile_id\n" if !$path_base;
+
+if ($outdir) {
+    if ($outdir eq '.') {
+        $outdir = $ENV{PWD};
+    }
+    my $base = basename($path_base);
+    $path_base = "$outdir/$base";
+}
 
 my $command = "diff_skycell.pl --diff_id $diff_id --skycell_id $skycell_id --diff_skyfile_id $diff_skyfile_id --outroot $path_base --run-state new";