Changeset 37544

Timestamp:

Oct 31, 2014, 5:03:04 PM (12 years ago)

Author:

watersc1

Message:

Changes to support addition of staticsky processing at lanl.

Location:

tags/ipp-pv3-20140717

Files:

• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/MANIFEST (modified) (1 diff)
• ippScripts/scripts/sc_mk_stack_mdc.pl (modified) (1 diff)
• ippScripts/scripts/sc_prepare_run.pl (modified) (4 diffs)
• ippScripts/scripts/sc_prepare_staticsky.pl (modified) (8 diffs)
• ippTasks/remote.pro (modified) (2 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/remotetool_definebyquery_staticsky.sql (added)
• ippTools/src/remotetool.c (modified) (1 diff)

tags/ipp-pv3-20140717/ippScripts/Build.PL

@@ -138 +138 @@
         scripts/stack_bkg_mk_mdc.pl
         scripts/sc_mk_stack_mdc.pl
+        scripts/sc_mk_staticsky_mdc.pl
         scripts/sc_prepare_camera.pl
         scripts/sc_prepare_chip.pl
         scripts/sc_prepare_stack.pl
+        scripts/sc_prepare_staticsky.pl
         scripts/sc_prepare_warp.pl
         scripts/sc_prepare_run.pl

tags/ipp-pv3-20140717/ippScripts/MANIFEST

@@ -51 +51 @@
 scripts/regenerate_background.pl
 scripts/sc_mk_stack_mdc.pl
+scripts/sc_mk_staticsky_mdc.pl
 scripts/sc_prepare_camera.pl
 scripts/sc_prepare_chip.pl
 scripts/sc_prepare_stack.pl
+scripts/sc_prepare_staticsky.pl
 scripts/sc_prepare_warp.pl
 scripts/sc_remote_exec.pl

tags/ipp-pv3-20140717/ippScripts/scripts/sc_mk_stack_mdc.pl

@@ -72 +72 @@
     if (! ((-e $cfile)&&(-s $cfile))) { # This cfile does not exist or is zero size.
         my $disk = $component_map{$cfile};
-        $cfile = "${remote_root}/tmp/${disk}";
+        $cfile = "${disk}";
         if (! ((-e $cfile)&&(-s $cfile))) { # The other expected cfile did not exist.
-            $cfile = '';
+#           $cfile = '';
             $error++;
-            die "Missing cfile!";
+            die "Missing cfile $cfile!";
         }
     }

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl

@@ -36 +36 @@
 $job_cost{"warp"}   = 110 / 60 / 60;
 $job_cost{"stack"}  = 1500 / 60 / 60;
+$job_cost{"staticsky"} = 2; # really?  avg time of 5720s?
 
 # This object holds how "expensive" a given job is in terms of cores on a node.  
@@ -44 +45 @@
 $job_subscription{"warp"}   = 1;
 $job_subscription{"stack"}  = 3;
+$job_subscription{"staticsky"} = 3;
 
 my $proc_per_node = 24;                     # processors per node
@@ -235 +237 @@
 }
 
-if ($stage eq "stack") {
+# Most jobs only take 1 core, but ensure that we split them appropriately
+$node_req *= $job_subscription{$stage};
+
+if (($stage eq "stack")||($stage eq "staticsky")) {
     $time_req += 2;
-    $node_req *= $job_subscription{$stage};
 } else {
     $time_req += 1; # Safety addition.
@@ -251 +255 @@
 print COMMAND "date\n";
 print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-if ($stage eq "stack") {
+if (($stage eq "stack")||($stage eq "staticsky")) {
     print COMMAND "${remote_root}/stask_stack.py $remote_config " . 'hosts.${SLURM_JOB_ID} '  . $job_subscription{$stage} . "\n";
 }

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_staticsky.pl

@@ -37 +37 @@
 }
 
-my ($remote_id,$stack_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$sky_id,$camera,$dbname,$verbose,$path_base,$no_update);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -60 +60 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my @return_component_list = ("LOG.EXP", 
-                             "PSPHOT.CHISQ.IMAGE","PSPHOT.CHISQ.MASK","PSPHOT.CHISQ.VARIANCE");
+my @return_component_list = ("LOG.EXP","PSPHOT.STACK.CONFIG","DBINFO.EXP");
+                             
+                         #    "PSPHOT.CHISQ.IMAGE","PSPHOT.CHISQ.MASK","PSPHOT.CHISQ.VARIANCE");
                              
-my @return_component_list_per_input = ("PSPHOT.STACK.OUTPUT.IMAGE","PSPHOT.STACK.OUTPUT.MASK","PSPHOT.STACK.OUTPUT.VARIANCE","PSPHOT.STACK.OUTPUT");
+my @return_component_list_per_input = (#"PSPHOT.STACK.OUTPUT.IMAGE","PSPHOT.STACK.OUTPUT.MASK","PSPHOT.STACK.OUTPUT.VARIANCE","PSPHOT.STACK.OUTPUT");
+                                       "PSPHOT.STACK.OUTPUT",
+                                       "PSPHOT.STACK.PSF.SAVE",
+                                       "PSPHOT.STACK.BACKMDL");
 
 # STEP 0: Open output files
@@ -91 +95 @@
 my ($reduction, $workdir, $tess_id, $skycell_id);
 {
-    my $command = "$staticskytool -pendingskycalrun -sky_id $sky_id";
+    my $command = "$staticskytool -todo -sky_id $sky_id";
     $command .= " -dbname $dbname " if defined($dbname);
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -151 +155 @@
         my $remote_path_base = &uri_local_to_remote($input_path_base);
         $input_path_base_string .= " $remote_path_base ";
-        $input_path_base_string .= " $stack_id ";
+        $input_path_base_string .= " $input_stack_id ";
         if ($verbose) { print STDERR "$input_path_base_string\n"; }
 
@@ -170 +174 @@
         ($ipp_disk,undef)    = &uri_convert($input_path_base . ".unconv.num.fits");
         print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
-        (undef,$remote_file) = &uri_to_outputs($input_path_base . ".cmf");
-        ($ipp_disk,undef)    = &uri_convert($input_path_base . ".cmf");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
-        (undef,$remote_file) = &uri_to_outputs($input_path_base . ".psf");
-        ($ipp_disk,undef)    = &uri_convert($input_path_base . ".psf");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+#         (undef,$remote_file) = &uri_to_outputs($input_path_base . ".cmf");
+#       ($ipp_disk,undef)    = &uri_convert($input_path_base . ".cmf");
+#       print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+#       (undef,$remote_file) = &uri_to_outputs($input_path_base . ".psf");
+#       ($ipp_disk,undef)    = &uri_convert($input_path_base . ".psf");
+#       print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
 
         $nInputs++;
@@ -188 +192 @@
 
     my $pps_command  = " psphotStack -input ${remote_outroot}.in.mdc ";
-    $pps_command    .= " ${remote_outroot} -stats ${remote_outroot}.stats ";
+    $pps_command    .= " ${remote_outroot} ";
     $pps_command    .= " -recipe PSPHOT $recipe_psphot ";
     $pps_command    .= " -recipe PPSUB  $recipe_ppsub ";
@@ -194 +198 @@
     $pps_command    .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
     $pps_command    .= " -threads $threads_req ";
-    $pps_command    .= " -dumpconfig ${remote_outroot}.mdc ";
+    $pps_command    .= " -dumpconfig ${remote_outroot}.psphotStack.mdc ";
 
     my $post_cmd_echo = " echo -n \"staticskytool  -addresult -sky_id $sky_id ";
-    $post_cmd_echo   .= " -path_base $ipp_outroot -uri UNKNOWN -num_inputs $nInputs ";
+    $post_cmd_echo   .= " -path_base $ipp_outroot -num_inputs $nInputs ";
     $post_cmd_echo   .=  " -dbname $dbname " if defined $dbname;
     $post_cmd_echo   .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo ";
@@ -220 +224 @@
         foreach my $component (@return_component_list_per_input) {
             my $filename = $ipprc->filename($component,$ipp_outroot,$stack_id);
-            my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_retrun( $filename );
+            my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename );
             my $remote_outroot_dir = dirname($ipp_disk);
             print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";

tags/ipp-pv3-20140717/ippTasks/remote.pro

@@ -26 +26 @@
 #  warp
   stack
+  staticsky
 end
 
@@ -291 +292 @@
       $command = sc_prepare_stack.pl --camera GPC1 --stack_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
     end
+    if ("$STAGE" == "staticsky")
+      $command = sc_prepare_staticsky.pl --camera GPC1 --sky_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot -dbname $DBNAME
+    end
 
     options $pageName

tags/ipp-pv3-20140717/ippTools/share/Makefile.am

@@ -508 +508 @@
         remotetool_definebyquery_warp.sql \
         remotetool_definebyquery_stack.sql \
+        remotetool_definebyquery_staticsky.sql \
         remotetool_dropcomponent.sql \
         remotetool_listcomponent.sql \

tags/ipp-pv3-20140717/ippTools/src/remotetool.c

@@ -144 +144 @@
   else if (!strcmp(stage,"stack")) {
     query = pxDataGet("remotetool_definebyquery_stack.sql");
+    if (!query) {
+      psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");
+      return(false);
+    }
+
+    if (label) {
+      psStringAppend(&whereOption, "\n AND (label = '%s')", label);
+    }
+  }
+  else if (!strcmp(stage,"staticsky")) {
+    query = pxDataGet("remotetool_definebyquery_staticsky.sql");
     if (!query) {
       psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");