Changeset 33324

Timestamp:

Feb 21, 2012, 4:11:24 PM (14 years ago)

Author:

bills

Message:

More changes for detectability queries. This version works.

Location:

trunk/pstamp/scripts

Files:

• Makefile.am (modified) (1 diff)
• dquery_finish.pl (modified) (1 diff)
• dqueryparse.pl (modified) (8 diffs)
• pstamp_job_run.pl (modified) (3 diffs)

trunk/pstamp/scripts/Makefile.am

@@ -29 +29 @@
         detect_query_create \
         detect_response_create \
+        dquery_job_run.pl \
         dquery_finish.pl \
         dqueryparse.pl \

trunk/pstamp/scripts/dquery_finish.pl

@@ -143 +143 @@
 
 if (-s $reg_list) {
-    my $command = "echo Skipping $dsreg --add $req_name --product $product --list $reg_list";
+    my $command = "$dsreg --add $req_name --product $product --list $reg_list";
     $command .= " --copy --datapath $outdir";
     $command .= " --type MOPS_DETECTABILITY_RESPONSE";

trunk/pstamp/scripts/dqueryparse.pl

@@ -100 +100 @@
 
 
+#
+# query is a hash which uses $fpa_id as the key
+# The values are hash references which have the various parameter name as the key
+#   The values of the parameter hashes are arrays which contain the values for individual rows 
+#   in the detectabilty query request
 my %query = ();
-my %image_list_hash = ();
 
 {
     #
     # Parse input request file using detect_query_read
+    # result is a simple text file
     #
     my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file";
@@ -139 +144 @@
     }
     #
-    # Identify target images.  This should properly collate targets on a single imfile.
+    # Identify target components.  This should properly collate targets on a single imfile.
     #
     my $query_id;
@@ -223 +228 @@
         $option_mask |= $PSTAMP_SELECT_VARIANCE;
         $option_mask |= $PSTAMP_SELECT_PSF;
-        my $need_magic = 1;
+
+        # magic is dead
+        my $need_magic = 0;
         if ($stage eq 'stack') {
             $need_magic = 0;
@@ -245 +252 @@
         foreach my $this_image_ref (@{ $pstamp_images_ref }) {
 
-            if (0) {
-                foreach my $key (sort (keys %{ $this_image_ref } )) {
-                    my $value = ${ $this_image_ref }{$key};
-                    if ($key eq 'row_index') {
-                        $value = join ' ', @{ $this_image_ref->{$key} };
-                    }
-                    print "$this_image_ref $key $value\n";
-                }
-            }
-
-
+            # loop over the rows that this component matched
             foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
                 $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
@@ -336 +333 @@
 foreach my $fpa_id (keys %query) {
     for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
-#        print WISDOM "$fpa_id\t";
-#        foreach my $key (keys %{ $query{$fpa_id} }) {
-#            print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
-#        }
-#        print WISDOM "\n";
-#        my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
-#        if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
-#            if ($data_state ne 'full') {
-                
-#                @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
-#                    ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
-#                     $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
-#            }
-#        }
         if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
             push @{ $rows_for_component{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
@@ -358 +341 @@
 }
 
-my $job_num = 0;
+#
+# Finally build the parameter lists and queue jobs for each component
+#
+
+my $job_num = 1;
 foreach my $fpa_id (keys %rows_for_component) {
     foreach my $component (keys % {$rows_for_component{$fpa_id} } ) {
@@ -368 +355 @@
             my $stage = $query{$fpa_id}{STAGE}[$i];
             my $stage_id = $query{$fpa_id}{STAGE_ID}[$i];
-            my $component = $query{$fpa_id}{STAGE_ID}[$i];
+            my $component = $query{$fpa_id}{COMPONENT_ID}[$i];
             print "Need to UPDATE $stage $stage_id $component from $data_state\n";
+    ####                           my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_;
             $dep_id = queue_update_run($req_id, $outdir, $label, $data_state, $stage, $stage_id, $component, 0, $imagedb);
         }
 
         my $outputBase = "$outdir/$job_num" . "_";
+
+        # params_file is an mdc description containing the parameters for the job
         my $params_file = $outputBase . "params.mdc";
 
         open PARAMS, ">$params_file" or my_die("failed to create $params_file", $PS_EXIT_UNKNOWN_ERROR);
+
+        # first the job params
         print PARAMS "dqueryJobParams METADATA\n";
         foreach my $key (keys %{ $query{$fpa_id} } ) {
@@ -383 +375 @@
         }
         print PARAMS "END\n\n";
+
+        # then one structure with the coordinates for each row
         print PARAMS "dqueryCoord MULTI\n";
 
+        my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG);
         foreach my $row ( @{ $rows_for_component{$fpa_id}{$component} }) {
-            my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG);
             print PARAMS "\ndqueryCoord METADATA\n";
             foreach my $key (@coordkeys) {

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -79 +79 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
-my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
+my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
 my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
 my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
@@ -303 +303 @@
     }
 } elsif ($jobType eq "detect_query") {
-    # detect_query jobs are basically holders to note that we need a file updated before we can continue.
-    # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
-    # run it again the same way.
-    my $outdir = dirname($outputBase);
-    my $argslist = "$outdir/parse.args";
-    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
-    my $argString = <ARGSLIST>;
-    close ARGSLIST;
-    chomp $argString;
+    # my $outdir = dirname($outputBase);
+    # my $argslist = "$outdir/parse.args";
+    # open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    # my $argString = <ARGSLIST>;
+    # close ARGSLIST;
+    # chomp $argString;
 
     # XXX: should we do any other sanity checking?
-    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
-
-    my $command = "$dqueryparse --job_id $job_id $argString";
+    # my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
+    my $command = "$dquery_job_run --job_id $job_id --output_base $outputBase";
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;
@@ -329 +326 @@
     } else {
         $jobStatus = $error_code >> 8;
-        my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
+        my_die("dquery_job_run.pl failed with error code: $jobStatus", $job_id, $jobStatus);
     }
 } elsif ($jobType eq "child") {