Changeset 35909

Timestamp:

Aug 8, 2013, 9:49:03 AM (13 years ago)

Author:

bills

Message:

extract photometric calibration information from the database and include
it in the fits headers of warp and chip stage postage stamp images

Location:

tags/ipp-20130712

Files:

• PS-IPP-PStamp (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (11 diffs)
• ippTools/share/releasetool_listrelexp.sql (modified) (1 diff, 1 prop)
• ippTools/share/warptool_warped.sql (modified) (1 diff, 1 prop)
• ippTools/src/releasetool.c (modified) (1 diff)
• pstamp (modified) (1 prop)
• pstamp/scripts/Makefile.am (modified) (1 diff)
• pstamp/scripts/psgetcalibinfo (copied) (copied from trunk/pstamp/scripts/psgetcalibinfo )
• pstamp/scripts/pstamp_job_run.pl (modified) (6 diffs)
• pstamp/src/ppstampArguments.c (modified) (2 diffs)
• pstamp/src/ppstampMakeStamp.c (modified) (1 diff)
• pstamp/src/ppstampOptions.c (modified) (2 diffs)
• pstamp/src/ppstampOptions.h (modified) (1 diff)

tags/ipp-20130712/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -176 +176 @@
         $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
         0, 0,   # fwhm cuts are not applied here
-        undef, undef, undef, $verbose);
+        undef, 0, # no cam run information
+        undef, undef, $verbose);
 
     return $results;
@@ -322 +323 @@
         0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
         getValOrZero($row->{FWHM_MIN}), getValOrZero($row->{FWHM_MAX}),
-        undef, $release_name, $survey,
+        undef, 0, $release_name, $survey,
         $verbose);
 
@@ -351 +352 @@
     my $fwhm_max = shift;
     my $selectedAstrom = shift;
+    my $selected_cam_id = shift;
     my $release_name = shift;
     my $survey = shift;
@@ -602 +604 @@
         # The image selectors are such that multiple runs my have be returned for the same exposure.
         # Return only the latest one.
-        $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
+        $images = filterRuns($stage, $choose_components, $need_magic, $images, $inverse, $verbose);
     }
 
@@ -699 +701 @@
                 $out->{astrom} = $selectedAstrom;
                 $out->{cam_path_base} = $selectedAstrom;
+                $out->{cam_id} = $selected_cam_id if $selected_cam_id;
                 if ($selectedAstrom =~ /\.smf$/) {
                     $out->{cam_path_base} =~ s/\.smf$//;
@@ -946 +949 @@
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
                     $fwhm_min, $fwhm_max,
-                    $chip->{astrom}, undef, undef,
+                    $chip->{astrom}, $chip->{cam_id}, undef, undef,
                     $verbose);
 
@@ -970 +973 @@
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
                     $fwhm_min, $fwhm_max,
-                    undef, $release_name, $survey,
+                    undef, 0, $release_name, $survey,
                     $verbose);
 
@@ -1302 +1305 @@
     $image->{astrom} = $astromFile;
     $image->{cam_path_base} = $camRoot;
+    $image->{cam_id} = $camRun->{cam_id};
 
     return 1;
@@ -1649 +1653 @@
 sub filterRuns {
     my $stage      = shift;
+    my $drop_duplicate_runs = shift;
     my $need_magic = shift;
     my $inputs     = shift;
@@ -1694 +1699 @@
         }
 
-        $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
+        # $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
 
         # skip if we need magicked run and this one has never been magicked
@@ -1702 +1707 @@
         }
 
-        if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) {
+        if (($exp_id == $last_exp_id) and ($drop_duplicate_runs || ($run_id != $last_run_id))) {
             print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed;
             next;

tags/ipp-20130712/ippTools/share/releasetool_listrelexp.sql

@@ -15 +15 @@
     relExp.zpt_stdev,
     relExp.mcal,
+    relExp.ubercal_dist,
     relExp.path_base,
     relExp.registered,

tags/ipp-20130712/ippTools/share/warptool_warped.sql

@@ -13 +13 @@
     rawExp.decl,
     rawExp.exp_time,
+    camRun.cam_id,
     magicDSRun.state AS dsRun_state,
     IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id

tags/ipp-20130712/ippTools/src/releasetool.c

@@ -778 +778 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC");
+        psStringAppend(&query, "\nAND priority > 0 ORDER BY exp_id, priority DESC");
+    } else {
+        psStringAppend(&query, "\nORDER BY exp_id, relexp_id DESC");
     }
 

tags/ipp-20130712/pstamp/scripts/Makefile.am

@@ -21 +21 @@
         pstamp_get_image_job.pl \
         psmkreq \
+        psgetcalibinfo \
         psstatus \
         pstampstopfaulted \

tags/ipp-20130712/pstamp/scripts/pstamp_job_run.pl

@@ -78 +78 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $psgetcalibinfo    = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
 my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
-my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
-my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
 
@@ -100 +99 @@
     # XXX: should we do any other sanity checking?
     my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
-
-    my $nan_masked = 1;
-    my $muggle = 0;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
-        # Attempt to find or create a muggle image
-        $nan_masked = 0;
-        $muggle = 1;
-    }
 
     if ($stage eq "raw") {
@@ -122 +111 @@
     my @file_list = ($params->{image});
     
-    if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
+    if ($options & $PSTAMP_SELECT_MASK) {
         $mask = $params->{mask};
         $fileArgs .= " -mask $mask";
@@ -162 +151 @@
     # find our output directory
     my $outdir = dirname($outputBase);
-    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
-    
-    if ($muggle) {
-        # first see if the original uncensored images are around
-        if (check_for_backups($params, \$fileArgs)) {
-            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
-            print "Making stamps from backup images\n";
-        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
-            # user required uncensored but since stage isn't chip we can't rebuild them
-            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
-        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
-            # we can only restore pixels for chip stage images if the data has been updated.
-            # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
-            # XXX: we should probably be looking explicitly at the job and checking for a dep_id
-            print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
-            # make stamps from uncensored images
-            $muggle = 0;
+
+    my ($calib_fd, $calibfile);
+    if ($stage eq 'chip' or $stage eq 'warp') {
+        my $cam_id = $params->{cam_id};
+        if (!$cam_id) {
+            carp "no cam_id found in job params\n";
+            exit $PS_EXIT_PROG_ERROR;
+        }
+        ($calib_fd, $calibfile) = tempfile ("$outdir/calib.XXXX", UNLINK => !$save_temps);
+        close $calib_fd;
+
+        my $command = "$psgetcalibinfo --cam_id $cam_id --output $calibfile --dbname $params->{imagedb}";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        my $exitStatus;
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
         } else {
-            # Try and replace the streaks from the recovery images
-
-            my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR);
-            $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps);
-            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
-            # find the "directory" of the input path_base
-            my $inputdir = dirname($image);
-            my $base = basename($image);
-            $tmpImage = "$tmproot/$base";
-
-            @file_list = ();
-
-            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
-            # stage work. Import the rule here.
-            my $recImage = "$inputdir/REC_$base";
-            my $newFileArgs = " -file $tmpImage";
-            $muggle_command .= " -image $image -recimage $recImage";
-            push @file_list, $recImage;
-
-            if ($mask) {
-                $base = basename($mask);
-                $tmpMask = "$tmproot/$base";
-                $newFileArgs .= " -mask $tmpMask";
-                my $recMask = "$inputdir/REC_$base";
-                $muggle_command .= " -mask $mask -recmask $recMask";
-                push @file_list, $recMask;
-            }
-
-            if ($variance) {
-                $base = basename($variance);
-                $tmpVariance = "$tmproot/$base";
-                $newFileArgs .= " -variance $tmpVariance";
-                my $recVariance = "$inputdir/REC_$base";
-                $muggle_command .= " -weight $variance -recweight $recVariance";
-                push @file_list, $recVariance;
-            }
-            if (check_files(0, @file_list)) {
-                # recovery files exist and are accessible restore the excised pixels
-
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $muggle_command, verbose => $verbose);
-                unless ($success) {
-                    my $exitStatus = WEXITSTATUS($error_code);
-                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
-                }
-
-                # set the ppstamp file arguments
-                $fileArgs = $newFileArgs;
-            } else {
-                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
-                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
-                }
-                # just make stamps from the censored images
-                print "Unable to restore uncensored images, will extract stamps from censored images\n";
-                # these files won't be used so zap them
-                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
-            }
-        }
-    }
-
-    # MAGIC IS DEAD
-    $nan_masked = 0;
+            print STDERR "psgetcalibinfo failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        exit $exitStatus if $exitStatus;
+
+        if (-s $calibfile == 0) {
+            print "no calibration information found for $cam_id\n";
+            $calibfile = undef;
+        }
+    }
+
+    # my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
 
     my $command = "$ppstamp $outputBase $argString $fileArgs";
@@ -247 +187 @@
     $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
     $command .= " -stage $stage";
-    $command .= " -censor_masked" if $nan_masked;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
+    $command .= " -forheader $calibfile" if $calibfile;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -523 +461 @@
 }
 
-sub check_for_backups {
-    my $params = shift;
-    my $r_fileArgs = shift;
-
-    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
-    my $results = runToolAndParse($command, $verbose);
-    my $dsComponent  = $results->[0];
-
-    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
-
-        print "magicDSFile state is full. Backup images should exist\n";
-
-        # replace the file names with the backup paths
-        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
-        my @args = split " ", $$r_fileArgs;
-
-        my $newFileArgs;
-        while (@args) {
-            my $a = shift @args;
-            my $f = shift @args;
-            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
-
-            my $backup_image_name = $ipprc->destreaked_filename($f);
-            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
-            $newFileArgs .= " $a $backup_image_name";
-        }
-        $$r_fileArgs = $newFileArgs;
-
-        return 1;
-    }
-
-    return 0;
-}
-
 my $neb;
 sub storage_object_exists

tags/ipp-20130712/pstamp/src/ppstampArguments.c

@@ -26 +26 @@
     fprintf(stderr, "   [-variance var_image] : variance image\n");
     fprintf(stderr, "   [-sources sources]    : sources cmf (ignored for chip stage)\n");
+    fprintf(stderr, "   [-forheader mdcfile   : metadata config file contaiing keywords to add to fits headers\n");
     fprintf(stderr, "   [-stage stage]        : stage of input image (raw, chip, warp, stack, diff)\n");
     fprintf(stderr, "   [-write_jpeg]         : write a JPEG  format of the image stamp\n");
@@ -135 +136 @@
         usage();
     }
+
+    if ((argnum = psArgumentGet(argc, argv, "-forheader"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        if (argnum >= argc) {
+            psError(PSTAMP_ERR_ARGUMENTS, true, "metadata config filename value required with option -forheader");
+            usage();
+        }
+
+        const char *filename = argv[argnum];
+        psArgumentRemove(argnum, &argc, argv);
+
+        unsigned int nFail = 0;
+        options->headerAdditions = psMetadataConfigRead(NULL, &nFail, filename, false);
+        if (nFail || !options->headerAdditions) {
+            psError(PS_ERR_UNKNOWN, false, "failed to read metadata from %s", filename);
+            return false;
+        }
+    }
     
     if ((argnum = psArgumentGet(argc, argv, "-no_censor_masked"))) {

tags/ipp-20130712/pstamp/src/ppstampMakeStamp.c

@@ -165 +165 @@
 
     ppstampVersionMetadata(outHDU->header, options);
+
+    // copy any user supplied keywords
+    if (options->headerAdditions) {
+        if (!psMetadataOverlay(outHDU->header, options->headerAdditions)) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to copy header additions to output\n");
+            return false;
+        }
+    }
+
 
     return true;

tags/ipp-20130712/pstamp/src/ppstampOptions.c

@@ -10 +10 @@
 {
     psFree(options->chipName);
+    psFree(options->headerAdditions);
 }
 
@@ -30 +31 @@
     options->cellName  = NULL;
     options->stage  = NULL;
+    options->headerAdditions = NULL;
     options->censorMasked = false;
     options->writeJPEG = false;

tags/ipp-20130712/pstamp/src/ppstampOptions.h

@@ -12 +12 @@
     psString    cellName;
     psString    stage;
+    psMetadata  *headerAdditions;
     bool        censorMasked;
     bool        writeJPEG;