Changeset 26246

Timestamp:

Nov 22, 2009, 11:47:44 AM (17 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/20091113

Files:

• . (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (6 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (2 diffs)
• archive/ducttape (modified) (1 prop)
• catalyst (modified) (1 prop)
• catalyst/2007.0-specs (modified) (1 prop)
• catalyst/2008.0-specs (modified) (1 prop)
• catalyst/2008.0-specs/hardened (modified) (1 prop)
• hardware (modified) (1 prop)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (1 prop)
• ippTasks/pstamp.pro (modified) (2 diffs)
• ippTests/tap (modified) (1 prop)
• psphot/doc/efficiency.txt (modified) (1 prop)
• psphot/src/psphotFake.c (modified) (1 prop)
• pstamp/scripts/pstamp_job_run.pl (modified) (8 diffs)
• pstamp/scripts/pstampparse.pl (modified) (1 diff)
• pstamp/src/pstamp.h (modified) (3 diffs)
• pstamp/test/ps1-0905001-all.txt (modified) (1 diff)

branches/eam_branches/20091113/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -166 +166 @@
     my $mask_name;
     my $weight_name;
+    my $cmf_name;
+    my $psf_name;
+    my $backmdl_name;
     my $base_name;
     my $want_astrom;
@@ -198 +201 @@
         $mask_name    = "PPIMAGE.CHIP.MASK";
         $weight_name  = "PPIMAGE.CHIP.VARIANCE";
+        $cmf_name     = "PSPHOT.OUTPUT";
+        $psf_name     = "PSPHOT.PSF.SAVE";
+        $backmdl_name = "PSPHOT.BACKMDL";
         $base_name    = "path_base"; # name of the field for the chiptool output
         $want_astrom  = 1;
@@ -209 +215 @@
         $mask_name    = "PSWARP.OUTPUT.MASK";
         $weight_name  = "PSWARP.OUTPUT.VARIANCE";
+        $cmf_name     = "PSWARP.OUTPUT.SOURCES";
+        $psf_name     = "PSPHOT.PSF.SKY.SAVE";
         $base_name    = "path_base"; # name of the field for the warptool output
     } elsif ($img_type eq "diff") {
@@ -218 +226 @@
         $mask_name   = "PPSUB.OUTPUT.MASK";
         $weight_name = "PPSUB.OUTPUT.VARIANCE";
+        $cmf_name    = "PPSUB.OUTPUT.SOURCES";
+        $psf_name     = "PSPHOT.PSF.SKY.SAVE";
         $base_name   = "path_base";
     } elsif ($img_type eq "stack") {
@@ -228 +238 @@
         $mask_name   = "PPSTACK.OUTPUT.MASK";
         $weight_name = "PPSTACK.OUTPUT.VARIANCE";
+        # this is wrong but gets the right answer. Need to figure out how to find the
+        # rule properly
+        #$cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
+        $cmf_name     = "PSWARP.OUTPUT.SOURCES";
+        $psf_name    = "PPSTACK.TARGET.PSF";
         $base_name   = "path_base";
     } else {
@@ -340 +355 @@
         $out->{image_db} = $image_db;
 
-        # find the mask and weight images
         if ($base) {
             $out->{image}  = $ipprc->filename($image_name,  $base, $class_id) if $image_name;
             $out->{mask}   = $ipprc->filename($mask_name,   $base, $class_id) if $mask_name;
             $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name;
+            $out->{cmf}    = $ipprc->filename($cmf_name, $base, $class_id) if $cmf_name;
+            $out->{psf}    = $ipprc->filename($psf_name, $base, $class_id) if $psf_name;
+            $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name;
         }
         $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom;

branches/eam_branches/20091113/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -22 +22 @@
                     $PSTAMP_SELECT_MASK
                     $PSTAMP_SELECT_WEIGHT
+                    $PSTAMP_SELECT_CMF
+                    $PSTAMP_SELECT_PSF
+                    $PSTAMP_SELECT_BACKMDL
                     $PSTAMP_SELECT_INVERSE
                     $PSTAMP_WAIT_FOR_UPDATE
@@ -48 +51 @@
 our $PSTAMP_SELECT_MASK      = 2;
 our $PSTAMP_SELECT_WEIGHT    = 4;
+our $PSTAMP_SELECT_CMF       = 8;
+our $PSTAMP_SELECT_PSF       = 16;
+our $PSTAMP_SELECT_BACKMDL   = 32;
+our $PSTAMP_SELECT_INVERSE   = 1024;
 
-our $PSTAMP_SELECT_INVERSE   = 1024;
 our $PSTAMP_WAIT_FOR_UPDATE  = 2048;
 

branches/eam_branches/20091113/ippTasks/pstamp.pro

@@ -421 +421 @@
         book getword pstampJob $pageName dbname -var DBNAME
         book getword pstampJob $pageName outputBase -var OUTPUT_BASE
+        book getword pstampJob $pageName options -var OPTIONS
 
         if ($VERBOSE > 1) 
@@ -428 +429 @@
         host anyhost
 
-        $run = pstamp_job_run.pl --job_id $JOB_ID --job_type $JOB_TYPE --rownum $ROWNUM --output_base $OUTPUT_BASE --redirect-output 
+        $run = pstamp_job_run.pl --job_id $JOB_ID --job_type $JOB_TYPE --rownum $ROWNUM --output_base $OUTPUT_BASE --options $OPTIONS --redirect-output 
         add_standard_args run
 

branches/eam_branches/20091113/pstamp/scripts/pstamp_job_run.pl

@@ -13 +13 @@
 use Carp;
 use File::Basename;
+use File::Copy;
 use Digest::MD5::File qw( file_md5_hex );
 use PS::IPP::PStamp::RequestFile qw( :standard );
@@ -20 +21 @@
 use PS::IPP::Metadata::Config;
 #use PS::IPP::Metadata::Stats;
-#use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Metadata::List qw( parse_md_list );
 
 use PS::IPP::Config qw( :standard );
 
-my ($job_id, $redirect_output, $outputBase, $rownum, $jobType); 
+my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
 my ($verbose, $dbname, $dbserver, $no_update);
 
@@ -32 +33 @@
     'rownum=s'          =>  \$rownum,
     'output_base=s'     =>  \$outputBase,
+    'options=s'         =>  \$options,
     'redirect-output'   =>  \$redirect_output,
     'dbname=s'          =>  \$dbname,
@@ -51 +53 @@
 my_die("rownum is required", $job_id, $PS_EXIT_PROG_ERROR) if !$rownum;
 my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
+
+$options = 1 if !$options;
 
 
@@ -107 +111 @@
         my $reglist = "$dir/reglist$job_id";
 
-        open F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+        my $F;
+        open $F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
 
         # Figure out what output images were produced
@@ -116 +121 @@
                            $PSTAMP_SELECT_WEIGHT => "wt.fits");
 
-        # we always create a stamp of the image
-        my $output_mask = $PSTAMP_SELECT_IMAGE;
-
-        # we search the argString for -mask and -variance. 
-        # searching the arg string allows us to avoid adding a column in pstampJob
-        $output_mask   |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
-        $output_mask   |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-variance/);
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_WEIGHT);
 
         foreach my $key (keys (%extensions)) {
@@ -137 +136 @@
 
             # XXX is pstamp always the right file type, if not where do we get the right one?
-            print F file_registration_line($filename, $path, "pstamp") . "\n";
-        }
-
-        close F;
+            print $F file_registration_line($filename, $path, "pstamp") . "\n";
+        }
+
+        get_other_outputs($F, $outputBase, $options);
+
+        close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
     } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
-        $jobStatus = $PSTAMP_NO_OVERLAP
+        $jobStatus = $PSTAMP_NO_OVERLAP;
     } else {
         my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
@@ -206 +207 @@
     }
 }
+        
+sub get_other_outputs {
+    my $f = shift;
+    my $output_base = shift;
+    my $options = shift;
+
+    if ($options & ( $PSTAMP_SELECT_CMF | $PSTAMP_SELECT_PSF | $PSTAMP_SELECT_BACKMDL)) {
+if (0) {
+        my $params_file = $output_base . ".mdc";
+        open (IN, "<$params_file") 
+            or my_die("failed to open params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $data = $mdcParser->parse(join "", (<IN>))
+            or my_die("failed to parse params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $components = parse_md_list($data);
+
+        my $n = scalar @$components;
+        if ($n != 1) {
+            my_die("params file $params_file contains unexpected number of components: $n",
+                $job_id, $PS_EXIT_PROG_ERROR);
+        }
+        my $comp = $components->[0];
+}
+        my $comp = read_params_file($output_base);
+
+        my $stage = $comp->{stage};
+
+        # raw files don't have any other data products
+        return 1 if $stage eq "raw";
+
+        # add the other data products if they are selected and exist.
+        # silently skip them if they don't exist. Perhaps this should be
+        # detected in pstampparse so that the user can be notified with 
+        # a message in parse_error.txt ("warp do not have a background model")
+        my $cmf_file = $comp->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+        my $psf_file = $comp->{psf} if ($options & $PSTAMP_SELECT_PSF);
+        my $backmdl_file = $comp->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
+
+        my $outdir = dirname($output_base);
+        my $basename = basename($output_base);
+        my ($rownum, $jobnum, $therest) = split /_/, $basename;
+        &my_die("failed to split basename: $basename", $job_id, $PS_EXIT_CONFIG_ERROR) 
+            if (!$therest or !$rownum or !$jobnum);
+
+        my $prefix = "${rownum}_${jobnum}_";
+
+        if ($cmf_file) {
+            print "cmf file is $cmf_file\n";
+            copy_and_register_file($f, $cmf_file, $outdir, $prefix);
+        }
+        if ($psf_file) {
+            print "psf_file is $psf_file\n";
+            copy_and_register_file($f, $psf_file, $outdir, $prefix);
+        }
+        if ($backmdl_file) {
+            print "backmdl_file is $backmdl_file\n";;
+            copy_and_register_file($f, $backmdl_file, $outdir, $prefix);
+        }
+    }
+}
+sub read_params_file {
+    my $output_base = shift;
+
+    my $params_file = $output_base . ".mdc";
+    open (IN, "<$params_file") 
+        or my_die("failed to open params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $data = $mdcParser->parse(join "", (<IN>))
+        or my_die("failed to parse params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $components = parse_md_list($data);
+
+    my $n = scalar @$components;
+    if ($n != 1) {
+        my_die("params file $params_file contains unexpected number of components: $n",
+                $job_id, $PS_EXIT_PROG_ERROR);
+    }
+    return $components->[0];
+}
+
+# copy_and_register_file ($f, $src, $destdir, $prefix);
+sub copy_and_register_file {
+    my $F = shift;
+    my $src = shift;
+    my $destdir = shift;
+    my $prefix = shift;
+
+    my $fn = $prefix . basename($src);
+    my $dst = "$destdir/$fn";
+
+    my $resolved = $ipprc->file_resolve($src);
+
+    my_die("failed to resolve $src", $job_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    copy($resolved, $dst) or my_die("failed to copy $resolved to $dst", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    print $F file_registration_line($fn, $dst, "fits") . "\n";
+}
 
 sub my_die

branches/eam_branches/20091113/pstamp/scripts/pstampparse.pl

@@ -371 +371 @@
             $args .= " -astrom $image->{astrom}" if $image->{astrom};
         }
+
+        $image->{job_args} = $args;
+
+        # XXX: we can get rid of the following everything that we need is
+        # in the params file
 
         $args .= " -file $imagefile";

branches/eam_branches/20091113/pstamp/src/pstamp.h

@@ -3 +3 @@
 
 // error codes returned to users in results flie
-// These must match the values in  PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm
+// PS-IPP-PStamp::RequestFile
+// These must match the values in the perl module PS-IPP-PStamp::RequestFile
+// i.e. PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm
+
 typedef enum {
         PSTAMP_SUCCESS          = 0,
@@ -21 +24 @@
 
 
-#define PSTAMP_SELECT_IMAGE  1
-#define PSTAMP_SELECT_MASK   2
-#define PSTAMP_SELECT_WEIGHT 4
+// values for options mask.
+#define PSTAMP_SELECT_IMAGE      1
+#define PSTAMP_SELECT_MASK       2
+#define PSTAMP_SELECT_WEIGHT     4
+#define PSTAMP_SELECT_CMF        8
+#define PSTAMP_SELECT_PSF       16
+#define PSTAMP_SELECT_BACKMDL   32
 #define PSTAMP_SELECT_INVERSE 1024
+
+#define PSTAMP_WAIT_FOR_UPDATE 2048
 
 #define PSTAMP_CENTER_IN_PIXELS 1
@@ -35 +44 @@
 #define STAMP_RESULTS_VERSION "1"
 
+// end of values tha must match PS-IPP-PStamp::RequestFile
+
 #endif

branches/eam_branches/20091113/pstamp/test/ps1-0905001-all.txt

@@ -22 +22 @@
 #
 # various looks at the location of SN candidate PS1-0905001
-1         243.035580     55.128140          250   250      2        stamp       7        gpc1    byid     raw   70853         null   null       null        null    0      0    |PS1-0905001 r
-2         243.035580     55.128140          250   250      2        stamp       7        gpc1    byid     chip  20272         null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
+1         243.035580     55.128140          250   250      2        stamp       63        gpc1    byid     raw   70853         null   null       null        null    0      0    |PS1-0905001 r
+2         243.035580     55.128140          250   250      2        stamp       63       gpc1    byid     chip  20272         null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
 
 # 3 and 4 will fail because the weight images was damaged by the destreaking
 # process
-3         243.035580     55.128140          250   250      2        stamp       7        gpc1    byid     warp  6323          null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
-4         243.035580     55.128140          250   250      2        stamp       7        gpc1    byid     diff  16880         null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
+3         243.035580     55.128140          250   250      2        stamp       63       gpc1    byid     warp  6323          null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
+4         243.035580     55.128140          250   250      2        stamp       63       gpc1    byid     diff  16880         null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
 # these will work
-103       243.035580     55.128140          250   250      2        stamp       3        gpc1    byid     warp  6323          null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
-104       243.035580     55.128140          250   250      2        stamp       3        gpc1    byid     diff  16880         null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
+103       243.035580     55.128140          250   250      2        stamp       59       gpc1    byid     warp  6323          null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
+104       243.035580     55.128140          250   250      2        stamp       59       gpc1    byid     diff  16880         null   null  MD08.200905.v1   null    0      0    |PS1-0905001 r
 
-5         243.035580     55.128140          250   250      2        stamp       7        gpc1    byid     stack 14032         null   null  null             null    0      0    |PS1-0905001 r
+5         243.035580     55.128140          250   250      2        stamp       63       gpc1    byid     stack 14032         null   null  null             null    0      0    |PS1-0905001 r
 
 6         243.035580     55.128140          250   250      2        stamp       1        gpc1    byexp    raw   o4973g0133o   null   null       null        null    0      0    |PS1-0905001 r