Changeset 34772 for branches/czw_branch/20120906/pstamp

Timestamp:

Dec 6, 2012, 6:32:19 PM (14 years ago)

Author:

watersc1

Message:

merge from trunk. Preliminary versions of stacktool/warptool updates and regenerate_background.pl script. The warp code is finished and tested, and I still need to get the stack version resolved.

Location:

branches/czw_branch/20120906/pstamp

Files:

• scripts (modified) (1 prop)
• scripts/dquery_finish.pl (modified) (2 diffs)
• scripts/pstamp_checkdependent.pl (modified) (5 diffs)
• scripts/pstamp_finish.pl (modified) (2 diffs)
• scripts/pstamp_request_file (modified) (8 diffs)
• scripts/pstamp_targetlist_file.pl (copied) (copied from trunk/pstamp/scripts/pstamp_targetlist_file.pl )
• scripts/pstampparse.pl (modified) (6 diffs)
• src/ppstampArguments.c (modified) (4 diffs)
• src/ppstampMakeStamp.c (modified) (15 diffs)
• src/ppstampOptions.c (modified) (1 diff)
• src/ppstampOptions.h (modified) (1 diff)
• src/ppstampParseCamera.c (modified) (1 diff)
• src/pstamp.h (modified) (1 diff)

branches/czw_branch/20120906/pstamp/scripts/dquery_finish.pl

@@ -128 +128 @@
     my $job_id = $job->{job_id};
 
+    if ($job->{fault}) {
+        # how do we get this information back to the user?
+        print STDERR "job: $job_id faulted with $job->{fault}\n";
+        next;
+    }
     my $outputBase = $job->{outputBase};
     my $results = "${outputBase}results.txt";
@@ -166 +171 @@
     my $results = shift;
 
-    open IN, "<$results_file" or my_die("failed to open $results_file", $PS_EXIT_UNKNOWN_ERROR);
+    open IN, "<$results_file" or my_die("failed to open $results_file\n", $PS_EXIT_UNKNOWN_ERROR);
 
     foreach my $line (<IN>) {

branches/czw_branch/20120906/pstamp/scripts/pstamp_checkdependent.pl

@@ -137 +137 @@
         print "skipping $command\n";
     }
-} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) {
+} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'goto_cleaned') or ($it->{state} eq 'update')) {
     #       For warp and diff stages we need to call the 'queue_update' subroutines even if the 
     #       data_state is update in order to check the state of inputs in earlier stages in the pipeline
@@ -260 +260 @@
             # caller will fault the jobs
             return $error_code;
+        } elsif ($chip->{state} eq 'goto_cleaned') {
+
+            # we don't need to check for full data_state
+            # if ($chip->{data_state} eq 'full') {
+            {
+                # cleanup must not be running. Set state to update. If this chip is not 'full' it will be
+                # set to be updated the next time this script is invoked
+                # XXX: In the very rare case that this chip run is actually in the running cleanup pantasks
+                # queue things may get confused but we can live with that
+                my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+                $command .= " -set_label $rlabel" if $rlabel;
+
+                if (!$no_update) {
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                                run(command => $command, verbose => $verbose);
+                    unless ($success) {
+                        my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+                    }
+                } else {
+                    print "skipping $command\n";
+                }
+                $queued_update = 1;
+            }
         } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
 
@@ -320 +343 @@
     }
 }
+
+    my $tess_id = $metadata->{tess_id};
+    if ($tess_id eq 'RINGS.V0') {
+        # we have stopped updating data from RINGS.V0
+        return $PSTAMP_GONE
+    }
     if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
          ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
@@ -343 +372 @@
 
     my $skycell = $metadata;
+
+    # if ($skycell->{state} eq 'goto_cleaned' and $skycell->{data_state} eq 'full') {
+    if ($skycell->{state} eq 'goto_cleaned') {
+        # cleanup has been queued, but hasn't finished, probably due to an error or cleanup is not running.
+        # It's "safe" to set the state to update. If the skycell is not full it will be set to update
+        # the next time this task runs
+        my $command = "$warptool -updaterun -set_state update -warp_id $warp_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to change state of ${stage}Run $stage_id from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+        return 0;
+    }
 
     # get the list of input chips for this skycell
@@ -428 +477 @@
     my $skycell = $metadata;
     my $skycell_id = $skycell->{skycell_id};
+
+    # if ($metadata->{state} eq 'goto_cleaned' and $metadata->{data_state} eq 'full') {
+    if ($metadata->{state} eq 'goto_cleaned') {
+        # cleanup must not be running. Set state to update. If the skycell is not 'full' it will be
+        # set to be updated the next time this script is invoked
+        print "changing state of diffRun $diff_id from goto_cleaned to update. skycell $skycell_id is full\n";
+        my $command = "$difftool -updaterun -set_state update -diff_id $diff_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+        return 0;
+    }
 
     if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {

branches/czw_branch/20120906/pstamp/scripts/pstamp_finish.pl

@@ -107 +107 @@
     my ($header, $rows) = read_request_file($req_file);
 
-    if (!$header or !$rows) {
-        # Since a request got queued, the request file must have been readable at some
-        # point 
+    my $action = $header->{ACTION};
+
+    if (!$header or (($action eq 'PROCESS') and !$rows)) {
+        # Since a request got queued, the request file must have been readable at some point 
         my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
     }
@@ -126 +127 @@
     }
 
-    # The results table definition file
-    my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
-
-    # data for the header
-    print $tdf "$req_name|$req_id|\n";
-
-    # get the list of jobs generated for this request
-    my @jobs;
-    {
-        my $command = "$pstamptool -listjob -req_id $req_id";
-        $command   .= " -dbname $dbname" if $dbname;
-        $command   .= " -dbserver $dbserver" if $dbserver;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            die("Unable to perform $command error code: $error_code");
-        }
-        my $output = join "", @$stdout_buf;
-        if (!$output) {
-            # This should not happen. A fake job should have been entered
-            my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
-        } else {
-            my $jobs = parse_md_fast($mdcParser, $output);
-
-            @jobs = @$jobs;
-        }
-    }
-
-    my $exp_info;
-    foreach my $job (@jobs) {
-        my $job_id = $job->{job_id};
-        my $job_type = $job->{jobType};
-        my $rownum = $job->{rownum};
-        my $fault = $job->{fault};
-        my $exp_id = $job->{exp_id};
-
-        my $error_string = get_error_string($fault);
-
-        if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
-            # this request had a duplicate request name yet the parser didn't give
-            # it an "ERROR style name.
-            my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
-        }
-        my ($row, $req_info, $project) = get_request_info($rows, $rownum);
-
-        my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-        my $image_db = $proj_hash->{dbname};
-        if (!$image_db and !$fault) {
-            # if project isn't resolvable, the paser should have faulted this job
-            my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
-        }
-
-        my $job_params = get_job_parameters($job);
-        my $stage = "";
-        if ($job_params) {
-            $stage = $job_params->{stage};
-        }
-
-        if ($stage ne 'stack') {
-            # get the metadata for the exposure (if any i.e. stack) 
-            # returns an appropriate string if !$exp_id
-            $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
-        } else {
-            my $filter = $job_params->{filter};
-            $filter = "0" if !$filter;
-            my $mjd_obs = $job_params->{mjd_obs};
-            $mjd_obs = "0" if !$mjd_obs;
-            $exp_info = "$mjd_obs|0|0|$filter|0|0";
-        }
-
-        if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
-            my $jreglist = "$outdir/reglist$job_id";
-            if (open JRL, "<$jreglist") {;
-                # process the reglist file to get the list of files produced by this job
-                foreach my $line (<JRL>) {
-
-                    # XXX: we are getting many cases where the size and/or md5sum calculated by
-                    # the job has changed by the time the request_finish has run
-                    # Don't
-                    # add line to the requests's reglist
-                    # ....
-#                    print $rlf $line;
-
-
-                    chomp $line;
-                    my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
-                    # ... instead let dsreg compute the paramters by leaving them blank
-                    print $rlf "$img_name|||$filetype|\n";
-
-                    # add line to the table definition file
-                    print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
-
-                    # ra_deg and dec_deg are the coordinates of center of the stamp
-                    # first assume that the image is compressed and check the first extension.
-                    # If not found check the PHU. If that doesn't work just set them to zero.
-                    # XXX do this more cleanly
-                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
-                    if (!defined $ra_deg) {
-                        (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
+    my $request_fault = 0;
+    if ($action eq 'PROCESS') {
+        # The results table definition file
+        my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
+        # data for the header
+        print $tdf "$req_name|$req_id|\n";
+        # get the list of jobs generated for this request
+        my @jobs;
+        {
+            my $command = "$pstamptool -listjob -req_id $req_id";
+            $command   .= " -dbname $dbname" if $dbname;
+            $command   .= " -dbserver $dbserver" if $dbserver;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                die("Unable to perform $command error code: $error_code");
+            }
+            my $output = join "", @$stdout_buf;
+            if (!$output) {
+                # This should not happen. A fake job should have been entered
+                my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
+            } else {
+                my $jobs = parse_md_fast($mdcParser, $output);
+
+                @jobs = @$jobs;
+            }
+        }
+
+        my $exp_info;
+        foreach my $job (@jobs) {
+            my $job_id = $job->{job_id};
+            my $job_type = $job->{jobType};
+            my $rownum = $job->{rownum};
+            my $fault = $job->{fault};
+            my $exp_id = $job->{exp_id};
+
+            my $error_string = get_error_string($fault);
+
+            if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
+                # this request had a duplicate request name yet the parser didn't give
+                # it an "ERROR style name.
+                my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
+            }
+            my ($row, $req_info, $project) = get_request_info($rows, $rownum);
+
+            my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+            my $image_db = $proj_hash->{dbname};
+            if (!$image_db and !$fault) {
+                # if project isn't resolvable, the paser should have faulted this job
+                my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
+            }
+
+            my $job_params = get_job_parameters($job);
+            my $stage = "";
+            if ($job_params) {
+                $stage = $job_params->{stage};
+            }
+
+            if ($stage ne 'stack') {
+                # get the metadata for the exposure (if any i.e. stack) 
+                # returns an appropriate string if !$exp_id
+                $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
+            } else {
+                my $filter = $job_params->{filter};
+                $filter = "0" if !$filter;
+                my $mjd_obs = $job_params->{mjd_obs};
+                $mjd_obs = "0" if !$mjd_obs;
+                $exp_info = "$mjd_obs|0|0|$filter|0|0";
+            }
+
+            if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
+                my $jreglist = "$outdir/reglist$job_id";
+                if (open JRL, "<$jreglist") {;
+                    # process the reglist file to get the list of files produced by this job
+                    foreach my $line (<JRL>) {
+
+                        # XXX: we are getting many cases where the size and/or md5sum calculated by
+                        # the job has changed by the time the request_finish has run
+                        # Don't
+                        # add line to the requests's reglist
+                        # ....
+    #                    print $rlf $line;
+
+
+                        chomp $line;
+                        my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
+                        # ... instead let dsreg compute the paramters by leaving them blank
+                        print $rlf "$img_name|||$filetype|\n";
+
+                        # add line to the table definition file
+                        print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|";
+
+                        # ra_deg and dec_deg are the coordinates of center of the stamp
+                        # first assume that the image is compressed and check the first extension.
+                        # If not found check the PHU. If that doesn't work just set them to zero.
+                        # XXX do this more cleanly
+                        my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
+                        if (!defined $ra_deg) {
+                            (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
+                        }
+                        $ra_deg = 0.0 if (!$ra_deg);
+                        $dec_deg = 0.0 if (!$dec_deg);
+                        print $tdf "$ra_deg|$dec_deg|";
+
+                        print $tdf "$exp_info|";
+                        print $tdf "$req_info|";
+                        print $tdf "\n";
                     }
-                    $ra_deg = 0.0 if (!$ra_deg);
-                    $dec_deg = 0.0 if (!$dec_deg);
-                    print $tdf "$ra_deg|$dec_deg|";
-
+                    close JRL;
+                } else {
+                    my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
+                        if $fault eq $PSTAMP_SUCCESS;
+                    print STDERR "no reglist file for job $job_id\n" if $verbose;
+                    print $tdf "$rownum|$fault|$error_string|0|$job_id|";
+                    print $tdf "0|0|";       # center of (non-existent) stamp
                     print $tdf "$exp_info|";
                     print $tdf "$req_info|";
                     print $tdf "\n";
                 }
-                close JRL;
             } else {
-                my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
-                    if $fault eq $PSTAMP_SUCCESS;
-                print STDERR "no reglist file for job $job_id\n" if $verbose;
-                print $tdf "$rownum|$fault|$error_string|0|$job_id|";
-                print $tdf "0|0|";       # center of (non-existent) stamp
-                print $tdf "$exp_info|";
-                print $tdf "$req_info|";
-                print $tdf "\n";
-            }
-        } else {
-            # XXX do list jobs
-            # we can probably arange things to use the code as above and skip the fileset registration
-            print STDERR "Unknown jobType: $job_type";
-            next;
-        }
-    }
-    close $rlf;
-    close $tdf;
-    my $request_fault = 0;
-    # make the results file
-    {
-        my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            print STDERR "Unable to perform $command error code: $error_code\n";
-            $request_fault = $error_code >> 8;
-        } else {
-            # dump a textual representation
-            my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
+                # XXX do list jobs
+                # we can probably arange things to use the code as above and skip the fileset registration
+                print STDERR "Unknown jobType: $job_type";
+                next;
+            }
+        }
+        close $tdf;
+        # make the results file
+        {
+            my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
             unless ($success) {
+                print STDERR "Unable to perform $command error code: $error_code\n";
                 $request_fault = $error_code >> 8;
-                my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
-            }
-        }
-    }
+            } else {
+                # dump a textual representation
+                my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $request_fault = $error_code >> 8;
+                    my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
+                }
+            }
+        }
+    } elsif ($action ne 'LIST') {
+        my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR);
+    } else {
+        # pstampparse did all of the work
+    }
+
+    close $rlf;
+
     if (!$request_fault) {
         # register the fileset

branches/czw_branch/20120906/pstamp/scripts/pstamp_request_file

@@ -49 +49 @@
                     value => undef
         },
+        { name => 'ACTION',
+                    writetype => TSTRING,
+                    comment => 'Action to perform: LIST or PROCESS',
+                    value => undef
+        },
+        { name => 'USERNAME',
+                    writetype => TSTRING,
+                    comment => 'username for request (optional)',
+                    value => undef
+        },
+        { name => 'EMAIL',
+                    writetype => TSTRING,
+                    comment => 'email for submittor (optional)',
+                    value => undef
+        },
 ];
 
 # Specification of columns to write
 my $columns = [ 
+        { name => 'ROWNUM',     type => 'J',   writetype => TULONG }, 
+
+        { name => 'CENTER_X',   type => 'D',  writetype => TDOUBLE },
+        { name => 'CENTER_Y',   type => 'D',  writetype => TDOUBLE },
+        { name => 'WIDTH',      type => 'D',  writetype => TDOUBLE },
+        { name => 'HEIGHT',     type => 'D',  writetype => TDOUBLE },
+        # 2 bits in COORD_MASK indicate what units of roi coords are
+        { name => 'COORD_MASK', type => 'J',  writetype => TULONG },
+
+        { name => 'JOB_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'OPTION_MASK',type => 'J',   writetype => TULONG },
+
+        # image selection parameters
+        { name => 'PROJECT',    type => '16A', writetype => TSTRING },
+        # new in version 2
+        { name => 'SURVEY_NAME',type => '16A', writetype => TSTRING },
+        { name => 'IPP_RELEASE', type => '16A', writetype => TSTRING },
+        # end of new in version 2
+        { name => 'REQ_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'IMG_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'ID',         type => '16A', writetype => TSTRING },           
+        { name => 'TESS_ID',    type => '64A', writetype => TSTRING },
+        { name => 'COMPONENT',  type => '64A', writetype => TSTRING },
+
+        { name => 'DATA_GROUP ',type => '64A', writetype => TSTRING },
+
+        { name => 'REQFILT',    type => '16A', writetype => TSTRING },
+        { name => 'MJD_MIN',    type => 'D',   writetype => TDOUBLE },
+        { name => 'MJD_MAX',    type => 'D',   writetype => TDOUBLE },
+        { name => 'RUN_TYPE',   type => '16A', writetype => TSTRING },
+
+        { name => 'COMMENT ',   type => '64A', writetype => TSTRING },
+];
+my $columns_v1 = [ 
         { name => 'ROWNUM',     type => 'J',   writetype => TULONG }, 
 
@@ -89 +138 @@
 }
 
+# build array of arrays for the column data. Note if the input file is version 1 then
+# some of thes will get dropped
 my @colData;
 foreach (@$columns) {
@@ -96 +147 @@
 
 my $minimum_cols = 6;
-my $numRows = read_data_for_table($in,'\s+', \@colData, $header, $minimum_cols); 
+my $extver = 2;
+my $numRows = read_data_for_table($in,'\s+', \@colData, $header, $minimum_cols, \$extver); 
 if (!$numRows) {
     print STDERR "no data in $input\n";
@@ -115 +167 @@
 $output = $req_name . ".fits" if !$output;
 
-my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header);
+my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $extver > 1 ? $columns : $columns_v1, $header);
 
 exit $status;
@@ -230 +282 @@
     my $header  = shift;    # rerence to array of header keyword descriptions
     my $minimum_required_vals = shift;
+    my $r_extver = shift;   # reference to extension version
 
     my $line_num = 0;
@@ -243 +296 @@
             my @vals = split /$sep/, $line;
             my $nvals = @vals;
-            die "number of header columns in input $nvals does not equal expected number of header words $nhead"
+            die "two few columns $nvals in header definition. Require at least 2" if @vals < 2;
+
+            $$r_extver = $vals[1];
+            if ($extver > 1) {
+                die "number of header columns in input $nvals does not equal expected number of header words $nhead"
                     if (@vals != @$header);
-
-            for (my $i=0; $i < @$header; $i++) {
-                $header->[$i]->{value} = $vals[$i];
-            }
-
-            last; # only one header line
-        }
+            } else {
+                # drop header keywords not used in version 1
+                for (my $i = @$header - 1; $i >= $nvals; $i--) {
+                    pop @$header;
+                }
+            }
+
+            for (my $i=0; $i < $nvals; $i++) {
+                my $val = $vals[$i];
+                if (defined $val) {
+                    $header->[$i]->{value} = $val;
+                } else {
+                    $header->[$i]->{value} = 'null';
+                }
+            }
+
+            last; # one line defines the header
+        }
+    }
+    if ($$r_extver == 1) {
+        # pop off the colData arrays to account for the unused columns
+        pop @colData;
+        pop @colData;
+        pop @colData;
     }
 
@@ -271 +345 @@
             exit 1;
         }
+        # trim leading whitespace from comment
+        $comment =~ s/^\s+//;
         if ($comment) {
             if ($comment =~ /#/) {
@@ -381 +457 @@
     print "If --output is omitted the output file name is set to REQ_NAME.fits\n\n";
     print "Header 1 Line.  Format:\n\n";
-    print "  REQ_NAME EXTVER\n\n";
+    print "REQ_NAME EXTVER ACTION USERNAME EMAIL\n\n";
     print "REQUEST specification (1 or more lines). Format:\n\n";
-    print "  ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX | COMMENT\n\n";
+    print "ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE | COMMENT\n\n";
 
     exit 0;

branches/czw_branch/20120906/pstamp/scripts/pstampparse.pl

@@ -93 +93 @@
 my $fields_output;
 {
-    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -102 +102 @@
     $fields_output = join "", @$stdout_buf;
 }
-my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
 
 # make sure the file contains what we are expecting
@@ -109 +109 @@
 my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
 my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
+my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
 
 
@@ -230 +230 @@
     # the images of interest for this new row doesn't match the list. 
     # process the list ...
-    $num_jobs += processRows(\@rowList);
+    $num_jobs += processRows($action, \@rowList);
 
     # and reset the list to contain just the new row
@@ -238 +238 @@
 # out of rows process the list
 if (scalar @rowList > 0) {
-    $num_jobs += processRows(\@rowList);
-}
-
-if (($mode eq "queue_job") and ($num_jobs eq 0)) {
+    $num_jobs += processRows($action, \@rowList);
+}
+
+if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
     print STDERR "no jobs created for $req_name\n" if $verbose;
     insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
@@ -387 +387 @@
 }
 
-sub processRows {
+sub list_targets {
     my $rowList = shift;
     my $num_jobs = 0;
+
+    $num_jobs = 1;
+    return $num_jobs;
+}
+
+sub processRows {
+    my $action = shift;
+    my $rowList = shift;
+    my $num_jobs = 0;
+
+    if ($action eq 'LIST') {
+        return list_targets($rowList);
+    }
 
     # all rows in the list are compatible

branches/czw_branch/20120906/pstamp/src/ppstampArguments.c

@@ -25 +25 @@
     fprintf(stderr, "   [-mask   mk_image]    : mask image\n");
     fprintf(stderr, "   [-variance var_image] : variance image\n");
+    fprintf(stderr, "   [-sources sources]    : sources cmf (ignored for chip stage)\n");
     fprintf(stderr, "   [-stage stage]        : stage of input image (raw, chip, warp, stack, diff)\n");
     fprintf(stderr, "   [-write_jpeg]         : write a JPEG  format of the image stamp\n");
+    fprintf(stderr, "   [-write_cmf]          : create an output cmf with the sources overlapping the stamp\n");
+    fprintf(stderr, "   [-wholefile]          : ignore the region of interest and process the entire input image\n");
     // fprintf(stderr, "   [-no_censor_masked]   : do not set masked pixels to NAN\n");
     fprintf(stderr, "\n");
@@ -92 +95 @@
         options->stage = psStringCopy(argv[argnum]);
         psArgumentRemove(argnum, &argc, argv);
+    } else {
+        // Should we require this? 
+        options->stage = psStringCopy("unknown");
     }
     if ((argnum = psArgumentGet(argc, argv, "-write_jpeg"))) {
         psArgumentRemove(argnum, &argc, argv);
         options->writeJPEG = true;
-    } else {
-        options->writeJPEG = false;
+    }
+    if ((argnum = psArgumentGet(argc, argv, "-write_cmf"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        options->writeCMF = true;
     }
 
-    pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist");
+    // XXX: Note: the various list options have never been tested with ppstamp to my knowledge
+    bool gotAstrom = false;
+    if ((argnum = psArgumentGet(argc, argv, "-astrom"))) {
+        gotAstrom = true;
+        pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist");
+    }
     pmConfigFileSetsMD(config->arguments, &argc, argv, "MASK",   "-mask", "-masklist");
     pmConfigFileSetsMD(config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist");
 
-    // the input file is a required argument; if not found, we will exit
+    if ((argnum = psArgumentGet(argc, argv, "-sources"))) {
+        pmConfigFileSetsMD(config->arguments, &argc, argv, "SOURCES", "-sources", "-sourceslist");
+        // supplying a sources file implies that we want to save the output
+        options->writeCMF = true;
+    } else if (options->writeCMF && !gotAstrom) {
+        // if we didn't get a sources file but the user wanted us to write the sources we must
+        // have an astrometry file supplied
+        // XXX: Is this too restrictive? Could the sources be contained in say the input file?
+        psError(PSTAMP_ERR_ARGUMENTS, true, "cannot write cmf file unless input sources are supplied\n");
+        usage();
+    }
+
+    // the input image file is a required argument; if not found, we will exit
     bool status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list");
     if (!status) {
@@ -126 +151 @@
     }
 
-    // finally the output file
+    // finally the only argument left must be output outroot
     if (argc < 2) {
         psError(PSTAMP_ERR_ARGUMENTS, true, "must specify OUTPUT\n");
@@ -135 +160 @@
     }
 
-    // Add the output image (which remains on the command-line) to the arguments list
     psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]);
-
-    // psMetadataPrint(stdout, config->arguments,0);
 
     return config;

branches/czw_branch/20120906/pstamp/src/ppstampMakeStamp.c

@@ -19 +19 @@
 static void chipToSky(pmAstromObj *pt, pmFPA *fpa, pmChip *chip);
 static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance);
+static bool copySources(pmReadout *outputReadout, pmReadout *inputReadout, pmReadout *astromReadout, psRegion extractRegion);
 
 // convert the input chip's transforms to the output
@@ -80 +81 @@
 }
 
-static bool copyMetadata(pmFPAfile *output, pmFPAfile *input, pmChip *inChip, ppstampOptions *options, pmAstromObj *center)
+static bool copyMetadata(pmFPAfile *output, pmFPAfile *input, pmChip *inChip, ppstampOptions *options, pmAstromObj *center, pmFPAfile *astrom, pmFPAview *inView)
 {
     pmChip    *outChip;
-    pmFPAview *view = pmFPAviewAlloc(0);
+    pmFPAview *outView = pmFPAviewAlloc(0);
 
     // our output file has a single chip
-    view->chip = 0;
-    outChip = pmFPAviewThisChip(view, output->fpa);
-    psFree(view);
-
-    // copy data from the input chip header to the output.
+    outView->chip = 0;
+    outChip = pmFPAviewThisChip(outView, output->fpa);
+    psFree(outView);
+    pmHDU *outHDU = pmHDUGetHighest(output->fpa, outChip, NULL);
+
+    // copy data from the input's chip header to the output.
     // since some of the keywords might be duplicated we may not want to copy both
 
-    pmHDU *inHDU  = pmHDUFromChip(inChip);
-    pmHDU *outHDU = pmHDUGetHighest(output->fpa, outChip, NULL);
+    pmHDU *inHDU  = NULL;
+    if (strcmp(options->stage, "raw") && input != astrom) {
+        // Copy the header from the astrometry file since it contains the psphot and psastro results
+        pmChip *astromChip = pmFPAviewThisChip(inView, astrom->fpa);
+        inHDU  = pmHDUFromChip(astromChip);
+    } else {
+        inHDU  = pmHDUFromChip(inChip);
+    }
 
     if (inHDU->header) {
@@ -102 +110 @@
     }
 
-    // copy the fpa concepts
+    // copy the fpa and chip concepts
     pmConceptsCopyFPA(output->fpa, input->fpa, false, false);
+    // Needed to preserve FPA.ZP
     pmConceptsCopyChip(outChip, inChip, false);
+
+    // copy the cell concepts (needed to get CELL.EXPOSURE which gets used to set EXPTIME)
     pmCell *outCell = outChip->cells->data[0]; // The only output cell
 
@@ -122 +133 @@
     }
 
-    // If input had WCS convert it for stamp
+    // If input had WCS convert it for the new image coordinate system of our stamp
     if (input->fpa->toSky) {
         // copy the input fpa's transforms
@@ -135 +146 @@
         }
     } else {
-        psWarning("No WCS present in input\n");
-    }
-
+        psLogMsg("ppstampMakeStamp", PS_LOG_WARN, "No WCS present in input.");
+    }
+
+    // Save our specific metadata
     psMetadataAddF64(outHDU->header, PS_LIST_TAIL, "RA_DEG", PS_META_REPLACE, "Right Ascension of stamp center", RAD_TO_DEG(center->sky->r));
     psMetadataAddF64(outHDU->header, PS_LIST_TAIL, "DEC_DEG", PS_META_REPLACE, "Declination of stamp center", RAD_TO_DEG(center->sky->d));
+
+    // XXX: TODO: save the region of interest boundary and the offset to be applied. 
+    // XXX: Gene thinks that there may be a way to handle this in the WCS
+
+    // Now a few sanity checks
+    // Make sure zero point in the fpa concepts matches the observed value in the input header
+    bool status;
+    float zpt_obs = psMetadataLookupF32(&status, outHDU->header, "ZPT_OBS");
+    if (status) {
+        psMetadataAddF32(output->fpa->concepts, PS_LIST_TAIL, "FPA.ZP", PS_META_REPLACE, "Magnitude zero point", zpt_obs);
+    }
 
     ppstampVersionMetadata(outHDU->header, options);
@@ -324 +347 @@
 
             if (options->censorMasked && !setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) {
-                 psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp mask image\n");
-                 status = false;
-                 break;
+                psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp mask image\n");
+                status = false;
+                break;
             }
         }
@@ -335 +358 @@
         status = true;
 
+        if (options->writeCMF) {
+            pmReadout *astromReadout = NULL;
+            if (astrom->fpa != input->fpa) {
+                astromReadout = pmFPAviewThisReadout(srcView, astrom->fpa);
+            }
+            if (!copySources(outReadout, astromReadout, astromReadout, extractRegion)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to extract sources from region of interest.\n");
+                status = false;
+                break;
+            }
+        }
+
         psFree(outReadout); // drop reference
     }
@@ -341 +376 @@
 
     if (status) {
-        status = copyMetadata(output, input, inChip, options, center);
+        // For raw stage we need to do more work to build a proper header
+        status = copyMetadata(output, input, inChip, options, center, astrom, view);
     }
     return status ? PS_EXIT_SUCCESS : PS_EXIT_UNKNOWN_ERROR;
@@ -428 +464 @@
     double dy = 0.5 * options->roip.dDEC / fpa->toSky->Ys;
 
-    if (dx > 5000) {
-        fprintf(stderr, "requested width %f too large reducing to 5000\n", dx);
-        dx = 5000;
-    }
-    if (dy > 5000) {
-        fprintf(stderr, "requested height %f too large reducing to 5000\n", dy);
-        dy = 5000;
+    // XXX: why do we limit this?
+    if (dx > 8000) {
+        psWarning( "requested width %f too large reducing to 8000\n", dx);
+        dx = 8000;
+    }
+    if (dy > 8000) {
+        psWarning( "requested height %f too large reducing to 8000\n", dy);
+        dy = 8000;
     }
 
@@ -480 +517 @@
     ppstampOverlap   returnval = PPSTAMP_OFF;
 
-//    fprintf(stderr, "ppstampChipBounds: %s\n", psRegionToString(*chipBounds));
-
     // set up the astrometry
     pmHDU *hdu = pmFPAviewThisHDU(view, astrom->fpa);
@@ -498 +533 @@
         }
     } else {
-        // we use a default FPA pixel scale of 1.0
         if (!pmAstromReadWCS (input->fpa, chip, hdu->header, 1.0) && mustHaveAstrometry) {
             psError(PS_ERR_UNKNOWN, false, "Unable to read WCS astrometry for input FPA.");
@@ -537 +571 @@
             center->chip->yErr = 0;
             onChip = true;
-            chipToSky(center, input->fpa, chip);
+            if (input->fpa->toSky) {
+                chipToSky(center, input->fpa, chip);
+            }
         }
     }
@@ -550 +586 @@
                 int width  = options->roip.dX;
                 int height = options->roip.dY;
-                if (width > 7000) {
-                    fprintf(stderr, "requested width %d too large reducing to 7000\n", width);
-                    width = 7000;
+                if (width > 8000) {
+                    psWarning( "requested width %d too large reducing to 8000\n", width);
+                    width = 8000;
                 }
-                if (height > 7000) {
-                    fprintf(stderr, "requested height %d too large reducing to 7000\n", height);
-                    height = 7000;
+                if (height > 8000) {
+                    psWarning( "requested height %d too large reducing to 8000\n", height);
+                    height = 8000;
                 }
 
@@ -675 +711 @@
 
     if (!foundOverlap && (returnval == PS_EXIT_SUCCESS)) {
-        fprintf(stderr, "ROI not found in input\n");
+        psWarning( "ROI not found in input\n");
         returnval = PSTAMP_NO_OVERLAP;
     }
@@ -739 +775 @@
     }
 
-    // XXX: this shouldn't be a fprintf
-    fprintf(stderr, "Censored %ld masked pixels\n", numCensored);
+    psLogMsg("ppstamp", PS_LOG_DETAIL, "Censored %ld masked pixels\n", numCensored);
 
     return true;
 }
+
+static bool copySources(pmReadout *outReadout, pmReadout *inReadout, pmReadout *astromReadout, psRegion extractRegion) {
+    bool status;
+
+    // first look for detections in the input readout. Those would have come from the -sources file
+    pmDetections *inDetections = psMetadataLookupPtr (&status, inReadout->analysis, "PSPHOT.DETECTIONS");
+    if (!inDetections) {
+        // No detections on the input readout. Try the astrometry readout if supplied.
+        if (astromReadout) {
+            inDetections = psMetadataLookupPtr (&status, astromReadout->analysis, "PSPHOT.DETECTIONS");
+            if (!inDetections) {
+                psLogMsg("ppstampMakeStamp", PS_LOG_WARN, "no detections found on input or astrometry readout\n");
+                return false;
+            } else {
+                psLogMsg("ppstampMakeStamp", PS_LOG_INFO, "using detections from astrom file\n");
+            }
+        } else {
+            psLogMsg("ppstampMakeStamp", PS_LOG_WARN, "no detections found on input readout\n");
+            return false;
+        }
+    }
+    if (!inDetections->allSources) {
+        psLogMsg("ppstampMakeStamp", PS_LOG_WARN, "no sources array found on detections structure\n");
+        return false;
+    }
+
+    // Ok we have sources, allocate a detections structure and copy them to the output readout
+    psArray *inSources = inDetections->allSources;
+
+    pmDetections *outDetections = pmDetectionsAlloc();
+    psMetadataAddPtr (outReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN,
+        "psphot detections", outDetections);
+
+    psArray *outSources = outDetections->allSources = psArrayAllocEmpty( 100 );
+
+    // include sources that are off the stamp by just a little bit
+    #define EXPAND_SIZE 16
+    psRegion sourceRegion = extractRegion;
+    sourceRegion.x0 -= EXPAND_SIZE;
+    sourceRegion.y0 -= EXPAND_SIZE;
+    sourceRegion.x1 += EXPAND_SIZE;
+    sourceRegion.y1 += EXPAND_SIZE;
+
+    for (int i = 0; i < inSources->n; i++ ) {
+        pmSource *src = inSources->data[i];
+
+        float x = src->peak->xf;
+        float y = src->peak->yf;
+        if (x < sourceRegion.x0 || x > sourceRegion.x1 ||
+            y < sourceRegion.y0 || y > sourceRegion.y1) {
+            continue;
+        }
+        psArrayAdd(outSources, 100, src);
+    }
+
+    return true;
+}

branches/czw_branch/20120906/pstamp/src/ppstampOptions.c

@@ -32 +32 @@
     options->censorMasked = false;
     options->writeJPEG = false;
+    options->writeCMF = false;
     options->nocompress = false;
     options->wholeFile = false;

branches/czw_branch/20120906/pstamp/src/ppstampOptions.h

@@ -14 +14 @@
     bool        censorMasked;
     bool        writeJPEG;
+    bool        writeCMF;
     bool        nocompress;
     //

branches/czw_branch/20120906/pstamp/src/ppstampParseCamera.c

@@ -120 +120 @@
     }
 
+    // Set up the input and output sources files if needed
+    if (options->writeCMF) {
+        bool status;
+
+        // see if -sources file was supplied. 
+        // If so define the file.
+        psPtr sourcesFile = psMetadataLookupStr(&status, config->arguments, "SOURCES");
+        if (sourcesFile) {
+            pmFPAfile *sources = pmFPAfileBindFromArgs(&status, input, config, "PPSTAMP.INPUT.SOURCES", "SOURCES");
+            if (!status) {
+                psError(psErrorCodeLast(), false, "Failed to load file definition for PPSTAMP.INPUT.SOURCES");
+                return false;
+            }
+            if (!sources && !astrom) {
+                psError(psErrorCodeLast(), false, "Failed to define input sources file");
+                return false;
+            }
+        } else {
+            psLogMsg ("ppstamp", PS_LOG_INFO, "output sources file requested but no input supplied. Will try the supplied astrometry file.\n");
+        }
+        
+        pmFPAfile *outputSources = NULL;
+        outputSources = pmFPAfileDefineSkycell(config, output->fpa, "PPSTAMP.OUTPUT.SOURCES");
+        if (!outputSources) {
+            psError(psErrorCodeLast(), false, "Failed to define output sources file");
+            return false;
+        }
+        outputSources->save = true;
+    }
+
     pmFPAfile *chipImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PPSTAMP.CHIP");
     if (!chipImage) {

branches/czw_branch/20120906/pstamp/src/pstamp.h

@@ -26 +26 @@
 
 // values for options mask.
-#define PSTAMP_SELECT_IMAGE      1
-#define PSTAMP_SELECT_MASK       2
-#define PSTAMP_SELECT_WEIGHT     4
-#define PSTAMP_SELECT_CMF        8
-#define PSTAMP_SELECT_PSF       16
-#define PSTAMP_SELECT_BACKMDL   32
-#define PSTAMP_SELECT_INVERSE 1024
-#define PSTAMP_SELECT_UNCONV  2048
-#define PSTAMP_USE_IMFILE_ID 16384
+#define PSTAMP_SELECT_IMAGE         1
+#define PSTAMP_SELECT_MASK          2
+#define PSTAMP_SELECT_WEIGHT        4
+#define PSTAMP_SELECT_SOURCES       8
+#define PSTAMP_SELECT_CMF           8
+#define PSTAMP_SELECT_PSF           16
+#define PSTAMP_SELECT_BACKMDL       32
+#define PSTAMP_SELECT_JPEG          64
+// unused                           128
+// unused                           256
+#define PSTAMP_SELECT_UNCOMPRESSED  512
+#define PSTAMP_SELECT_INVERSE       1024
+#define PSTAMP_SELECT_UNCONV        2048
+#define PSTAMP_RESTORE_BACKGROUND   4096
+// unused                           8192
+#define PSTAMP_USE_IMFILE_ID        16384
 
-#define PSTAMP_NO_WAIT_FOR_UPDATE 32768
+#define PSTAMP_NO_WAIT_FOR_UPDATE   32768
+#define PSTAMP_REQUEST_UNCENSORED  0x10000
+#define PSTAMP_REQUIRE_UNCENSORED  0x20000
 
 #define PSTAMP_CENTER_IN_PIXELS 1