Index: branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 36680) @@ -34,4 +34,5 @@ use POSIX; use Time::HiRes qw(gettimeofday); +use IO::Handle; my $dvo_verbose = 0; @@ -176,5 +177,6 @@ $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 0, 0, # fwhm cuts are not applied here - undef, undef, undef, $verbose); + undef, 0, # no cam run information + undef, undef, $verbose); return $results; @@ -243,4 +245,9 @@ if (isnull($filter)) { $filter = undef; + } + + if ($stage eq 'stack_summary') { + # stack_summary jobs are so different from others that we lookup in a specialized function + return lookup_stack_summary($ipprc, $row, $imagedb, $tess_id, $component, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); } @@ -322,5 +329,5 @@ 0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, getValOrZero($row->{FWHM_MIN}), getValOrZero($row->{FWHM_MAX}), - undef, $release_name, $survey, + undef, 0, $release_name, $survey, $verbose); @@ -351,4 +358,5 @@ my $fwhm_max = shift; my $selectedAstrom = shift; + my $selected_cam_id = shift; my $release_name = shift; my $survey = shift; @@ -460,4 +468,12 @@ $skycell_id = $component; $choose_components = 1; + if ($req_type eq 'byskycell' and $skycenter) { + # Add a wide coordinate cut to the releasetool command. + # This greatly speeds up the query by reducing the number of exposures considered + # to those near the region of interest. + # The actual radius value that we use is not critical because the skycell cut does the relevant + # selection. + $command .= " -ra $row->{CENTER_X} -decl $row->{CENTER_Y} -radius 5.0"; + } } else { $command = "$warptool -pstamp_order -dbname $imagedb"; @@ -602,5 +618,5 @@ # The image selectors are such that multiple runs my have be returned for the same exposure. # Return only the latest one. - $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose); + $images = filterRuns($stage, $choose_components, $need_magic, $images, $inverse, $verbose); } @@ -685,7 +701,8 @@ $out->{data_state} = $out->{state} = $out->{stack_state}; } else { - # XXX: Consider looking up skycal results even if we are + $out->{data_state} = $out->{state}; + + # XXX: Consider looking up skycal and staticsky results even if we are # not using releasetool - $out->{data_state} = $out->{state}; } } @@ -699,4 +716,5 @@ $out->{astrom} = $selectedAstrom; $out->{cam_path_base} = $selectedAstrom; + $out->{cam_id} = $selected_cam_id if $selected_cam_id; if ($selectedAstrom =~ /\.smf$/) { $out->{cam_path_base} =~ s/\.smf$//; @@ -722,7 +740,13 @@ $out->{mask} = $ipprc->filename($mask_name, $mask_base, $class_id) if $mask_name; $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name; - $out->{cmf} = $ipprc->filename($cmf_name, $base, $class_id) if $cmf_name; - $out->{psf} = $ipprc->filename($psf_name, $base, $class_id) if $psf_name; $out->{backmdl}= $ipprc->filename($backmdl_name,$base, $class_id) if $backmdl_name; + if ($image->{staticsky_path_base}) { + $out->{psf} = $ipprc->filename("PSPHOT.STACK.PSF.SAVE", $image->{staticsky_path_base}, $image->{stack_id}); + # if astrom was not set above to the skycal cmf get the sources file from staticsky + $out->{cmf} = $ipprc->filename("PSPHOT.STACK.OUTPUT", $image->{staticsky_path_base}, $image->{stack_id}) if !$out->{astrom}; + } else { + $out->{psf} = $ipprc->filename($psf_name, $base, $class_id) if $psf_name; + $out->{cmf} = $ipprc->filename($cmf_name, $base, $class_id) if $cmf_name; + } } @@ -731,4 +755,118 @@ return $output; +} + +sub lookup_stack_summary { + my ($ipprc, $row, $imagedb, $tess_id, $component, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose) = @_; + + $row->{error_code} = $PSTAMP_NO_IMAGE_MATCH; + + $component = '' if $component eq 'all'; + + my $req_type = $row->{REQ_TYPE}; + my $results; + if ($req_type eq 'byid') { + my $sass_id = $row->{ID}; + my $command = "$stacktool -dbname $imagedb -summary -sass_id $sass_id"; + $results = runToolAndParse($command, $verbose); + } else { + my ($release_name, $survey, $default_tess_id) = get_release_info($row); + if (!$tess_id and $default_tess_id) { + $tess_id = $default_tess_id; + } + + my $skycells; + if ($req_type eq 'bycoord') { + my $ra = $row->{CENTER_X}; + my $dec = $row->{CENTER_Y}; + $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose); + } elsif ($req_type eq 'byskycell') { + if (!defined $tess_id or !defined $component) { + print STDERR "Error: TESS_ID and COMPONENT are required for REQ_TYPE byskycell\n"; + $row->{error_code} = $PSTAMP_INVALID_REQUEST; + } + my $skycell = { + tess_id => $tess_id, + component => $component + }; + push @$skycells, $skycell + } else { + print STDERR "Error: $req_type is not a valid REQ_TYPE for IMG_TYPE stack_summary\n"; + $row->{error_code} = $PSTAMP_INVALID_REQUEST; + return undef; + } + + if ($skycells) { + # We have a list of skycells. Find matching projection cells. + # and then matching rows in stackSummary + + my %projection_cells_found; + foreach my $skycell (@$skycells) { + my $tess_id = $skycell->{tess_id}; + my $skycell_id = $skycell->{component}; + + my $projection_cell = skycell_id_to_projection_cell($skycell_id); + + # only need to include a projection cell once + next if $projection_cells_found{$projection_cell}; + + $projection_cells_found{$projection_cell} = 1; + + # print "$tess_id $skycell_id $projection_cell\n"; + + my $command; + if ($release_name or $survey) { + $command = "$releasetool -dbname $imagedb"; + if ($release_name) { + $command .= " -release_name $release_name"; + } else { + $command .= " -priority_order"; + } + $command .= " -surveyName $survey" if $survey; + } else { + $command = "$stacktool -dbname $imagedb"; + } + $command .= " -summary -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + + $command .= " -filter $filter" if $filter; + $command .= " -data_group $data_group" if $data_group; + + my $these_results = runToolAndParse($command, $verbose); + push @$results, @$these_results if $these_results; + } + } + } + + if ($results) { + foreach my $summary (@$results) { + my $path_base = $summary->{path_base}; + my $projection_cell = $summary->{projection_cell}; + + # we need to flesh out the hashes returned with values that the parser expects + $summary->{stage} = 'stack_summary'; + $summary->{component} = $projection_cell; + $summary->{stage_id} = $summary->{sass_id}; + $summary->{path_base} = "$path_base.$projection_cell"; + + # The stack_summary stage doesn't follow the usual ipp conventions for file rules + # and path_base. We leave it up to the program that runs the job to handle this. + # However, the parser uses the image parameter to set the job's outputBase (by removing the .fits) + # so we need to set it. Just use the new path_base. + $summary->{image} = $summary->{path_base}; + $summary->{state} = 'full'; + $summary->{data_state} = 'full'; + $summary->{fault} = 0; + $summary->{imagedb} = $imagedb; + $summary->{row_index} = []; + push @{$summary->{row_index}}, 0; + + # XXX: Are there any other required parameters? + # TODO: create a function to validate the components returned + # and make sure that any required elements are there. + } + } + + return $results; } @@ -946,5 +1084,5 @@ $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $fwhm_min, $fwhm_max, - $chip->{astrom}, undef, undef, + $chip->{astrom}, $chip->{cam_id}, undef, undef, $verbose); @@ -964,5 +1102,5 @@ setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH); } else { - # XXX: We are not applying the fwhm cuts + # XXX: We are not applying the fwhm cuts unless we use releasetool foreach my $skycell (@$skycells) { my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef, @@ -970,5 +1108,5 @@ $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $fwhm_min, $fwhm_max, - undef, $release_name, $survey, + undef, 0, $release_name, $survey, $verbose); @@ -1166,5 +1304,11 @@ my @lines; - { + my $looked_up_fast; + if ($requested_tess_id) { + # try using the last method first + $looked_up_fast = lookup_skycells_fast($ipprc, \@lines, $requested_tess_id, $ra, $dec, $verbose); + } + + if (!$looked_up_fast) { my $command = "$whichimage $ra $dec"; $command .= " --tess_id $requested_tess_id" if $requested_tess_id; @@ -1302,4 +1446,5 @@ $image->{astrom} = $astromFile; $image->{cam_path_base} = $camRoot; + $image->{cam_id} = $camRun->{cam_id}; return 1; @@ -1649,4 +1794,5 @@ sub filterRuns { my $stage = shift; + my $drop_duplicate_runs = shift; my $need_magic = shift; my $inputs = shift; @@ -1694,5 +1840,5 @@ } - $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id)); + # $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id)); # skip if we need magicked run and this one has never been magicked @@ -1702,5 +1848,5 @@ } - if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) { + if (($exp_id == $last_exp_id) and ($drop_duplicate_runs || ($run_id != $last_run_id))) { print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed; next; @@ -1873,4 +2019,197 @@ return $diff_mode; } + +sub setErrorCodesForRows { + my $rowList = shift; + my $error_code = shift; + + foreach my $row (@$rowList) { + $row->{error_code} = $error_code; + } + +} + +# Convert from skycell_id to projection_cell +# This code is not particularly elegant but I think that it matches the SQL used +# to create stackAssocations does. + +sub skycell_id_to_projection_cell { + my $skycell_id = shift; + + # split skycell_id into '.' separated components + my @strs = split '\.', $skycell_id; + + my $num = scalar @strs; + + # projection_cell is 'skycell_id' plus all .num values except the last one + my $projection_cell = $strs[0]; + for (my $i=1; $i < $num - 1; $i++) { + $projection_cell .= ".$strs[$i]"; + } + + return $projection_cell; +} + +# lookup_skycells_fast() +# Lookup using a "skycell server" process which is implemented by running dvoImagesAtCoords in "server" mode. +# Repeated calls to this function will reuse an existing process. This avoids having to loading the +# tessellation over and over again for each skycell lookup. +# When this parse process exists, the pipes are closed which causes the skycell server to exit. + +my $scs_tess_id; # tess_id for existing scs (skycell server) (if any) +my $scs_out; # reference filehandle for sending data to the scs +my $scs_in; # filehandle for response from scs + +sub lookup_skycells_fast { + my ($ipprc, $results, $tess_id, $ra, $dec, $verbose) = @_; + + # see if we need to start the scs (skycell server) + + my $very_verbose = 0; + + if (!$scs_tess_id or ($scs_tess_id ne $tess_id)) { + if ($scs_tess_id) { + # tess_id has changed, close down existing scs + close $scs_out; + $scs_out = undef; + close $scs_in; + $scs_in = undef; + $scs_tess_id = undef; + } + + # convert tess_id to a directory name + my $tess_dir = $ipprc->tessellation_catdir( $tess_id ); + unless ($tess_dir) { + print STDERR "Unrecognized tess_id: $tess_id\n"; + return 0; + } + + # convert tess_dir to an absolute directory name + my $tess_dir_resolved = $ipprc->convert_filename_absolute( $tess_dir ); + unless ($tess_dir_resolved and -d $tess_dir_resolved) { + print STDERR "failed to resolve tessellation directory for $tess_id\n"; + return 0; + } + + # start an scs + print "Starting skycell server for $tess_dir_resolved.\n"; + + # create pipes for communicating with the server + pipe $scs_in, SERVER_WRITER; + pipe SERVER_READER, $scs_out; + + # set our output to be unbuffered + $scs_out->autoflush(1); + + # FORK + my $scs_pid = fork(); + + if (!$scs_pid) { + unless (defined $scs_pid) { + # This is still in parent process. No joy. + print STDERR "fork of skycell server failed: $!"; + return 0; + } + + # This code is running in the child process - the skycell server + # Close file handles for the parent's ends of the pipes + close $scs_in; + close $scs_out; + # close stdio filehandles. we are about to redirect them + # (STDERR stays open pointing to the parser log) + close STDIN; + close STDOUT; + + # redirect our stdio file handles to the proper end of the pipes + open (STDIN, "<&SERVER_READER") or die "\nSCS: failed to redirect stdin"; + open (STDOUT, ">>&SERVER_WRITER") or die "\nSCS: failed to redirect stdout"; + + # All set. Now exec dvoImagesAtCoords + + my $command = "$dvoImagesAtCoords -D CATDIR $tess_dir_resolved -coords -"; + + print STDERR "SCS: execing $command\n" if $very_verbose; + + unless(exec $command) { + # note parent will notice that we are gone due to the pipes getting broken + # when this child dies. + die "SCS: failed to exec $command"; + } + } + + # This code is running in the parent process (the parser). + # We have succesfully launched the scs process. + # Close file handles for the scs' end of the pipes. + close SERVER_WRITER; + close SERVER_READER; + + # remember the tess_id + $scs_tess_id = $tess_id; + } + + # send coordinates to the skycell server and wait for the results + # (which come nearly instantly which is the point of doing this) + + # If pipes to the skycell server die, don't abort. + # Our reads and writes detect errors and act sensibly. + local $SIG{PIPE} = 'IGNORE'; + + # write the coordinates to the pipe + my $write_ok = print $scs_out "1 $ra $dec\n"; + if (!$write_ok) { + # server process probably didn't start properly or died. Fail. + # Caller will attempt to use the conventional method. + print STDERR "write to skycell server failed. Error is $!\n"; + return 0; + } + + print STDERR " sent coordinates to skycell server\n" if $very_verbose; + + my $received_done = 0; + + # Wait for response. If pipe breaks the read returns with no data. + while (my $line = <$scs_in>) { + chomp $line; + if ($line eq 'DONE') { + # No more results for these coordinates. + $received_done = 1; + print STDERR " received DONE\n" if $very_verbose; + last; + } + print STDERR " received $line\n" if $very_verbose; + + # The caller expects the output to be in the format used by whichimage + # which omits the point number and includes the tess_id. + my ($ptnum, $raout, $decout, $skycell_id) = split " ", $line; + + my $out = "$raout $decout $tess_id $skycell_id"; + + # print the result to the log + print "$out\n" if $verbose; + + push @$results, $out; + } + + print STDERR "Out of wait for responses loop\n" if $very_verbose; + + if (!$received_done) { + # something has gone wrong. + print STDERR "Read loop exited without receiving DONE.\n"; + + die "BAILING out" if $very_verbose; + + # forget about the existing server instance + $scs_tess_id = undef; + close $scs_out; + close $scs_in; + + # drop any results received so far + @$results = (); + return 0; + } + + return 1; +} + Index: branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 36680) @@ -29,4 +29,6 @@ $PSTAMP_SELECT_BACKMDL $PSTAMP_SELECT_JPEG + $PSTAMP_SELECT_EXP + $PSTAMP_SELECT_NUM $PSTAMP_SELECT_UNCOMPRESSED $PSTAMP_SELECT_INVERSE @@ -35,6 +37,6 @@ $PSTAMP_USE_IMFILE_ID $PSTAMP_NO_WAIT_FOR_UPDATE - $PSTAMP_REQUEST_UNCENSORED - $PSTAMP_REQUIRE_UNCENSORED + $PSTAMP_SELECT_EXPJPEG + $PSTAMP_SELECT_NUMJPEG $PSTAMP_SUCCESS $PSTAMP_FIRST_ERROR_CODE @@ -53,4 +55,5 @@ $PSTAMP_NO_OVERLAP $PSTAMP_NOT_AUTHORIZED + $PSTAMP_NO_VALID_PIXELS ); our %EXPORT_TAGS = (standard => [@EXPORT_OK]); @@ -71,6 +74,6 @@ our $PSTAMP_SELECT_BACKMDL = 32; our $PSTAMP_SELECT_JPEG = 64; -# unused 128 -# unused 256 +our $PSTAMP_SELECT_EXP = 128; +our $PSTAMP_SELECT_NUM = 256; our $PSTAMP_SELECT_UNCOMPRESSED = 512; our $PSTAMP_SELECT_INVERSE = 1024; @@ -82,7 +85,10 @@ our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768; -# these bits will be repurposed -our $PSTAMP_REQUEST_UNCENSORED = 0x10000; -our $PSTAMP_REQUIRE_UNCENSORED = 0x20000; +our $PSTAMP_SELECT_EXPJPEG = 0x10000; +our $PSTAMP_SELECT_NUMJPEG = 0x20000; + +# these bits have been repurposed. They were only exposed to MOPS and IFA and they have adapted. +#our $PSTAMP_REQUEST_UNCENSORED = 0x10000; +#our $PSTAMP_REQUIRE_UNCENSORED = 0x20000; # job and result codes @@ -97,12 +103,13 @@ our $PSTAMP_INVALID_REQUEST = 21; our $PSTAMP_UNKNOWN_PROJECT = 22; -our $PSTAMP_UNKNOWN_PRODUCT = 22; #this error code was mis-named it is left for compatabiliyt +our $PSTAMP_UNKNOWN_PRODUCT = 22; #this error code was a typo it is left for compatabiliy our $PSTAMP_NO_IMAGE_MATCH = 23; our $PSTAMP_NOT_DESTREAKED = 24; our $PSTAMP_NOT_AVAILABLE = 25; -our $PSTAMP_GONE = 26; # this value is used in ippTools +our $PSTAMP_GONE = 26; # this value is also used in ippTools our $PSTAMP_NO_JOBS_QUEUED = 27; our $PSTAMP_NO_OVERLAP = 28; our $PSTAMP_NOT_AUTHORIZED = 29; +our $PSTAMP_NO_VALID_PIXELS = 30; @@ -143,5 +150,5 @@ PSTAMP_NO_OVERLAP PSTAMP_NOT_AUTHORIZED -PSTAMP_NOT_AUTHORIZED +PSTAMP_NO_VALID_PIXELS );