Index: /tags/ipp-20111222/pstamp/scripts/Makefile.am =================================================================== --- /tags/ipp-20111222/pstamp/scripts/Makefile.am (revision 33324) +++ /tags/ipp-20111222/pstamp/scripts/Makefile.am (revision 33325) @@ -29,4 +29,5 @@ detect_query_create \ detect_response_create \ + dquery_job_run.pl \ dquery_finish.pl \ dqueryparse.pl \ Index: /tags/ipp-20111222/pstamp/scripts/detect_query_read =================================================================== --- /tags/ipp-20111222/pstamp/scripts/detect_query_read (revision 33324) +++ /tags/ipp-20111222/pstamp/scripts/detect_query_read (revision 33325) @@ -222,6 +222,7 @@ if (($colData{FPA_ID}[$i] eq "Not_Set")&&(($colData{'FILTER'}[$i] ne "Not_Set")&& ($colData{'MJD-OBS'}[$i] ne "Not_Set"))) { - my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]); - my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1); + my $mjd_obs = int $colData{'MJD-OBS'}[$i]; + my $dateobs_begin = mjd_to_dateobs($mjd_obs); + my $dateobs_end = mjd_to_dateobs($mjd_obs + 1); my $ra = $colData{'RA1_DEG'}[$i]; my $dec = $colData{'DEC1_DEG'}[$i]; @@ -229,7 +230,13 @@ my $cmd; if ($colData{STAGE}[$i] eq 'SSdiff') { - $cmd = "$difftool -listrun -dbname $dbname -filter $filter "; - $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end "; - $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 4"; + # Assumption: All Stack Stack diffs are for a medium deep field + # go find which one the coordinates are in + # XXX: If we don't find one we'll just get the first SSdiff + # in the filter on the day in question + my $field = find_md_field($ra, $dec, 1.6); + $cmd = "$difftool -listssrun -dbname $dbname -filter $filter "; + $cmd .= " -mjd_obs_begin $mjd_obs -mjd_obs_end " . (1 + $mjd_obs); + $cmd .= " -limit 1"; + $cmd .= " -tess_id $field%" if $field; } elsif ($colData{STAGE}[$i] eq 'WSdiff') { @@ -366,3 +373,35 @@ +# Given an RA and DEC return the MD field that contains it if any. +# This is much faster than actuallly examining the tessellation +# using dvoImagesAtCoords + +sub find_md_field { + my $ra_deg = shift; + my $dec_deg = shift; + my $radius_deg = shift; + + my @MD = qw(MD01 MD02 MD03 MD04 MD05 MD06 MD07 MD08 MD09 MD10); + my @MD_RA = (35.875, 53.10, 130.592, 150.000, 161.917, 185.000, 213.454, 242.787, 333.98, 352.312); + my @MD_DEC = (-4.25, -27.8, 44.317, 2.20, 58.083, 47.117, 53.243, 54.95, 0.283, -0.433); + my $PS_DEG_RAD = 0.017453292519943; + + my $ra = $ra_deg * $PS_DEG_RAD; + my $dec = $dec_deg * $PS_DEG_RAD; + my $radius = $radius_deg * $PS_DEG_RAD; + + for (my $i = 0; $i < scalar @MD; $i++) { + + my $dist = acos((cos($PS_DEG_RAD * $MD_DEC[$i]) * cos($dec) * cos($PS_DEG_RAD * $MD_RA[$i] - $ra)) +(sin($PS_DEG_RAD * $MD_DEC[$i]) * sin($dec))); + + if ($dist < $radius) { + return $MD[$i]; + } + } + print STDERR "failed to find MD field containing $ra_deg $dec_deg\n"; + + return undef; +} + + __END__ Index: /tags/ipp-20111222/pstamp/scripts/detectability_respond.pl =================================================================== --- /tags/ipp-20111222/pstamp/scripts/detectability_respond.pl (revision 33324) +++ /tags/ipp-20111222/pstamp/scripts/detectability_respond.pl (revision 33325) @@ -103,7 +103,7 @@ } # End reading wisdom. else { -# -# Parse input request file using detect_query_read (as it's already written). -# + # + # Parse input request file using detect_query_read (as it's already written). + # my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -118,4 +118,6 @@ { my @column_names = (); + # split output into lines skip until the line which lists the column names is found + # Parse subsequent lines foreach my $entry (split /\n/, (join "", @$stdout_buf)) { if ($entry =~ /^#/) { @@ -123,19 +125,19 @@ shift(@column_names); # Dump the hash sign. } - else { + elsif (scalar @column_names) { # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE my %row_data; @row_data{@column_names} = (split /\s+/, $entry); for (my $i = 0; $i <= $#column_names; $i++) { - push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; - $Nrows = scalar(keys(%query)); -# print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; - } - } - } - } -# -# Identify target images. This should properly collate targets on a single imfile. -# + push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; + $Nrows = scalar(keys(%query)); + # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; + } + } + } + } + # + # Identify target images. This should properly collate targets on a single imfile. + # foreach my $fpa_id (keys %query) { my %temp_hash; @@ -207,4 +209,5 @@ } elsif ($fpa_id eq 'Not_Set') { + # no matching file was found skip next; } @@ -222,5 +225,5 @@ $need_magic = 0; } - my $mjd_min = $mjd; + my $mjd_min = int $mjd; my $mjd_max = $mjd + 1; @@ -239,82 +242,86 @@ foreach my $this_image_ref (@{ $pstamp_images_ref }) { - foreach my $key (sort (keys %{ $this_image_ref } )) { - my $value = ${ $this_image_ref }{$key}; - if ($key eq 'row_index') { - $value = join ' ', @{ $this_image_ref->{$key} }; - } - -# print "$this_image_ref $key $value\n"; - foreach my $valid_index (@{ $this_image_ref->{row_index} }) { - $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; - $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; - $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; - $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; - $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; - $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; - $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; - $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; - - if (exists($this_image_ref->{astrom})) { - $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; - } - else { - $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; - } - if (exists($this_image_ref->{class_id})) { - $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; - $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; - - } - else { - $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; - $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; - } - - $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; - if (exists($this_image_ref->{data_state})) { - $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; - } - else { - $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; - } - $query{$fpa_id}{FAULT}[$valid_index] = 0; + + if (0) { + foreach my $key (sort (keys %{ $this_image_ref } )) { + my $value = ${ $this_image_ref }{$key}; + if ($key eq 'row_index') { + $value = join ' ', @{ $this_image_ref->{$key} }; + } + print "$this_image_ref $key $value\n"; + } + } + + + foreach my $valid_index (@{ $this_image_ref->{row_index} }) { + $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; + $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; + $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; + $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; + $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; + $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; + $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; + $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; + + if (exists($this_image_ref->{astrom})) { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; + } + else { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; + } + if (exists($this_image_ref->{class_id})) { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; + + } + else { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; + } + + $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; + if (exists($this_image_ref->{data_state})) { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; + } + else { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; + } + $query{$fpa_id}{FAULT}[$valid_index] = 0; $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked}; - if ($stage eq 'chip') { - $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; - } - - # Determine if the data exists. - if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or - ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or - ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or - ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or - ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or - ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { - - # image is gone and it's not coming back - $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; - } - elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) { - $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; - } - elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { - $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; - - # updating stacks isn't implemented - if (($stage eq 'stack')) { - $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; - } - # updating old burntool data isn't implemented - elsif ($stage eq 'chip') { - if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and - (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { - $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + if ($stage eq 'chip') { + $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; + } + + # Determine if the data exists. + if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { + + # image is gone and it's not coming back + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; + } + elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; + } + elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; + + # updating stacks isn't implemented + if (($stage eq 'stack')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + # updating old burntool data isn't implemented + elsif ($stage eq 'chip') { + if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and + (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; } - } - } # End determining error faults. - } - } - } + } + } # End determining error faults. + } + } } } # End calculating wisdom @@ -322,27 +329,30 @@ my %processing_request; -if (!$job_id) { - open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); - foreach my $fpa_id (keys %query) { - for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) { - print WISDOM "$fpa_id\t"; - foreach my $key (keys %{ $query{$fpa_id} }) { - print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; +if ($job_id) { + $wisdom_file = "$wisdom_file.$job_id"; +} +open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); +foreach my $fpa_id (keys %query) { + for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) { + print WISDOM "$fpa_id\t"; + foreach my $key (keys %{ $query{$fpa_id} }) { + print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; + } + print WISDOM "\n"; + my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; + if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { + if ($data_state ne 'full') { + + @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = + ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], + $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); } - print WISDOM "\n"; - my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; - if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { - if ($data_state ne 'full') { - - @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = - ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], - $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); - } - push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; - } + push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; } } - close(WISDOM); - +} +close(WISDOM); + +if (! defined $job_id) { # If there is anything that needs to be updated, create the update request list, and then exit the program. my $exit_code = 0; @@ -399,10 +409,12 @@ # if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) { if ($fault = check_component($stage, $stage_id, $component, $imagedb)) { - $query{$fpa_id}{PROC_ERROR}[$index] = $fault; - - $query{$fpa_id}{NPIX}[$index] = 0; - $query{$fpa_id}{QFACTOR}[$index] = 0.0; - $query{$fpa_id}{FLUX}[$index] = 0.0; - $query{$fpa_id}{FLUX_SIG}[$index] = 0.0; + foreach my $i (@{ $processing_request{$fpa_id}{$image} }) { + $query{$fpa_id}{PROC_ERROR}[$i] = $fault; + + $query{$fpa_id}{NPIX}[$i] = 0; + $query{$fpa_id}{QFACTOR}[$i] = 0.0; + $query{$fpa_id}{FLUX}[$i] = 0.0; + $query{$fpa_id}{FLUX_SIG}[$i] = 0.0; + } next; @@ -704,4 +716,8 @@ $inHeader->{FPA_ID}->{value} = $fpa_id; } + + my @keysa = keys %{$query{$fpa_id}}; + + print "$fpa_id: @keysa\n"; push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} }; Index: /tags/ipp-20111222/pstamp/scripts/dquery_finish.pl =================================================================== --- /tags/ipp-20111222/pstamp/scripts/dquery_finish.pl (revision 33324) +++ /tags/ipp-20111222/pstamp/scripts/dquery_finish.pl (revision 33325) @@ -13,4 +13,6 @@ use File::Temp qw( tempfile ); use File::Copy; +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); use PS::IPP::Metadata::Config; @@ -69,13 +71,5 @@ exit 0; } -if (0) { -my $prod_dir = "$outputDataStoreRoot/$product"; -if (! -e $prod_dir ) { - # something must have gone wrong at the parse stage - update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose); - die "product directory does not exist $prod_dir"; -} -my $outdir = "$prod_dir/$req_name"; -} + if (! -e $outdir ) { # something must have gone wrong at the parse stage @@ -97,4 +91,5 @@ { my $command = "$pstamptool -listjob -req_id $req_id"; + $command .= " -jobType detect_query"; # temporary $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -117,28 +112,33 @@ } -# XXX: have the jobs produce the reglist as with postage stamp requests -my ($REGLIST, $reg_list) = tempfile("$outdir/reqlist.XXXX", UNLINK => !$save_temps); - +my ($REGLIST, $reg_list) = tempfile("$outdir/reglist.XXXX", UNLINK => !$save_temps); +my $response_file = "response.$req_id.fits"; +my $response_path = "$outdir/$response_file"; + +print $REGLIST "$response_file|||table|\n"; + +my @columns = qw(ROWNUM PROC_ERROR NPIX QFACTOR FLUX FLUX_SIG FPA_ID STAGE STAGE_ID COMPONENT CMFFILE); +my %colData; +foreach my $col (@columns) { + $colData{$col} = []; +} + +my $i = 0; foreach my $job (@jobs) { - next if $job->{parent_id}; my $job_id = $job->{job_id}; - my $response_file = "response.${job_id}.fits"; - my $response_path = "$outdir/$response_file"; - - if (-e $response_path) { - # the job generated a response file put it into the Data Store - print $REGLIST "$response_file|||table|\n"; - - # do the same if we have an error file. Should the parse data be uploaded as well? - my $err_file = "parse_error.txt"; - if (-e "$outdir/$err_file") { - print $REGLIST "$err_file|||text|\n"; - } - } else { - print STDERR "detect_query response file for job $job_id not found\n" if $verbose; - $request_fault = $PS_EXIT_UNKNOWN_ERROR; - } + + my $outputBase = $job->{outputBase}; + my $results = "${outputBase}results.txt"; + + read_results($results, \%colData); + + my $cmf = $colData{CMFFILE}->[$i]; + print $REGLIST "$cmf|||table|\n"; + + $i++; } close $REGLIST; + +write_response_file($response_path, $req_name, \%colData); if (-s $reg_list) { @@ -162,4 +162,206 @@ exit 0; +sub read_results { + my $results_file = shift; + my $results = shift; + + open IN, "<$results_file" or my_die("failed to open $results_file", $PS_EXIT_UNKNOWN_ERROR); + + foreach my $line () { + # skip header + next if $line =~ /ROWNUM/; + chomp $line; + my @words = split " ", $line; + my ($rownum, $proc_error, $npix, $qfactor, $flux, $flux_sig, $fpa_id, $stage, $stage_id, $component, $cmf) = @words; + push @{$results->{ROWNUM}}, $rownum; + push @{$results->{PROC_ERROR}}, $proc_error; + push @{$results->{NPIX}}, $npix; + push @{$results->{QFACTOR}}, $qfactor; + push @{$results->{FLUX}}, $flux; + push @{$results->{FLUX_SIG}}, $flux_sig; + push @{$results->{FPA_ID}}, $fpa_id; + push @{$results->{STAGE}}, $stage; + push @{$results->{STAGE_ID}}, $stage_id; + push @{$results->{COMPONENT}}, $component; + push @{$results->{CMFFILE}}, $cmf; + } + # XXX: make sure that the size of each of these arrays is the same +} + +sub write_response_file { + my $output = shift; + my $query_id = shift; + my $results = shift; + + my $columns; + my $headers; + + my $EXTVER_IS_1 = 0 ; # (scalar(keys(%query)) == 1); +# print "EXTVER: $EXTVER_IS_1\n"; + my ($FPA_ID,$MJD_OBS,$filter,$obscode,$status); + if ($EXTVER_IS_1 == 1) { + # Specification of columns to write + $columns = [ + # matching rownum from detectability original request + { name => 'ROWNUM', type => '20A', writetype => TSTRING }, + # any errors that occurred during processing + { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, + # number of pixels used in hypothetical PSF for the query detection + { name => 'DETECT_N', type => 'V', writetype => TULONG }, + # detectibility, indicating the fraction of PSF pixels detetable by IPP + { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, + # flux of the target source + { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, + # error in the flux of the target source + { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, + ]; + + # Header translation table + $headers = { + 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, + comment => 'MOPS Query ID for this batch query' }, + 'FPA_ID' => { name => 'FPA_ID', type => TSTRING, + comment => 'original FPA_ID used at ingest' }, +# 'MJD-OBS' => { name => 'FPA_ID', type => TSTRING, +# comment => 'original FPA_ID used at ingest' }, +# 'FILTER' => { name => 'FPA_ID', type => TSTRING, +# comment => 'original FPA_ID used at ingest' }, +# 'OBSCODE' => { name => 'FPA_ID', type => TSTRING, +# comment => 'original FPA_ID used at ingest' }, + }; + } + else { + # Specification of columns to write + $columns = [ + # matching rownum from detectability original request + { name => 'ROWNUM', type => '20A', writetype => TSTRING}, + # any errors that occurred during processing + { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, + # number of pixels used in hypothetical PSF for the query detection + { name => 'DETECT_N', type => 'V', writetype => TULONG }, + # detectibility, indicating the fraction of PSF pixels detetable by IPP + { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, + # flux of the target source + { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, + # error in the flux of the target source + { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, + # The FPA That would be in the header if it were to be there. + { name => 'FPA_ID', type => '20A', writetype => TSTRING }, + ]; + + # Header translation table + $headers = { + 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, + comment => 'MOPS Query ID for this batch query' }, + }; + } + + # Parse the list of columns + my @colNames; # Names of columns + my @colTypes; # Types of columns + my %colData; # Data for each column + my @colWriteType; # type to use to write + foreach my $colSpec ( @$columns) { + push @colNames, $colSpec->{name}; + push @colTypes, $colSpec->{type}; + push @colWriteType, $colSpec->{writetype}; + $colData{$colSpec->{name}} = []; + } + + my $inHeader = { }; + + # Hack to force the data to match the detect_response_create formats + + $inHeader->{QUERY_ID}->{value} = $query_id; + if ($EXTVER_IS_1 == 1) { + my $fpa_id = $results->{FPA_ID}->[0]; + $inHeader->{FPA_ID}->{value} = $fpa_id; + } + + # Fill the table columns with the data, making sure the flux is defined + + @{$colData{'ROWNUM'}} = @{ $results->{ROWNUM} }; + @{$colData{'ERROR_CODE'}} = @{ $results->{PROC_ERROR} }; + @{$colData{'DETECT_N'}} = @{ $results->{NPIX} }; + @{$colData{'DETECT_F'}} = @{ $results->{QFACTOR} }; + @{$colData{'TARGET_FLUX'}} = @{ $results->{FLUX} }; + @{$colData{'TARGET_FLUX_SIG'}} = @{ $results->{FLUX_SIG} }; + @{$colData{'FPA_ID'}} = @{ $results->{FPA_ID} }; + + my $numRows = 0; + # Back to detect_response_create: + $status = 0; +# print "$output\n"; + if (-e $output) { + unlink $output or die "failed to unlink existing response file $output\n"; + } + my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); + check_fitsio( $status ); + $outFits->create_img( 16, 0, undef, $status ); + check_fitsio( $status ); + my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; + + $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status); + check_fitsio( $status ); + + # TODO: get the extension version number from somewhere common + $outFits->write_key( TINT, 'EXTVER', 2, 'Version of this Extension', $status ); + check_fitsio( $status ); + + # Write the Extension keywords + foreach my $keyword ( keys %$headers ) { + my $value = $inHeader->{$keyword}->{value}; # Header keyword value + unless (defined $value) { + print "Can't find header keyword $keyword\n"; + next; + } + $value =~ s/\'//g; + my $name = $headers->{$keyword}->{name}; # New name + my $type = $headers->{$keyword}->{type}; # Type + my $comment = $headers->{$keyword}->{comment}; # Comment + $outFits->write_key( $type, $name, $value, $comment, $status ); + check_fitsio( $status ); + } + + for (my $i = 0; $i < scalar @colNames; $i++) { + my $colName = $colNames[$i];# Column name + my $writeType = $colWriteType[$i]; + my $numRows = scalar(@{$colData{$colName}}); + unless(defined($writeType)) { + print "write type undefined for $colName\n"; + } + unless(defined($numRows)) { + print "num Rows undefined for $colName\n"; + } + unless(defined($status)) { + print "status undefined for $colName\n"; + } + unless(defined($colData{$colName})) { + print "col data undefined for $colName\n"; + } + unless(defined($colName)) { + print "column name undefined for $i\n"; + } + print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n"; + $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status ); + print STDERR "$writeType $i $numRows $colName $status\n"; + check_fitsio( $status ); + } + $outFits->close_file( $status ); + return($status); +} + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + carp("CFITSIO error: $status => $msg"); + die "CFITSIO error: $msg\n"; + } +} sub update_request { my $req_id = shift; @@ -181,2 +383,13 @@ } } + +sub my_die { + my $msg = shift; + my $fault = shift; + + print STDERR $msg; + + update_request($req_id, $fault); + + exit $fault; +} Index: /tags/ipp-20111222/pstamp/scripts/dquery_job_run.pl =================================================================== --- /tags/ipp-20111222/pstamp/scripts/dquery_job_run.pl (revision 33325) +++ /tags/ipp-20111222/pstamp/scripts/dquery_job_run.pl (revision 33325) @@ -0,0 +1,400 @@ +#! /usr/bin/env perl +# +# +# Run a detectability query job +# +# + +use strict; +use warnings; + +use Sys::Hostname; +use Carp; +use File::Basename; +use File::Copy; +use IPC::Cmd 0.36 qw( can_run run ); +use Getopt::Long qw( GetOptions ); +use Pod::Usage qw( pod2usage ); +use File::Temp qw( tempfile tempdir); + +use PS::IPP::PStamp::RequestFile qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); +use PS::IPP::Config qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +# +# Set up +# +my $host = hostname(); + +my $EXTVER = 1.0; +my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; +my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project); +my ($output_base,$dbname,$dbserver,$verbose,$save_temps); +GetOptions( + 'output_base=s' => \$output_base, + 'job_id=s' => \$job_id, + 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, + ) or pod2usage(2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --job_id --output_base", + -exitval => 3, + ) unless + defined $job_id and defined $output_base; + +my $psphotForced = can_run('psphotForced') or (warn "Can't find psphotForced" and $missing_tools = 1); +my $ppCoord = can_run('ppCoord') or (warn "Can't find ppCoord" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $workdir = dirname($output_base); +my $params_file = "${output_base}params.mdc"; + +unless (defined $verbose) { + $verbose = 0; +} + +my $ipprc = PS::IPP::Config->new(); +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} + +my $imagedb = 'gpc1'; +my %args; + +open IN, "<$params_file" + or my_die("failed to open params file: $params_file", "", "", "", "", "", "", $PS_EXIT_UNKNOWN_ERROR); + +my $data = $mdcParser->parse(join "", ()) + or my_die("failed to parse paramsfile: $params_file", "", "", "", "", "", "", $PS_EXIT_UNKNOWN_ERROR); +my $refs = parse_md_list($data); +if (scalar @$refs < 2) { + my_die("invalid paramsfile: $params_file", "", "", "", "", "", "", $PS_EXIT_UNKNOWN_ERROR); +} + +my $params = shift @$refs; +my @rows = @$refs; + +my $nRows = scalar @rows; +print "Read $nRows rows from $params_file\n"; + +#exit 0; + +# run ppCoord and psphotForced to calculate the required data. +my $fpa_id = $params->{FPA_ID}; +my $fault = $params->{FAULT}; +my $catalog = $params->{CATALOG}; +my $psf = $params->{PSF}; +my $image = $params->{IMAGE}; +my $mask = $params->{MASK}; +my $weight= $params->{WEIGHT}; +my $stage = $params->{STAGE}; +my $stage_id = $params->{STAGE_ID}; +my $component = $params->{COMPONENT_ID}; +# print "Input is from $stage $stage_id $component\n"; + +# XXX: TODO: if there's a fault or quality problem, then we can't process this image. +# Perhaps we shouldn't even get here + +# Create coordinate file to convert to positions. +my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$job_id.XXXX", + UNLINK => !$save_temps); +my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$job_id.XXXX", + UNLINK => !$save_temps); +foreach my $row (@rows) { + print $coordfile "$row->{RA1_DEG} $row->{DEC1_DEG}\n"; +} + +# Convert the sky coordinates to image coordinates with ppCoord. +my $command = "ppCoord -astrom $catalog -radec $coordname"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); +unless ($success) { + my_die("Unable to perform $command. Error_code: $error_code", + $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'}, + $params->{FILTER},$params->{OBSCODE},$params->{STAGE}, + $error_code); +} +my @response = split /\n/, (join "", @$stdout_buf); +my $i = 0; +foreach my $line (@response) { + my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line; + print $targetfile "$r_x $r_y\n"; + my $row = $rows[$i]; + if ((abs($r_ra - $row->{RA1_DEG}) < 1e-8) && (abs($r_dec - $row->{DEC1_DEG}))) { + $row->{X_PXL} = $r_x; + $row->{Y_PXL} = $r_y; + $row->{EXTENSION_BASE} = $r_chip; + } else { + $error_code = $PS_EXIT_PROG_ERROR; + my_die("Unable to match input RA/DEC with output RA/DEC: ($row->{RA1_DEG},$row->{DEC1_DEG}) -> ($r_ra,$r_dec) $error_code", + $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'}, + $params->{FILTER},$params->{OBSCODE},$params->{STAGE}, + $error_code); + } + $i++; +} + +# Run psphotForced on the target list. +my $outroot = "${output_base}detectability.${stage}.${stage_id}"; +$outroot .= ".$component" unless $stage eq 'chip'; + +$params->{PROC_ERROR} = 0; +my $psphot_cmd = "$psphotForced -psf $psf "; +$psphot_cmd .= "-file $image "; +$psphot_cmd .= "-mask $mask "; +$psphot_cmd .= "-variance $weight "; +$psphot_cmd .= "-srctext $targetname $outroot "; +$psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 "; + +( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $psphot_cmd, verbose => $verbose); +unless ($success) { + # $params->{PROC_ERROR} = $PSTAMP_SYSTEM_ERROR; + my $rc = $error_code >> 8; + my_die("$command failed: $error_code $rc", + $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'}, + $params->{FILTER},$params->{OBSCODE},$params->{STAGE}, + $rc); +} + +my $results = "${output_base}results.txt"; +open OUT, ">$results" or + my_die("Failed to open results file $results", + $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'}, + $params->{FILTER},$params->{OBSCODE},$params->{STAGE}, + $PS_EXIT_UNKNOWN_ERROR); + +# Read the output cmf and produce the data we need for the response file +my $class_id = $params->{CLASS_ID}; +my $cmf = "${outroot}.${class_id}.cmf"; +my $cmfbase = basename($cmf); +my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$rows[0]->{EXTENSION_BASE}); +$i = 0; +print OUT "ROWNUM PROC_ERROR NPIX QFACTOR FLUX FLUX_SIG FPA_ID STAGE STAGE_ID COMPONENT CMFFILE\n"; +foreach my $row (@rows) { + $row->{PROC_ERROR} = $params->{PROC_ERROR}; + + $row->{NPIX} = ${ $tmp_Npix }[$i]; + $row->{QFACTOR} = ${ $tmp_Qfact }[$i]; + $row->{FLUX} = ${ $tmp_flux }[$i]; + $row->{FLUX_SIG} = ${ $tmp_flux_error }[$i]; + + print OUT "$row->{ROWNUM} $row->{PROC_ERROR} $row->{NPIX} $row->{QFACTOR} $row->{FLUX} $row->{FLUX_SIG} $fpa_id $stage $stage_id $component $cmfbase\n"; + $i++; +} +close(OUT); + +exit 0; + +# +# Utilities +# + +# Taken largely from detect_query_read +sub read_cmf_file { + my $cmf_file = shift; + my $class_id = shift; + my $extname = $class_id . ".psf"; + + my @tmp_Npix = (); + my @tmp_Qfact = (); + my @tmp_flux = (); + my @tmp_flux_err = (); + + my ($detect_F_col,$detect_N_col,$detect_flux_col,$detect_mag_col) = (-1, -1, -1, -1); + + + my $status = 0; + my $inFits = Astro::FITS::CFITSIO::open_file( $cmf_file, READONLY, $status ); # Open CMF + check_fitsio($status); + $inFits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); + + my $inHeader = $inFits->read_header(); + + my $CMFversion = $inHeader->{EXTTYPE}; + $CMFversion =~ s/\'//g; + $CMFversion =~ s/\s+//g; + + # This is the only data we actually care about + my $column_defs; + if ($CMFversion eq 'PS1_V2') { + $column_defs = [ + { name => 'PSF_QF', type => '1E', writetype => TDOUBLE }, + { name => 'PSF_NPIX', type => '1J', writetype => TLONG }, + { name => 'PSF_INST_MAG', type => '1E', writetype => TDOUBLE }, + { name => 'PSF_INST_MAG_SIG', type => '1E', writetype => TDOUBLE }, + ]; + } + elsif ($CMFversion eq 'PS1_DV1') { + $column_defs = [ + { name => 'PSF_QF', type => '1E', writetype => TDOUBLE }, + { name => 'PSF_NPIX', type => '1J', writetype => TLONG }, + { name => 'PSF_INST_FLUX', type => '1E', writetype => TDOUBLE }, + { name => 'PSF_INST_FLUX_SIG', type => '1E', writetype => TDOUBLE }, + ]; + } + + my @colNames; + my @colTypes; + my @colWriteType; + my %colData; + foreach my $colSpec (@$column_defs) { + push @colNames, $colSpec->{name}; + push @colTypes, $colSpec->{type}; + push @colWriteType, $colSpec->{writetype}; + } + my $numRows; + $inFits->get_num_rows($numRows, $status) and check_fitsio($status); + + my $correct_error = 0; +# print STDERR "Ncols:" . $#{ $column_defs } . "\n"; + foreach my $col (@$column_defs) { + my ($col_num,$col_type,$col_data); + $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status); + if ($status != 0) { + $status = 0; + next; + } + $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); + $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) and check_fitsio($status); +# print STDERR "$col\t>>" . $col->{name} . "<<\t>>" . @{ $col_data } . "<<\n"; + if ($col->{name} eq 'PSF_QF') { + @tmp_Qfact = @{ $col_data }; + } + elsif ($col->{name} eq 'PSF_NPIX') { + @tmp_Npix = @{ $col_data }; + } + elsif ($col->{name} eq 'PSF_INST_MAG') { + @tmp_flux = map { $_ = 10**(-0.4 * $_) } @{ $col_data }; + } + elsif ($col->{name} eq 'PSF_INST_MAG_SIG') { + @tmp_flux_err = map { $_ = $_ / (2.5 * (log(exp(1)) / log(10))) } @{ $col_data }; + $correct_error = 1; + } + elsif ($col->{name} eq 'PSF_INST_FLUX') { + @tmp_flux = @{ $col_data }; + } + elsif ($col->{name} eq 'PSF_INST_FLUX_SIG') { + @tmp_flux_err = @{ $col_data }; + } + } + if ($correct_error) { + for (my $c = 0; $c <= $#tmp_flux_err; $c++) { + $tmp_flux_err[$c] = $tmp_flux_err[$c] * $tmp_flux[$c]; + } + } + $inFits->close_file( $status ) and check_fitsio($status); +# print STDERR "$CMFversion\n"; +# print STDERR "Q: $#tmp_Qfact\t@tmp_Qfact\nN: $#tmp_Npix\t@tmp_Npix\n"; + return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux, \@tmp_flux_err); +} + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + carp("CFITSIO error: $status => $msg"); + die "CFITSIO error: $msg\n"; + } +} + +sub check_component { + my ($stage, $stage_id, $component, $imagedb) = @_; + + print "Checking status of component $stage $stage_id $component\n"; + + my $command; + if ($stage eq 'diff') { + $command = "difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; + } elsif ($stage eq 'stack') { + $command = "stacktool -sumskyfile -stack_id $stage_id"; + } elsif ($stage eq 'warp') { + $command = "warptool -warped -warp_id $stage_id -skycell_id $component"; + } elsif ($stage eq 'chip') { + $command = "chiptool -processedimfile -chip_id $stage_id -class_id $component"; + } else { + die("check_component not implemented for stage $stage yet"); + } + + $command .= " -dbname $imagedb"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my $rc = $error_code >> 8; + carp "$command failed $error_code $rc"; + exit $PS_EXIT_UNKNOWN_ERROR; + } + my $output = join "", @$stdout_buf; + my $metadata = $mdcParser->parse($output); + my $results = parse_md_list($metadata); + if (scalar @$results != 1) { + carp "$command returned too many components: " . scalar @$results; + exit $PS_EXIT_UNKNOWN_ERROR; + } + my $it = $results->[0]; + + if ($stage eq 'stack') { + $it->{data_state} = $it->{state}; + } + + my $return_status = 0; + if ($it->{quality}) { + print " Bad quality: $it->{quality}\n"; + $return_status = $PSTAMP_GONE; + } elsif ($it->{fault}) { + print " Faulted: $it->{fault}\n"; + $return_status = $PSTAMP_GONE; + } elsif ($it->{data_state} ne 'full') { + # XXX does this work for stack? + carp " Faulted: $it->{data_state}\n"; + $return_status = $PSTAMP_GONE; + } else { + print " Component ok.\n"; + } + + return $return_status; +} + +sub my_die { + my $message = shift; + my $QUERY_ID = shift; + my $FPA_ID = shift; + my $MJD_OBS = shift; + my $FILTER = shift; + my $OBSCODE = shift; + my $STAGE = shift; + + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp("$message : $QUERY_ID $FPA_ID $MJD_OBS $FILTER $OBSCODE $STAGE"); + exit($exit_code); + +} + +sub exit_with_failure { + my $status = shift; + my $message = shift; + carp("$message"); + exit($status); +} + Index: /tags/ipp-20111222/pstamp/scripts/dqueryparse.pl =================================================================== --- /tags/ipp-20111222/pstamp/scripts/dqueryparse.pl (revision 33324) +++ /tags/ipp-20111222/pstamp/scripts/dqueryparse.pl (revision 33325) @@ -1,87 +1,82 @@ -#!/usr/bin/env perl -# -# parse a MOPS_DETCTABILITY_QUERY table and create a results file +#! /usr/bin/env perl +# +# +# Create a response to a MOPS_DETECTABILITY_QUERY +# # use strict; use warnings; + +use Sys::Hostname; use Carp; +use File::Basename; +use File::Copy; +use IPC::Cmd 0.36 qw( can_run run ); use Getopt::Long qw( GetOptions ); use Pod::Usage qw( pod2usage ); -use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile tempdir); use PS::IPP::PStamp::RequestFile qw( :standard ); use PS::IPP::PStamp::Job qw( :standard ); -use PS::IPP::Config qw($PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - metadataLookupStr - metadataLookupBool - caturi - ); - -my ($no_update, $imagedb, $label); -my ($req_file, $req_id, $outdir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps); -my ($job_id,$rownum); # stuff from the post-update world -# -# parse args -# - +use PS::IPP::Config qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); + +# use Astro::FITS::CFITSIO qw( :constants ); +# Astro::FITS::CFITSIO::PerlyUnpacking(1); + +my $host = hostname(); + +my $mode = 'queue_job'; + +my ($req_id, $product, $need_magic, $missing_tools, $project, $label); +my ($request_file, $outdir, $dbname, $dbserver, $verbose, $save_temps, $no_update); GetOptions( - 'file=s' => \$req_file, - 'req_id=s' => \$req_id, - 'job_id=s' => \$job_id, - 'rownum=s' => \$rownum, - 'outdir=s' => \$outdir, - 'label=s' => \$label, - 'product=s' => \$product, - 'mode=s' => \$mode, - 'dbname=s' => \$dbname, - 'dbserver=s' => \$dbserver, - 'verbose' => \$verbose, - 'save-temps' => \$save_temps, -) or pod2usage(2); - -die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job"); -die "--file or --job_id is required" if !$req_file; - -if ($mode ne "list_uri") { - die "req_id is required" if !$req_id; - die "outdir is required" if !$outdir; - die "product is required" if !$product; -} -my $missing_tools; -my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1); -my $detectresponse = can_run('detectability_respond.pl') or - (warn "Can't find detectability_respond.pl" and $missing_tools = 1); -my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1); - + 'file=s' => \$request_file, + 'req_id=s' => \$req_id, + 'outdir=s' => \$outdir, + 'label=s' => \$label, + 'product=s' => \$product, + 'mode=s' => \$mode, + 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, + 'no-update' => \$no_update, + ) or pod2usage(2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file --req_id --outdir --dbname", + -exitval => 3, + ) unless + defined $request_file and defined $outdir and defined $dbname and defined $req_id; + +my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1); +my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); +my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); -} + exit($PS_EXIT_CONFIG_ERROR); +} + +unless (defined $verbose) { + $verbose = 0; +} + +my $ipprc = PS::IPP::Config->new(); +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + if (!$dbserver) { - my $ipprc = PS::IPP::Config->new(); - $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); -} - -# just deal with these arguments once and for all -$pstamptool .= " -dbname $dbname" if $dbname; -$pstamptool .= " -dbserver $dbserver" if $dbserver; -$detectresponse .= " --dbname $dbname" if $dbname; - -$no_update = 1 if $mode eq "list_job"; - -# Unless we're running as a job, write the parse arguments in case we need to rerun this parsing. -if (!$job_id) { - my $argslist = "$outdir/parse.args"; - open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); - print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --outdir $outdir --file $req_file\n"; - close ARGSLIST; + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} + +$pstamptool .= " -dbname $dbname -dbserver $dbserver"; + +# Project name is hardcoded to gpc1 for the moment. +$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver); +my $imagedb = $project->{dbname}; +if (!$imagedb) { + my_die("failed to find imagedb for project: $project", $PS_EXIT_CONFIG_ERROR); } @@ -89,5 +84,5 @@ my $fields_output; { - my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID"; + my $command = "echo $request_file | $fields -x 0 EXTNAME EXTVER QUERY_ID"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -97,168 +92,353 @@ my (undef, $extname, $extver, $req_name) = split " ", $fields_output; -my_die("$req_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR) +my_die("$request_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR) if !(defined($extname) and defined($extver) and defined($req_name)); -my_die("$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR) +my_die("$request_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR) if $extname ne "MOPS_DETECTABILITY_QUERY"; -my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR) +my_die("$request_file is version $extver expecting 1 or 2", $PS_EXIT_PROG_ERROR) if ($extver ne 1) and ($extver ne 2); -# Set up the workdir for this query. -if (! -e $outdir ) { - mkdir $outdir or my_die("cannot create output directory $outdir", $PS_EXIT_PROG_ERROR); -} elsif (! -d $outdir ) { - my_die ("output fileset directory $outdir exists but is not a directory", $PS_EXIT_PROG_ERROR); -} - - -# Pass along the request file to the response generator. -my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits"; -my $fault; -# my $data_to_update = ''; + +# +# query is a hash which uses $fpa_id as the key +# The values are hash references which have the various parameter name as the key +# The values of the parameter hashes are arrays which contain the values for individual rows +# in the detectabilty query request +my %query = (); + { - my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir"; - if ($job_id) { - $command .= " --job_id $job_id"; - } else { - $command .= " --ignore-wisdom"; - } - $command .= " --save-temps" if $save_temps; - $command .= " --verbose" if $verbose; - + # + # Parse input request file using detect_query_read + # result is a simple text file + # + my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $dqr_command, verbose => $verbose); unless ($success) { - warn("Warn! Unable to perform $command error code: $error_code"); - } - # Use the fault code to see if we can regenerate the missing data. - $fault = $error_code >> 8; - if ($fault == $PSTAMP_NOT_AVAILABLE) { - unless (-e "$outdir/update_request.dat") { - my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR); - } - warn("Some inputs are not available and must be updated."); - } - -} - -my $result; -unless ($job_id) { - # We are running as a parse job - # If we returned correctly with a valid response file, get a job ID - # for the completed work, and move the response to a standardized name. - if ($fault == 0) { - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; - $command .= " -job_type detect_query -state stop -fault 0"; - $command .= " -rownum 1"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if ($success) { - $job_id = join "", @$stdout_buf; - chomp $job_id; - if ($job_id && -e $response_file) { - rename $response_file, "$outdir/response.${job_id}.fits"; + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $dqr_command error code: $error_code"); + } + my $Nrows = 0; + { + my @column_names = (); + # split output into lines skip until the line which lists the column names is found + # Parse subsequent lines + foreach my $entry (split /\n/, (join "", @$stdout_buf)) { + if ($entry =~ /^#/) { + @column_names = split /\s+/, $entry; + shift(@column_names); # Dump the hash sign. + } + elsif (scalar @column_names) { + # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE + my %row_data; + @row_data{@column_names} = (split /\s+/, $entry); + for (my $i = 0; $i <= $#column_names; $i++) { + push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; + $Nrows = scalar(keys(%query)); + # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; + } + } + } + } + # + # Identify target components. This should properly collate targets on a single imfile. + # + my $query_id; + foreach my $fpa_id (keys %query) { + my %temp_hash; + my $query_style = 'byexp'; + my $stage; + my $filter; + my $mjd; + # Confirm that we only have one stage/filter/mjd + if ($fpa_id ne 'Not_Set') { # We only need to check things that aren't the known odd case. + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1; + $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1; + $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1; + } + if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) { + $stage = (keys(%{ $temp_hash{STAGE} }))[0]; + } + else { + exit_with_failure(21,"Too many STAGEs specified"); + } + if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) { + $filter = (keys(%{ $temp_hash{FILTER} }))[0]; + } + else { + exit_with_failure(21,"Too many FILTERs specified"); + } + if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) { + $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0]; + } + else { + exit_with_failure(21,"Too many MJD-OBS specified"); + } + } + + # Set up a rowList with default values + my @rowList; + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i]; + $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i]; + $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i]; + $rowList[$i]->{COORD_MASK} = 0; + # Set default values + $query{$fpa_id}{BAD_COMPONENT}[$i] = 1; + $query{$fpa_id}{IMAGE}[$i] = 'no_image'; + $query{$fpa_id}{MASK}[$i] = 'no_mask'; + $query{$fpa_id}{WEIGHT}[$i] = 'no_weight'; + $query{$fpa_id}{PSF}[$i] = 'no_psf'; + + $query{$fpa_id}{STAGE_ID}[$i] = 'no_id'; + $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb'; + $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic'; + $query{$fpa_id}{MAGICKED}[$i] = 'no_magic'; + $query{$fpa_id}{CATALOG}[$i] = 'no_catalog'; + $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component'; + $query{$fpa_id}{CLASS_ID}[$i] = 'no_class'; + + $query{$fpa_id}{STATE}[$i] = 'no_state'; + $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate'; + $query{$fpa_id}{FAULT}[$i] = 'no_fault'; + $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate'; + } + + # Determine the query style for this fpa_id + if ($fpa_id =~ /o.*g.*o/) { + $query_style = 'byexp'; + } + elsif ($fpa_id =~ /\d+/) { + $query_style = 'byid'; + } + elsif ($fpa_id eq 'Not_Set') { + # no matching file was found skip + next; + } + else { + exit_with_failure(21,"Parse error in request file"); + } + # Set common request components + my $option_mask |= 1; + $option_mask |= $PSTAMP_SELECT_IMAGE; + $option_mask |= $PSTAMP_SELECT_MASK; + $option_mask |= $PSTAMP_SELECT_VARIANCE; + $option_mask |= $PSTAMP_SELECT_PSF; + + # magic is dead + my $need_magic = 0; + if ($stage eq 'stack') { + $need_magic = 0; + } + my $mjd_min = int $mjd; + my $mjd_max = $mjd + 1; + + my $req_filter; + if ($filter ne 'Not_Set') { + $req_filter = $filter . '%'; + } + + # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. + my $pstamp_images_ref = locate_images($ipprc,$imagedb, + \@rowList, + $query_style,$stage, + $fpa_id,undef,undef, + $option_mask,$need_magic, + $mjd_min,$mjd_max,$req_filter,undef,$verbose); + + foreach my $this_image_ref (@{ $pstamp_images_ref }) { + + # loop over the rows that this component matched + foreach my $valid_index (@{ $this_image_ref->{row_index} }) { + $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; + $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; + $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; + $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; + $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; + $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; + $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; + $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; + + if (exists($this_image_ref->{astrom})) { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; + } + else { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; + } + if (exists($this_image_ref->{class_id})) { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; + + } + else { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; + } + + $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; + if (exists($this_image_ref->{data_state})) { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; + } + else { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; + } + $query{$fpa_id}{FAULT}[$valid_index] = 0; + $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked}; + if ($stage eq 'chip') { + $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; + } + + # Determine if the data exists. + if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { + + # image is gone and it's not coming back + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; + } + elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; + } + elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; + + # updating stacks isn't implemented + if (($stage eq 'stack')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + # updating old burntool data isn't implemented + elsif ($stage eq 'chip') { + if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and + (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + } + } # End determining error faults. } - $result = 0; + } + } +} + +my %update_request; +my %rows_for_component; +my %faulted_rows; + +# now build a hash using image name as key which contains the list of rows for that image +foreach my $fpa_id (keys %query) { + for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) { + if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { + push @{ $rows_for_component{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - } - } - elsif ($fault == $PSTAMP_NOT_AVAILABLE) { - # Failed to run correctly, which means that we need to queue a job and flag data for updating. - # first create a parent job for the actual detectabilty query - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/1_"; - $command .= " -job_type detect_query -state run -fault 0 -is_parent"; - $command .= " -rownum 1"; - - my $parent_job_id; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if ($success) { - # pstamptool returns the job_id of the new job - $parent_job_id = join "", @$stdout_buf; - chomp $parent_job_id; - if ($parent_job_id) { - $result = 0; + $faulted_rows{$query{$fpa_id}{ROWNUM}[$i]} = $query{$fpa_id}{FAULT}[$i]; + } + } +} + +# +# Finally build the parameter lists and queue jobs for each component +# + +my $job_num = 1; +foreach my $fpa_id (keys %rows_for_component) { + foreach my $component (keys % {$rows_for_component{$fpa_id} } ) { + my $i = $rows_for_component{$fpa_id}{$component}[0]; + my $dep_id = 0; + my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; + if ($data_state ne 'full') { + # Queue this data to be updated + my $stage = $query{$fpa_id}{STAGE}[$i]; + my $stage_id = $query{$fpa_id}{STAGE_ID}[$i]; + my $component = $query{$fpa_id}{COMPONENT_ID}[$i]; + print "Need to UPDATE $stage $stage_id $component from $data_state\n"; + #### my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_; + $dep_id = queue_update_run($req_id, $outdir, $label, $data_state, $stage, $stage_id, $component, 0, $imagedb); + } + + my $outputBase = "$outdir/$job_num" . "_"; + + # params_file is an mdc description containing the parameters for the job + my $params_file = $outputBase . "params.mdc"; + + open PARAMS, ">$params_file" or my_die("failed to create $params_file", $PS_EXIT_UNKNOWN_ERROR); + + # first the job params + print PARAMS "dqueryJobParams METADATA\n"; + foreach my $key (keys %{ $query{$fpa_id} } ) { + my $value = $query{$fpa_id}{$key}[0]; + print PARAMS "\t$key\t\tSTR\t$value\n"; + } + print PARAMS "END\n\n"; + + # then one structure with the coordinates for each row + print PARAMS "dqueryCoord MULTI\n"; + + my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG); + foreach my $row ( @{ $rows_for_component{$fpa_id}{$component} }) { + print PARAMS "\ndqueryCoord METADATA\n"; + foreach my $key (@coordkeys) { + my $value = $query{$fpa_id}{$key}[$row]; + print PARAMS "\t$key\t\tSTR\t$value\n"; + } + print PARAMS "END\n"; + } + close(PARAMS); + + # use first rownum for this component as the rownum for the job + my $rownum = $query{$fpa_id}{ROWNUM}[$rows_for_component{$fpa_id}{$component}[0]]; + + my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outputBase -job_type detect_query -state run -rownum $rownum"; + $command .= " -dep_id $dep_id" if $dep_id; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if ($success) { + my $job_id = join "", @$stdout_buf; + print " Queued job: $job_id\n"; } else { - print STDERR "pstamptool did not return a valid job_id for parent job\n"; - $result = $PS_EXIT_UNKNOWN_ERROR; + my $exit_status = $error_code >> 8; + $exit_status = $PS_EXIT_UNKNOWN_ERROR unless $exit_status; + my_die("Unable to perform $command error code: $error_code", $exit_status); } } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - } - # now create child jobs for each dependent. All these jobs do is finish once the dependency - # is satisfied - if (!$result) { - # Get the dependency id for the data we need to have updated. - my $job_num = 2; - open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR); - while () { - my $data_to_update = $_; - chomp($data_to_update); - my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update); - - # Link this request to a job and link that job to the dependent - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/${job_num}_"; - $command .= " -job_type child -state run -fault 0 -parent_id $parent_job_id"; - $command .= " -rownum 1"; # XXX: we should choose a correct rownum - $command .= " -dep_id $dep_id" if $dep_id; - - $job_num++; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if ($success) { - $job_id = join "", @$stdout_buf; - chomp $job_id; - $result = 0; - } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - last; - } - } - close(UPDATE_REQUEST); - } - } - else { - my_die ("Parse fault!!", $fault); - } - # This does not set the request state to stop. That will happen with the request_finish.pl script, - # which will notice that we've inserted the stopped job and decide we're finished. Easy enough. - { - my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product"; - $command .= " -set_fault $result" if $result; - + print STDERR "skipping $command\n"; + } + $job_num++; + } +} + +# XXX: flesh this out once we have a request without duplicate ROWNUMS +foreach my $rownum (keys %faulted_rows ) { + my $fault = $faulted_rows{$rownum}; + $fault = $PSTAMP_NO_IMAGE_MATCH if ($fault eq 'no_fault'); + print STDERR "insert faulted job for $rownum: $fault\n"; +} + +{ + my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product"; + # $command .= " -set_fault $result" if $result; + + unless ($no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR); - } - } -} else { - # We are running as a job presumably because an input needed - # to be regenerated - if ($fault) { - # in some cases we will need to finish off the request - my_die ("Run fault!!", $fault); - } - if (-e $response_file) { - rename $response_file, "$outdir/response.${job_id}.fits"; - } -} - - + my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR); + } + } else { + print STDERR "Skipping $command\n"; + } +} + exit 0; - -# If we have to queue an update run, do so and create a new dependent sub queue_update_run { - my ($req_id, $job_id, $outdir, $label, $data_to_update) = @_; - - my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update; + my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_; + +# my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update; if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { @@ -293,14 +473,20 @@ return($dep_id); } - - + sub my_die { - my $msg = shift; - my $fault = shift; - - carp $msg; - - # we don't fault the request here pstamp_parser_run.pl handles that if necessary - exit $fault; -} + my $message = shift; + my $exit_code = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp("$message"); + exit($exit_code); +} + +sub exit_with_failure { + my $status = shift; + my $message = shift; + my_die($message, $status); +} + Index: /tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl (revision 33324) +++ /tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl (revision 33325) @@ -79,5 +79,5 @@ my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); -my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1); +my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1); my $streaksreplace = can_run('streaksreplace') or (warn "Can't find streaksreplace" and $missing_tools = 1); my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); @@ -303,18 +303,15 @@ } } elsif ($jobType eq "detect_query") { - # detect_query jobs are basically holders to note that we need a file updated before we can continue. - # Load the argument list that dqueryparse should have created the first time it ran, so we know how to - # run it again the same way. - my $outdir = dirname($outputBase); - my $argslist = "$outdir/parse.args"; - open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); - my $argString = ; - close ARGSLIST; - chomp $argString; + # my $outdir = dirname($outputBase); + # my $argslist = "$outdir/parse.args"; + # open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); + # my $argString = ; + # close ARGSLIST; + # chomp $argString; # XXX: should we do any other sanity checking? - my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; - - my $command = "$dqueryparse --job_id $job_id $argString"; + # my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; + + my $command = "$dquery_job_run --job_id $job_id --output_base $outputBase"; $command .= " --dbname $dbname" if $dbname; $command .= " --dbserver $dbserver" if $dbserver; @@ -329,5 +326,5 @@ } else { $jobStatus = $error_code >> 8; - my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus); + my_die("dquery_job_run.pl failed with error code: $jobStatus", $job_id, $jobStatus); } } elsif ($jobType eq "child") {