Index: trunk/pstamp/scripts/dqueryparse.pl =================================================================== --- trunk/pstamp/scripts/dqueryparse.pl (revision 33296) +++ trunk/pstamp/scripts/dqueryparse.pl (revision 33299) @@ -1,87 +1,82 @@ -#!/usr/bin/env perl +#! /usr/bin/env perl # -# parse a MOPS_DETCTABILITY_QUERY table and create a results file +# +# Create a response to a MOPS_DETECTABILITY_QUERY +# # use strict; use warnings; + +use Sys::Hostname; use Carp; +use File::Basename; +use File::Copy; +use IPC::Cmd 0.36 qw( can_run run ); use Getopt::Long qw( GetOptions ); use Pod::Usage qw( pod2usage ); -use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile tempdir); use PS::IPP::PStamp::RequestFile qw( :standard ); use PS::IPP::PStamp::Job qw( :standard ); -use PS::IPP::Config qw($PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - metadataLookupStr - metadataLookupBool - caturi - ); - -my ($no_update, $imagedb, $label); -my ($req_file, $req_id, $outdir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps); -my ($job_id,$rownum); # stuff from the post-update world -# -# parse args -# - +use PS::IPP::Config qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); + +# use Astro::FITS::CFITSIO qw( :constants ); +# Astro::FITS::CFITSIO::PerlyUnpacking(1); + +my $host = hostname(); + +my $mode = 'queue_job'; + +my ($req_id, $product, $need_magic, $missing_tools, $project, $label); +my ($request_file, $outdir, $dbname, $dbserver, $verbose, $save_temps, $no_update); GetOptions( - 'file=s' => \$req_file, - 'req_id=s' => \$req_id, - 'job_id=s' => \$job_id, - 'rownum=s' => \$rownum, - 'outdir=s' => \$outdir, - 'label=s' => \$label, - 'product=s' => \$product, - 'mode=s' => \$mode, - 'dbname=s' => \$dbname, - 'dbserver=s' => \$dbserver, - 'verbose' => \$verbose, - 'save-temps' => \$save_temps, -) or pod2usage(2); - -die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job"); -die "--file or --job_id is required" if !$req_file; - -if ($mode ne "list_uri") { - die "req_id is required" if !$req_id; - die "outdir is required" if !$outdir; - die "product is required" if !$product; -} -my $missing_tools; -my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1); -my $detectresponse = can_run('detectability_respond.pl') or - (warn "Can't find detectability_respond.pl" and $missing_tools = 1); -my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1); - + 'file=s' => \$request_file, + 'req_id=s' => \$req_id, + 'outdir=s' => \$outdir, + 'label=s' => \$label, + 'product=s' => \$product, + 'mode=s' => \$mode, + 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, + 'no-update' => \$no_update, + ) or pod2usage(2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file --req_id --outdir --dbname", + -exitval => 3, + ) unless + defined $request_file and defined $outdir and defined $dbname and defined $req_id; + +my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1); +my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); +my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); -} + exit($PS_EXIT_CONFIG_ERROR); +} + +unless (defined $verbose) { + $verbose = 0; +} + +my $ipprc = PS::IPP::Config->new(); +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + if (!$dbserver) { - my $ipprc = PS::IPP::Config->new(); - $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); -} - -# just deal with these arguments once and for all -$pstamptool .= " -dbname $dbname" if $dbname; -$pstamptool .= " -dbserver $dbserver" if $dbserver; -$detectresponse .= " --dbname $dbname" if $dbname; - -$no_update = 1 if $mode eq "list_job"; - -# Unless we're running as a job, write the parse arguments in case we need to rerun this parsing. -if (!$job_id) { - my $argslist = "$outdir/parse.args"; - open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); - print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --outdir $outdir --file $req_file\n"; - close ARGSLIST; + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} + +$pstamptool .= " -dbname $dbname -dbserver $dbserver"; + +# Project name is hardcoded to gpc1 for the moment. +$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver); +my $imagedb = $project->{dbname}; +if (!$imagedb) { + my_die("failed to find imagedb for project: $project", $PS_EXIT_CONFIG_ERROR); } @@ -89,5 +84,5 @@ my $fields_output; { - my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID"; + my $command = "echo $request_file | $fields -x 0 EXTNAME EXTVER QUERY_ID"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -97,168 +92,359 @@ my (undef, $extname, $extver, $req_name) = split " ", $fields_output; -my_die("$req_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR) +my_die("$request_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR) if !(defined($extname) and defined($extver) and defined($req_name)); -my_die("$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR) +my_die("$request_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR) if $extname ne "MOPS_DETECTABILITY_QUERY"; -my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR) +my_die("$request_file is version $extver expecting 1 or 2", $PS_EXIT_PROG_ERROR) if ($extver ne 1) and ($extver ne 2); -# Set up the workdir for this query. -if (! -e $outdir ) { - mkdir $outdir or my_die("cannot create output directory $outdir", $PS_EXIT_PROG_ERROR); -} elsif (! -d $outdir ) { - my_die ("output fileset directory $outdir exists but is not a directory", $PS_EXIT_PROG_ERROR); -} - - -# Pass along the request file to the response generator. -my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits"; -my $fault; -# my $data_to_update = ''; + +my %query = (); +my %image_list_hash = (); + { - my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir"; - if ($job_id) { - $command .= " --job_id $job_id"; - } else { - $command .= " --ignore-wisdom"; - } - $command .= " --save-temps" if $save_temps; - $command .= " --verbose" if $verbose; - + # + # Parse input request file using detect_query_read + # + my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $dqr_command, verbose => $verbose); unless ($success) { - warn("Warn! Unable to perform $command error code: $error_code"); - } - # Use the fault code to see if we can regenerate the missing data. - $fault = $error_code >> 8; - if ($fault == $PSTAMP_NOT_AVAILABLE) { - unless (-e "$outdir/update_request.dat") { - my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR); - } - warn("Some inputs are not available and must be updated."); - } - -} - -my $result; -unless ($job_id) { - # We are running as a parse job - # If we returned correctly with a valid response file, get a job ID - # for the completed work, and move the response to a standardized name. - if ($fault == 0) { - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; - $command .= " -job_type detect_query -state stop -fault 0"; - $command .= " -rownum 1"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if ($success) { - $job_id = join "", @$stdout_buf; - chomp $job_id; - if ($job_id && -e $response_file) { - rename $response_file, "$outdir/response.${job_id}.fits"; + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $dqr_command error code: $error_code"); + } + my $Nrows = 0; + { + my @column_names = (); + # split output into lines skip until the line which lists the column names is found + # Parse subsequent lines + foreach my $entry (split /\n/, (join "", @$stdout_buf)) { + if ($entry =~ /^#/) { + @column_names = split /\s+/, $entry; + shift(@column_names); # Dump the hash sign. + } + elsif (scalar @column_names) { + # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE + my %row_data; + @row_data{@column_names} = (split /\s+/, $entry); + for (my $i = 0; $i <= $#column_names; $i++) { + push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; + $Nrows = scalar(keys(%query)); + # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; + } + } + } + } + # + # Identify target images. This should properly collate targets on a single imfile. + # + my $query_id; + foreach my $fpa_id (keys %query) { + my %temp_hash; + my $query_style = 'byexp'; + my $stage; + my $filter; + my $mjd; + # Confirm that we only have one stage/filter/mjd + if ($fpa_id ne 'Not_Set') { # We only need to check things that aren't the known odd case. + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1; + $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1; + $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1; + } + if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) { + $stage = (keys(%{ $temp_hash{STAGE} }))[0]; + } + else { + exit_with_failure(21,"Too many STAGEs specified"); + } + if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) { + $filter = (keys(%{ $temp_hash{FILTER} }))[0]; + } + else { + exit_with_failure(21,"Too many FILTERs specified"); + } + if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) { + $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0]; + } + else { + exit_with_failure(21,"Too many MJD-OBS specified"); + } + } + + # Set up a rowList with default values + my @rowList; + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i]; + $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i]; + $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i]; + $rowList[$i]->{COORD_MASK} = 0; + # Set default values + $query{$fpa_id}{BAD_COMPONENT}[$i] = 1; + $query{$fpa_id}{IMAGE}[$i] = 'no_image'; + $query{$fpa_id}{MASK}[$i] = 'no_mask'; + $query{$fpa_id}{WEIGHT}[$i] = 'no_weight'; + $query{$fpa_id}{PSF}[$i] = 'no_psf'; + + $query{$fpa_id}{STAGE_ID}[$i] = 'no_id'; + $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb'; + $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic'; + $query{$fpa_id}{MAGICKED}[$i] = 'no_magic'; + $query{$fpa_id}{CATALOG}[$i] = 'no_catalog'; + $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component'; + $query{$fpa_id}{CLASS_ID}[$i] = 'no_class'; + + $query{$fpa_id}{STATE}[$i] = 'no_state'; + $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate'; + $query{$fpa_id}{FAULT}[$i] = 'no_fault'; + $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate'; + } + + # Determine the query style for this fpa_id + if ($fpa_id =~ /o.*g.*o/) { + $query_style = 'byexp'; + } + elsif ($fpa_id =~ /\d+/) { + $query_style = 'byid'; + } + elsif ($fpa_id eq 'Not_Set') { + # no matching file was found skip + next; + } + else { + exit_with_failure(21,"Parse error in request file"); + } + # Set common request components + my $option_mask |= 1; + $option_mask |= $PSTAMP_SELECT_IMAGE; + $option_mask |= $PSTAMP_SELECT_MASK; + $option_mask |= $PSTAMP_SELECT_VARIANCE; + $option_mask |= $PSTAMP_SELECT_PSF; + my $need_magic = 1; + if ($stage eq 'stack') { + $need_magic = 0; + } + my $mjd_min = int $mjd; + my $mjd_max = $mjd + 1; + + my $req_filter; + if ($filter ne 'Not_Set') { + $req_filter = $filter . '%'; + } + + # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. + my $pstamp_images_ref = locate_images($ipprc,$imagedb, + \@rowList, + $query_style,$stage, + $fpa_id,undef,undef, + $option_mask,$need_magic, + $mjd_min,$mjd_max,$req_filter,undef,$verbose); + + foreach my $this_image_ref (@{ $pstamp_images_ref }) { + + if (0) { + foreach my $key (sort (keys %{ $this_image_ref } )) { + my $value = ${ $this_image_ref }{$key}; + if ($key eq 'row_index') { + $value = join ' ', @{ $this_image_ref->{$key} }; + } + print "$this_image_ref $key $value\n"; + } } - $result = 0; + + + foreach my $valid_index (@{ $this_image_ref->{row_index} }) { + $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; + $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; + $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; + $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; + $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; + $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; + $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; + $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; + + if (exists($this_image_ref->{astrom})) { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; + } + else { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; + } + if (exists($this_image_ref->{class_id})) { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; + + } + else { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; + } + + $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; + if (exists($this_image_ref->{data_state})) { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; + } + else { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; + } + $query{$fpa_id}{FAULT}[$valid_index] = 0; + $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked}; + if ($stage eq 'chip') { + $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; + } + + # Determine if the data exists. + if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { + + # image is gone and it's not coming back + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; + } + elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; + } + elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; + + # updating stacks isn't implemented + if (($stage eq 'stack')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + # updating old burntool data isn't implemented + elsif ($stage eq 'chip') { + if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and + (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + } + } # End determining error faults. + } + } + } +} + +my %update_request; +my %rows_for_component; +my %faulted_rows; + +# now build a hash using image name as key which contains the list of rows for that image +foreach my $fpa_id (keys %query) { + for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) { +# print WISDOM "$fpa_id\t"; +# foreach my $key (keys %{ $query{$fpa_id} }) { +# print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; +# } +# print WISDOM "\n"; +# my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; +# if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { +# if ($data_state ne 'full') { + +# @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = +# ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], +# $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); +# } +# } + if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { + push @{ $rows_for_component{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - } - } - elsif ($fault == $PSTAMP_NOT_AVAILABLE) { - # Failed to run correctly, which means that we need to queue a job and flag data for updating. - # first create a parent job for the actual detectabilty query - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/1_"; - $command .= " -job_type detect_query -state run -fault 0 -is_parent"; - $command .= " -rownum 1"; - - my $parent_job_id; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if ($success) { - # pstamptool returns the job_id of the new job - $parent_job_id = join "", @$stdout_buf; - chomp $parent_job_id; - if ($parent_job_id) { - $result = 0; + $faulted_rows{$query{$fpa_id}{ROWNUM}[$i]} = $query{$fpa_id}{FAULT}[$i]; + } + } +} + +my $job_num = 0; +foreach my $fpa_id (keys %rows_for_component) { + foreach my $component (keys % {$rows_for_component{$fpa_id} } ) { + my $i = $rows_for_component{$fpa_id}{$component}[0]; + my $dep_id = 0; + my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; + if ($data_state ne 'full') { + # Queue this data to be updated + my $stage = $query{$fpa_id}{STAGE}[$i]; + my $stage_id = $query{$fpa_id}{STAGE_ID}[$i]; + my $component = $query{$fpa_id}{STAGE_ID}[$i]; + print "Need to UPDATE $stage $stage_id $component from $data_state\n"; + $dep_id = queue_update_run($req_id, $outdir, $label, $data_state, $stage, $stage_id, $component, 0, $imagedb); + } + + my $outputBase = "$outdir/$job_num" . "_"; + my $params_file = $outputBase . "params.mdc"; + + open PARAMS, ">$params_file" or my_die("failed to create $params_file", $PS_EXIT_UNKNOWN_ERROR); + print PARAMS "dqueryJobParams METADATA\n"; + foreach my $key (keys %{ $query{$fpa_id} } ) { + my $value = $query{$fpa_id}{$key}[0]; + print PARAMS "\t$key\t\tSTR\t$value\n"; + } + print PARAMS "END\n\n"; + print PARAMS "dqueryCoord MULTI\n"; + + foreach my $row ( @{ $rows_for_component{$fpa_id}{$component} }) { + my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG); + print PARAMS "\ndqueryCoord METADATA\n"; + foreach my $key (@coordkeys) { + my $value = $query{$fpa_id}{$key}[$row]; + print PARAMS "\t$key\t\tSTR\t$value\n"; + } + print PARAMS "END\n"; + } + close(PARAMS); + + # use first rownum for this component as the rownum for the job + my $rownum = $query{$fpa_id}{ROWNUM}[$rows_for_component{$fpa_id}{$component}[0]]; + + my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outputBase -job_type detect_query -state run -rownum $rownum"; + $command .= " -dep_id $dep_id" if $dep_id; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if ($success) { + my $job_id = join "", @$stdout_buf; + print " Queued job: $job_id\n"; } else { - print STDERR "pstamptool did not return a valid job_id for parent job\n"; - $result = $PS_EXIT_UNKNOWN_ERROR; + my $exit_status = $error_code >> 8; + $exit_status = $PS_EXIT_UNKNOWN_ERROR unless $exit_status; + my_die("Unable to perform $command error code: $error_code", $exit_status); } } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - } - # now create child jobs for each dependent. All these jobs do is finish once the dependency - # is satisfied - if (!$result) { - # Get the dependency id for the data we need to have updated. - my $job_num = 2; - open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR); - while () { - my $data_to_update = $_; - chomp($data_to_update); - my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update); - - # Link this request to a job and link that job to the dependent - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/${job_num}_"; - $command .= " -job_type child -state run -fault 0 -parent_id $parent_job_id"; - $command .= " -rownum 1"; # XXX: we should choose a correct rownum - $command .= " -dep_id $dep_id" if $dep_id; - - $job_num++; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if ($success) { - $job_id = join "", @$stdout_buf; - chomp $job_id; - $result = 0; - } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - last; - } - } - close(UPDATE_REQUEST); - } - } - else { - my_die ("Parse fault!!", $fault); - } - # This does not set the request state to stop. That will happen with the request_finish.pl script, - # which will notice that we've inserted the stopped job and decide we're finished. Easy enough. - { - my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product"; - $command .= " -set_fault $result" if $result; - + print STDERR "skipping $command\n"; + } + $job_num++; + } +} + +# XXX: flesh this out once we have a request without duplicate ROWNUMS +foreach my $rownum (keys %faulted_rows ) { + my $fault = $faulted_rows{$rownum}; + $fault = $PSTAMP_NO_IMAGE_MATCH if ($fault eq 'no_fault'); + print STDERR "insert faulted job for $rownum: $fault\n"; +} + +{ + my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product"; + # $command .= " -set_fault $result" if $result; + + unless ($no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR); - } - } -} else { - # We are running as a job presumably because an input needed - # to be regenerated - if ($fault) { - # in some cases we will need to finish off the request - my_die ("Run fault!!", $fault); - } - if (-e $response_file) { - rename $response_file, "$outdir/response.${job_id}.fits"; - } -} - - + my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR); + } + } else { + print STDERR "Skipping $command\n"; + } +} + exit 0; - -# If we have to queue an update run, do so and create a new dependent sub queue_update_run { - my ($req_id, $job_id, $outdir, $label, $data_to_update) = @_; - - my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update; + my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_; + +# my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update; if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { @@ -293,14 +479,20 @@ return($dep_id); } - - + sub my_die { - my $msg = shift; - my $fault = shift; - - carp $msg; - - # we don't fault the request here pstamp_parser_run.pl handles that if necessary - exit $fault; -} + my $message = shift; + my $exit_code = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp("$message"); + exit($exit_code); +} + +sub exit_with_failure { + my $status = shift; + my $message = shift; + my_die($message, $status); +} +