Index: trunk/pstamp/scripts/pstampparse.pl =================================================================== --- trunk/pstamp/scripts/pstampparse.pl (revision 24855) +++ trunk/pstamp/scripts/pstampparse.pl (revision 24921) @@ -13,4 +13,5 @@ use PS::IPP::PStamp::RequestFile qw( :standard ); use PS::IPP::PStamp::Job qw( :standard ); +use File::Temp qw(tempfile); my $verbose; @@ -22,4 +23,6 @@ my $out_dir; my $product; +my $save_temps; +my $no_update; GetOptions( @@ -32,4 +35,6 @@ 'dbserver=s'=> \$dbserver, 'verbose' => \$verbose, + 'save-temps'=> \$save_temps, + 'no-update' => \$no_update, ); @@ -56,4 +61,5 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1); +my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); @@ -62,4 +68,6 @@ exit ($PS_EXIT_CONFIG_ERROR); } + +$no_update = 1 if $mode eq "list_job"; my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -87,5 +95,5 @@ die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1"); -if ($req_id) { +if ($req_id and !$no_update) { my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; $command .= " -outProduct $product"; @@ -117,25 +125,10 @@ } +my @rowList; my $num_jobs = 0; +my $imageList; +my $stage; +my $need_magic; foreach my $row (@$rows) { - my $rownum = $row->{ROWNUM}; - my $job_type = $row->{JOB_TYPE}; - my $project = $row->{PROJECT}; - my $req_type = $row->{REQ_TYPE}; - my $img_type = $row->{IMG_TYPE}; - my $id = $row->{ID}; - my $class_id = $row->{CLASS_ID}; - my $filter = $row->{REQFILT}; - my $mjd_min = $row->{MJD_MIN}; - my $mjd_max = $row->{MJD_MAX}; - my $x = $row->{CENTER_X}; - my $y = $row->{CENTER_Y}; - my $w = $row->{WIDTH}; - my $h = $row->{HEIGHT}; - my $coord_mask = $row->{COORD_MASK}; - my $option_mask= $row->{OPTION_MASK}; - - $class_id = "" if ($class_id eq "null" or $class_id eq "all"); - # XXX: TODO: sanity check all parameters @@ -144,156 +137,75 @@ # add a job with the proper fault code. If there is a db or config error we should probably just # trash the request. + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + + # parameters that select the images of interest + my $project = $row->{PROJECT}; + my $req_type = $row->{REQ_TYPE}; + $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $class_id = $row->{CLASS_ID}; + my $component = $row->{COMPONENT}; + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $option_mask= $row->{OPTION_MASK}; + + die "region of interest is required to make postage stamps" + if (($job_type eq "stamp") and ! validROI($row)); + + my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + $class_id = "" if (defined($class_id) and ($class_id eq "null" or $class_id eq "all")); + $component = "" if (defined($component) and ($component eq "null" or $component eq "all")); + die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri"); + + # note: resolve_project remembers the last project returned so avoids running + # pstamptool every time my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); die "project $project not found\n" unless $proj_hash; - my $image_db = $proj_hash->{dbname}; - my $camera = $proj_hash->{camera}; - my $need_magic = $proj_hash->{need_magic}; - - my $roi_string; - # XXX we're depending on other code to insure valid values for roi components - # this is checked in and ppstamp but I should check here so that we don't get that far, - # but not today - if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { - $roi_string = "-pixcenter $x $y"; + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + $need_magic = $proj_hash->{need_magic}; + + # collect rows with the same images of interest in a list so that they + # can be processed optimially + if (@rowList) { + my $firstRow = $rowList[0]; + if ($skycenter and same_images_of_interest($row, $firstRow)) { + push @rowList, $row; + # On to the next row + next; } else { - $roi_string = "-skycenter $x $y"; - } - if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { - $roi_string .= " -pixrange $w $h"; - } else { - $roi_string .= " -arcrange $w $h"; - } - } - - die "region of interest is required to make postage stamps" - if (($job_type eq "stamp") && !defined($roi_string)); - - # find the uris for this request + # Process the jobs for this set of rows + $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList); + @rowList = (); + } + } if ($req_type eq "bycoord") { - die "region of interest is required for request type bycoord" if !defined($roi_string) ; - die "center must be specified in sky coordintes for bycoord" - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS); - } - - # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification - my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id, - $x, $y, $mjd_min, $mjd_max, $filter, $verbose); - - if (!$images) { + die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter); + die "lookup bycoord is not yet implemented"; + } else { + + # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this + # request specification. An array reference is returned + my ($x, $y); + $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $class_id, $skycenter, + $x, $y, $mjd_min, $mjd_max, $filter, $verbose); + } + + if (!$imageList or !@$imageList) { print STDERR "no matching images found for row $rownum\n"; next; } - if ($mode eq "list_uri") { - foreach my $image (@$images) { - print "$image->{image}\n"; - } - } elsif ($job_type eq "get_image") { - # XXX TODO: Get rid of this block and use the same code as the stamp jobs - # XXX This doesn't work to get the mask and weight images - my $listfile = "$out_dir/filelist"; - - - # map our img_type to the Data Store file types. - my %filelist_img_types = ( "raw" => "chip", - "chip" => "chipproc", - "warp" => "warp", - "stack"=>"stack", - "diff" => "diff"); - - my $filelist_img_type = $filelist_img_types{$img_type}; - - open LISTFILE, ">$listfile" - or die "failed to open file list: $listfile while parsing get_image request $req_id"; - - foreach my $image (@$images) { - my $class_id = $image->{class_id} ? $image->{class_id} : ""; - print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n"; - } - close LISTFILE; - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image" - . " -uri $listfile -outputBase $out_dir -rownum $rownum"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; - if ($mode eq "list_job") { - print "$command\n"; - } else { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - die "failed to queue job for request $req_id: rownum: $rownum"; - } - } - } else { - # sequence number for the the job for a request spec. Not to be confused with job_id - my $job_num = 0; - - # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images - # use -list instead of -file - foreach my $image (@$images) { - my $uri = $image->{image}; - if (($img_type ne "stack") and ($need_magic and !$image->{magicked})) { - print STDERR "skippping non-magicked image $uri\n" if $verbose; - next; - } - my $exp_id = $image->{exp_id}; - - my $args = $roi_string ? $roi_string : ""; - $args .= " -class_id $class_id" if $class_id; - - $job_num++; - - my $output_base = "$out_dir/${rownum}_${job_num}"; - - # add astrometry file for raw and chip images if one is available - if (($img_type eq "chip") || ($img_type eq "raw")) { - $args .= " -astrom $image->{astrom}" if $image->{astrom}; - } - - if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { - $args .= " -mask $image->{mask}"; - } - if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { - $args .= " -weight $image->{weight}"; - } - - # XXX: TODO: here is where we need to check whether or not the source inputs still exist - # and if not, queue an update job and set the job state appropriately. - - my $newState = "run"; - - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" - . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum" - . " -state $newState"; - $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; - - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } else { - # mode eq "queue_job" - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - # should we keep going for other uris? If so how do we report that some - # of the work that the request wanted isn't going to get done - die "failed to queue job for request $req_id"; - } - } - } - } + push @rowList, $row; +} + +if (@rowList) { + $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList); } @@ -309,2 +221,274 @@ exit 0; } + + +sub makeJobsForRow +{ + my $row = shift; + my $imageList = shift; + my $have_skycells = shift; + my $need_magic = shift; + + # loop over images + my $job_num = 0; + foreach my $image (@$imageList) { + my $component; + my $class_id; # get rid of this use componet + if ($have_skycells) { + $component = $image->{skycell_id}; + } else { + $component = $image->{class_id}; + $class_id = $component; + } + + # skip this component if it is not in the list for this row + next if !$row->{components}->{$component}; + + my $rownum = $row->{ROWNUM}; + + my $roi_string; + + # XXX we're depending on other code to insure valid values for roi components + # this is checked in and ppstamp but I should check here so that we don't get that far, + # but not today + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $w = $row->{WIDTH}; + my $h = $row->{HEIGHT}; + my $coord_mask = $row->{COORD_MASK}; + if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { + if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { + $roi_string = "-pixcenter $x $y"; + } else { + $roi_string = "-skycenter $x $y"; + } + if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { + $roi_string .= " -pixrange $w $h"; + } else { + $roi_string .= " -arcrange $w $h"; + } + } + + my $uri = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + print STDERR "skippping non-magicked image $uri\n" if $verbose; + next; + } + my $exp_id = $image->{exp_id}; + + my $args = $roi_string ? $roi_string : ""; + $args .= " -class_id $class_id" if $class_id; + + $job_num++; + + my $output_base = "$out_dir/${rownum}_${job_num}"; + + # add astrometry file for raw and chip images if one is available + if (($stage eq "chip") || ($stage eq "raw")) { + $args .= " -astrom $image->{astrom}" if $image->{astrom}; + } + + if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) && $image->{mask} ) { + $args .= " -mask $image->{mask}"; + } + if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { + $args .= " -weight $image->{weight}"; + } + + # XXX: TODO: here is where we need to check whether or not the source inputs still exist + # and if not, queue an update job and set the job state appropriately. + + my $newState = "run"; + + $num_jobs++; + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum" + . " -state $newState"; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + + if ($mode eq "list_job") { + # this is a debugging mode, just print the pstamptool that would have run + # this is sort of like the mode -noupdate that some other tools support + print "$command\n"; + } else { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + die "failed to queue job for request $req_id"; + } + } + } + return $num_jobs; +} + +sub makeJobs +{ + my $mode = shift; + my $stage = shift; + my $need_magic = shift; + my $rowList = shift; + my $imageList = shift; + + my $firstRow = $rowList[0]; + my $job_type = $firstRow->{JOB_TYPE}; + + my $num_jobs = 0; + + if ($mode eq "list_uri") { + foreach my $image (@$imageList) { + print "$image->{image}\n"; + } + } elsif ($job_type eq "get_image") { + die "get_image jobs not implemented yet"; + } else { + my $thisRun; + + my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); + foreach my $row (@$rowList) { + print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; + $row->{components} = {}; + } + close $pointsList; + + my $have_skycells; + if (($stage eq "raw") or ($stage eq "chip")) { + $have_skycells = 0; + } else { + $have_skycells = 1; + } + + my $tess_dir_abs; + my $last_tess_id = ""; + while ($thisRun = getOneRun($stage, $imageList)) { + { + my $command = "$dvoImagesAtCoords $pointsListName"; + if ($have_skycells) { + my $tess_id = $thisRun->[0]->{tess_id}; + if ($tess_id ne $last_tess_id) { + $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); + $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); + $last_tess_id = $tess_id; + } + $command .= " -D CATDIR $tess_dir_abs"; + } else { + my $astrom = $thisRun->[0]->{astrom}; + die "no astrometry file found" if !$astrom; + $command .= " -astrom $astrom"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + my $rc = $error_code >> 8; + die "dvoImagesAtCoords failed: $rc"; + } + # now we have a list of row numbers and components + # eventually we might want to multiple stamp requests for the same image + # into the same ppstamp job but not yet. For now we will queue a new + my @lines = split "\n", join "", @$stdout_buf; + foreach my $line (@lines) { + # parse the line, ignoring the ra and dec + my ($rownum, undef, undef, $component) = split " ", $line; + + # I guess since we need this function we should be useing a hash for rowList + my $row = findRow($rownum, $rowList); + $row->{components}->{$component} = 1; + } + } + + + # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images + # use -list instead of -file + foreach my $row (@$rowList) { + $num_jobs += makeJobsForRow($row, $thisRun, $have_skycells, $need_magic); + } + } + } + return $num_jobs; +} + +sub get_run_id +{ + my $stage = shift; + my $image = shift; + + if ($stage eq "raw") { + return $image->{exp_id}; + } elsif ($stage eq "chip") { + return $image->{chip_id}; + } elsif ($stage eq "warp") { + return $image->{warp_id}; + } elsif ($stage eq "stack") { + return $image->{stack_id}; + } elsif ($stage eq "diff") { + return $image->{diff_id}; + } else { + die "unenexpected stage: $stage found"; + } +} + +# extract components from the imageList that have the same run id and return the list +sub getOneRun { + my $stage = shift; + my $imageList = shift; + + # return if array is empty + return undef if ! @$imageList; + + my $last_run_id = 0; + my @runList; + while ($imageList->[0]) { + my $run_id = get_run_id($stage, $imageList->[0]); + + last if ($last_run_id and ($run_id ne $last_run_id)); + + my $image = shift @$imageList; + push @runList, $image; + $last_run_id = $run_id; + } + return \@runList; +} + +sub same_images_of_interest { + my $r1 = shift; + my $r2 = shift; + + return 0 if ($r1->{project} ne $r2->{project}); + return 0 if ($r1->{job_type} ne $r2->{job_type}); + return 0 if ($r1->{req_type} ne $r2->{req_type}); + return 0 if ($r1->{img_type} ne $r2->{img_type}); + return 0 if ($r1->{id} ne $r2->{id}); +# XXX: turn this on when we change the request file format to change class_id into component +# return false if ($r1->{component} ne $r2->{component}); + + return 1; +} + +sub validROI +{ + my $row = shift; + return 0 if !defined $row->{CENTER_X}; + return 0 if !defined $row->{CENTER_Y}; + return 0 if !defined $row->{WIDTH}; + return 0 if !defined $row->{HEIGHT}; + + return 1; +} + +sub findRow +{ + my $rownum = shift; + my $rowList = shift; + + foreach my $row (@$rowList) { + return $row if $row->{ROWNUM} eq $rownum; + } + return undef; +}